private async Task <List <FileData> > EnumerateDownloadableFilesFromHashList(string path)
{
HashList = await SmartCopy.GetHashList(remote, true);
List <FileData> workingList = HashList.Where(f => f.RelativePath.StartsWith(path)).ToList();
OnReportTotalFilesCount?.Invoke(workingList.Count);
var retv = new List <FileData>();
foreach (var a in HashList)
{
string hash = "";
if (local.Exists(a.RelativePath))
{
local.Read(a.RelativePath, async(stream) => {
await Task.Yield();
hash = await GetHash(stream);
}).Wait();
}
var gzSuffix = "";
if (options.UseGZip)
{
gzSuffix = GZIP_FILE_SUFFIX;
}
while ((a.Hash != hash || !local.Exists(a.RelativePath)))
{
if (remote.Exists(a.RelativePath + gzSuffix))
{
retv.Add(a);
}
}
}
return(retv);
}
/// <summary>
/// Copies files from local/origin accessor to remote/destination accessor
/// Uploads data from the local to the remote accessor
/// </summary>
public async Task MirrorUp(string path = "")
{
if (remote == null)
{
throw new NullReferenceException("The Remote server was not specified, do call SetRemote(IFileAccessor) to specify it.");
}
var totalFilesCount = CountFiles(local, path);
OnReportTotalFilesCount?.Invoke(totalFilesCount);
await Mirror(local, remote, path, MirrorWay.Up);
}
/// <summary>
/// Copies files remote/destination from accessor to local/origin accessor
/// Downloads data from the remote to the local accessor
/// </summary>
public async Task MirrorDown(string path = "")
{
if (remote == null)
{
throw new NullReferenceException("The Remote server was not specified, do call SetRemote(IFileAccessor) to specify it.");
}
if (options.UseHashList)
{
await MirrorFromList(path);
}
else
{
var totalFilesCount = CountFiles(remote, path);
OnReportTotalFilesCount?.Invoke(totalFilesCount);
await Mirror(remote, local, path, MirrorWay.Down);
}
}
private async Task MirrorFromList(string path)
{
HashList = await SmartCopy.GetHashList(remote, true);
int numWorkers = options.Multithreaded ? options.NumWorkers : 1;
var wq = new WorkQueuer("SmartCopy_Operation", numWorkers, false);
var bufferSize = (int)options.BufferSize / options.NumWorkers;
if (bufferSize < 0)
{
bufferSize = 256 * 1024;
}
List <FileData> workingList = HashList.Where(f => f.RelativePath.StartsWith(path)).ToList();
workingList = workingList.GroupBy(f => f.RelativePath).Select(g => g.First()).ToList();
workingList.RemoveAll(f => Excludes.Any(excl => CheckMatch(f.RelativePath, excl)));
OnReportTotalFilesCount?.Invoke(workingList.Count);
foreach (var a in HashList)
{
wq.Enqueue(async() => {
await Task.Yield();
if (a.RelativePath == HASHLIST_FILENAME || Excludes.Any(excl => CheckMatch(a.RelativePath, excl)))
{
return;
}
string hash = "";
if (local.Exists(a.RelativePath))
{
await local.Read(a.RelativePath, async(stream) => {
await Task.Yield();
hash = await GetHash(stream);
});
}
var processed = false;
var gzSuffix = "";
if (options.UseGZip)
{
gzSuffix = GZIP_FILE_SUFFIX;
}
int maxTries = 10;
while ((a.Hash != hash || !local.Exists(a.RelativePath)) && maxTries-- > 0)
{
processed = true;
if (remote.Exists(a.RelativePath + gzSuffix))
{
await remote.Read(a.RelativePath + gzSuffix, async(downStream) => {
await Task.Yield();
//local.Delete(a.RelativePath);
await local.Write(a.RelativePath + "_$ft_new", async(fileStream) => {
await Task.Yield();
if (options.UseGZip)
{
downStream = new GZipStream(downStream, CompressionMode.Decompress);
}
await downStream.CopyToAsync(fileStream, bufferSize);
});
var oldTempName = a.RelativePath + "_$ft_old-" + IntEx.GenerateShortRid();
if (local.Exists(a.RelativePath))
{
local.Rename(a.RelativePath, oldTempName);
}
local.Rename(a.RelativePath + "_$ft_new", a.RelativePath);
if (local.Exists(a.RelativePath))
{
await local.Read(a.RelativePath, async(stream) => {
await Task.Yield();
hash = await GetHash(stream);
});
}
try {
if (local.Exists(oldTempName))
{
local.Delete(oldTempName);
}
} catch (Exception x) {
local.Hide(oldTempName);
}
if (a.RelativePath.EndsWith(".dll"))
{
}
});
}
await local.Read(a.RelativePath, async(stream) => {
await Task.Yield();
hash = await GetHash(stream);
});
if (a.Hash != hash)
{
Console.Error.Write($"Hash Mismatch: {a.RelativePath}{a.Hash}/{hash}");
}
}
OnReportProcessedFile?.Invoke(processed, a.RelativePath);
}, async(ex) => {
await Task.Yield();
OnFileCopyException?.Invoke(a.RelativePath, ex);
});
}
wq.Start();
await wq.Stop(true);
DeleteExtras(local, path, workingList, true);
}
