private async Task <List <FileData> > EnumerateDownloadableFilesFromHashList(string path)
{
HashList = await SmartCopy.GetHashList(remote, true);
List <FileData> workingList = HashList.Where(f => f.RelativePath.StartsWith(path)).ToList();
OnReportTotalFilesCount?.Invoke(workingList.Count);
var retv = new List <FileData>();
foreach (var a in HashList)
{
string hash = "";
if (local.Exists(a.RelativePath))
{
local.Read(a.RelativePath, async(stream) => {
await Task.Yield();
hash = await GetHash(stream);
}).Wait();
}
var gzSuffix = "";
if (options.UseGZip)
{
gzSuffix = GZIP_FILE_SUFFIX;
}
while ((a.Hash != hash || !local.Exists(a.RelativePath)))
{
if (remote.Exists(a.RelativePath + gzSuffix))
{
retv.Add(a);
}
}
}
return(retv);
}
private async Task <List <FileData> > EnumerateDownloadableFiles(IFileSystem origin, IFileSystem destination, string path = "")
{
List <FileData> retv = new List <FileData>();
if (options.UseHashList)
{
HashList = await SmartCopy.GetHashList(this.remote, true);
}
workedFiles = 0;
foreach (var f in origin.GetFilesIn(path))
{
if (f == HASHLIST_FILENAME)
{
continue;
}
if (Excludes.Any(excl => CheckMatch(f, excl)))
{
continue;
}
var changed = CopyDecisionCriteria != null ?
await CopyDecisionCriteria(f) :
await Changed(origin, destination, f);
if (changed)
{
retv.Add(new FileData {
RelativePath = f,
Date = origin.GetLastModified(f)?.Ticks ?? 0,
Hash = "", // GetHash(origin, f),
Length = origin.GetSize(f)
});
}
}
if (options.Recursive)
{
foreach (var dir in origin.GetDirectoriesIn(path))
{
destination.MkDirs(dir);
var newAdds = await EnumerateDownloadableFiles(origin, destination, dir);
foreach (var a in newAdds)
{
retv.Add(a);
}
}
}
return(retv);
}
private async Task Mirror(IFileSystem origin, IFileSystem destination, string path, MirrorWay way)
{
int numWorkers = options.Multithreaded ? options.NumWorkers : 1;
var wq = new WorkQueuer("SmartCopy_Operation", numWorkers, false);
if (options.UseHashList)
{
HashList = await SmartCopy.GetHashList(destination, way == MirrorWay.Down);
}
workedFiles = 0;
var files = await EnumerateFilesForMirroring(origin, destination, path, way);
foreach (var f in files)
{
wq.Enqueue(async() => {
await Task.Yield();
OnFileStaged?.Invoke(f.RelativePath);
if (f.Changed)
{
switch (way)
{
case MirrorWay.Up:
await processFileUp(origin, destination, f.RelativePath);
break;
case MirrorWay.Down:
await processFileDown(origin, destination, f.RelativePath);
break;
}
}
OnReportProcessedFile?.Invoke(f.Changed, f.RelativePath);
}, async x => {
await Task.Yield();
OnFileCopyException?.Invoke(f.RelativePath, x);
});
}
if (options.AllowDelete)
{
int DeleteLimit = 15;
destination.ForFilesIn(path, (f) => {
if (DeleteLimit < 1)
{
return;
}
if (Excludes.Any(x => CheckMatch(f, x)))
{
return;
}
if (!origin.Exists(f))
{
if (DeleteLimit-- > 0)
{
destination.Delete(f);
return;
}
}
});
}
wq.Start();
await wq.Stop(true);
if (Debugger.IsAttached)
{
Debugger.Break();
}
await SaveHashList(origin, destination);
}
private async Task MirrorFromList(string path)
{
HashList = await SmartCopy.GetHashList(remote, true);
int numWorkers = options.Multithreaded ? options.NumWorkers : 1;
var wq = new WorkQueuer("SmartCopy_Operation", numWorkers, false);
var bufferSize = (int)options.BufferSize / options.NumWorkers;
if (bufferSize < 0)
{
bufferSize = 256 * 1024;
}
List <FileData> workingList = HashList.Where(f => f.RelativePath.StartsWith(path)).ToList();
workingList = workingList.GroupBy(f => f.RelativePath).Select(g => g.First()).ToList();
workingList.RemoveAll(f => Excludes.Any(excl => CheckMatch(f.RelativePath, excl)));
OnReportTotalFilesCount?.Invoke(workingList.Count);
foreach (var a in HashList)
{
wq.Enqueue(async() => {
await Task.Yield();
if (a.RelativePath == HASHLIST_FILENAME || Excludes.Any(excl => CheckMatch(a.RelativePath, excl)))
{
return;
}
string hash = "";
if (local.Exists(a.RelativePath))
{
await local.Read(a.RelativePath, async(stream) => {
await Task.Yield();
hash = await GetHash(stream);
});
}
var processed = false;
var gzSuffix = "";
if (options.UseGZip)
{
gzSuffix = GZIP_FILE_SUFFIX;
}
int maxTries = 10;
while ((a.Hash != hash || !local.Exists(a.RelativePath)) && maxTries-- > 0)
{
processed = true;
if (remote.Exists(a.RelativePath + gzSuffix))
{
await remote.Read(a.RelativePath + gzSuffix, async(downStream) => {
await Task.Yield();
//local.Delete(a.RelativePath);
await local.Write(a.RelativePath + "_$ft_new", async(fileStream) => {
await Task.Yield();
if (options.UseGZip)
{
downStream = new GZipStream(downStream, CompressionMode.Decompress);
}
await downStream.CopyToAsync(fileStream, bufferSize);
});
var oldTempName = a.RelativePath + "_$ft_old-" + IntEx.GenerateShortRid();
if (local.Exists(a.RelativePath))
{
local.Rename(a.RelativePath, oldTempName);
}
local.Rename(a.RelativePath + "_$ft_new", a.RelativePath);
if (local.Exists(a.RelativePath))
{
await local.Read(a.RelativePath, async(stream) => {
await Task.Yield();
hash = await GetHash(stream);
});
}
try {
if (local.Exists(oldTempName))
{
local.Delete(oldTempName);
}
} catch (Exception x) {
local.Hide(oldTempName);
}
if (a.RelativePath.EndsWith(".dll"))
{
}
});
}
await local.Read(a.RelativePath, async(stream) => {
await Task.Yield();
hash = await GetHash(stream);
});
if (a.Hash != hash)
{
Console.Error.Write($"Hash Mismatch: {a.RelativePath}{a.Hash}/{hash}");
}
}
OnReportProcessedFile?.Invoke(processed, a.RelativePath);
}, async(ex) => {
await Task.Yield();
OnFileCopyException?.Invoke(a.RelativePath, ex);
});
}
wq.Start();
await wq.Stop(true);
DeleteExtras(local, path, workingList, true);
}