public async Task <int> ExecuteAsync()
{
var srcStorage = (IStorage) new FileSystemStorage(mySource);
IReadOnlyCollection <string> srcFiles;
{
var validator = new Validator(myLogger, srcStorage, "Source");
var srcStorageFormat = await validator.ValidateStorageMarkersAsync();
var tagItems = await validator.LoadTagItemsAsync();
validator.DumpProducts(tagItems);
validator.DumpProperties(tagItems);
var(totalSize, files) = await validator.GatherDataFilesAsync();
var(statistics, _) = await validator.ValidateAsync(tagItems, files, srcStorageFormat, Validator.ValidateMode.Validate);
myLogger.Info($"[{DateTime.Now:s}] Done with source validation (size: {totalSize.ToKibibyte()}, files: {files.Count + tagItems.Count}, warnings: {statistics.Warnings}, errors: {statistics.Errors})");
if (statistics.HasProblems)
{
myLogger.Error("Found some issues in source storage, uploading was interrupted");
return(1);
}
srcFiles = tagItems.Select(x => x.Key).Concat(files).ToList();
}
var dstValidator = new Validator(myLogger, myStorage, "Destination");
var dstStorageFormat = await dstValidator.CreateOrValidateStorageMarkersAsync(myNewStorageFormat);
{
myLogger.Info($"[{DateTime.Now:s}] Checking file compatibility...");
var uploadFiles = new List <Tuple <string, string> >();
{
var statistics = new Statistics();
ILogger logger = new LoggerWithStatistics(myLogger, statistics);
var hash = SHA256.Create();
long existFiles = 0;
foreach (var srcFile in srcFiles)
{
logger.Info($" Checking {srcFile}");
var dstFile = TagUtil.IsDataFile(srcFile) && srcFile.ValidateAndFixDataPath(dstStorageFormat, out var fixedFile) == PathUtil.ValidateAndFixErrors.CanBeFixed
? fixedFile
: srcFile;
if (await myStorage.ExistsAsync(dstFile))
{
Interlocked.Increment(ref existFiles);
var dstLen = await myStorage.GetLengthAsync(dstFile);
var srcLen = await srcStorage.GetLengthAsync(srcFile);
if (srcLen != dstLen)
{
logger.Error($"The file {srcFile} length {srcLen} differs then the destination length {dstLen}");
}
else
{
var dstHash = await myStorage.OpenForReadingAsync(dstFile, stream => hash.ComputeHashAsync(stream));
var srcHash = await srcStorage.OpenForReadingAsync(srcFile, stream => hash.ComputeHashAsync(stream));
if (!srcHash.SequenceEqual(dstHash))
{
logger.Error($"The file {srcFile} hash {srcHash.ToHex()} differs then the destination hash {dstHash.ToHex()}");
}
}
}
else
{
uploadFiles.Add(Tuple.Create(srcFile, dstFile));
}
}
myLogger.Info($"[{DateTime.Now:s}] Done with compatibility (new files: {uploadFiles.Count}, same files: {existFiles}, warnings: {statistics.Warnings}, errors: {statistics.Errors})");
if (statistics.HasProblems)
{
myLogger.Error("Found some issues in source storage, uploading was interrupted");
return(1);
}
}
if (uploadFiles.Count > 0)
{
myLogger.Info($"[{DateTime.Now:s}] Uploading...");
long totalSize = 0;
foreach (var(srcFile, dstFile) in uploadFiles)
{
myLogger.Info($" Uploading {srcFile}");
await using var memoryStream = new MemoryStream();
await srcStorage.OpenForReadingAsync(srcFile, stream => stream.CopyToAsync(memoryStream));
await myStorage.CreateForWritingAsync(dstFile, TagUtil.IsTagFile(dstFile)?AccessMode.Private : AccessMode.Public, memoryStream);
Interlocked.Add(ref totalSize, memoryStream.Length);
}
await myStorage.InvalidateExternalServicesAsync();
myLogger.Info($"[{DateTime.Now:s}] Done with uploading (size: {totalSize.ToKibibyte()}, files: {uploadFiles.Count})");
}
}
return(0);
}
public void DataFileTest(bool expected, string file)
{
Assert.AreEqual(expected, TagUtil.IsDataFile(file.NormalizeSystem()));
}