public async Task <int> ExecuteAsync()
{
var srcStorage = (IStorage) new FileSystemStorage(mySource);
IReadOnlyCollection <string> srcFiles;
{
var validator = new Validator(myLogger, srcStorage, "Source");
var srcStorageFormat = await validator.ValidateStorageMarkersAsync();
var tagItems = await validator.LoadTagItemsAsync();
validator.DumpProducts(tagItems);
validator.DumpProperties(tagItems);
var(totalSize, files) = await validator.GatherDataFilesAsync();
var(statistics, _) = await validator.ValidateAsync(tagItems, files, srcStorageFormat, Validator.ValidateMode.Validate);
myLogger.Info($"[{DateTime.Now:s}] Done with source validation (size: {totalSize.ToKibibyte()}, files: {files.Count + tagItems.Count}, warnings: {statistics.Warnings}, errors: {statistics.Errors})");
if (statistics.HasProblems)
{
myLogger.Error("Found some issues in source storage, uploading was interrupted");
return(1);
}
srcFiles = tagItems.Select(x => x.Key).Concat(files).ToList();
}
var dstValidator = new Validator(myLogger, myStorage, "Destination");
var dstStorageFormat = await dstValidator.CreateOrValidateStorageMarkersAsync(myNewStorageFormat);
{
myLogger.Info($"[{DateTime.Now:s}] Checking file compatibility...");
var uploadFiles = new List <Tuple <string, string> >();
{
var statistics = new Statistics();
ILogger logger = new LoggerWithStatistics(myLogger, statistics);
var hash = SHA256.Create();
long existFiles = 0;
foreach (var srcFile in srcFiles)
{
logger.Info($" Checking {srcFile}");
var dstFile = TagUtil.IsDataFile(srcFile) && srcFile.ValidateAndFixDataPath(dstStorageFormat, out var fixedFile) == PathUtil.ValidateAndFixErrors.CanBeFixed
? fixedFile
: srcFile;
if (await myStorage.ExistsAsync(dstFile))
{
Interlocked.Increment(ref existFiles);
var dstLen = await myStorage.GetLengthAsync(dstFile);
var srcLen = await srcStorage.GetLengthAsync(srcFile);
if (srcLen != dstLen)
{
logger.Error($"The file {srcFile} length {srcLen} differs then the destination length {dstLen}");
}
else
{
var dstHash = await myStorage.OpenForReadingAsync(dstFile, stream => hash.ComputeHashAsync(stream));
var srcHash = await srcStorage.OpenForReadingAsync(srcFile, stream => hash.ComputeHashAsync(stream));
if (!srcHash.SequenceEqual(dstHash))
{
logger.Error($"The file {srcFile} hash {srcHash.ToHex()} differs then the destination hash {dstHash.ToHex()}");
}
}
}
else
{
uploadFiles.Add(Tuple.Create(srcFile, dstFile));
}
}
myLogger.Info($"[{DateTime.Now:s}] Done with compatibility (new files: {uploadFiles.Count}, same files: {existFiles}, warnings: {statistics.Warnings}, errors: {statistics.Errors})");
if (statistics.HasProblems)
{
myLogger.Error("Found some issues in source storage, uploading was interrupted");
return(1);
}
}
if (uploadFiles.Count > 0)
{
myLogger.Info($"[{DateTime.Now:s}] Uploading...");
long totalSize = 0;
foreach (var(srcFile, dstFile) in uploadFiles)
{
myLogger.Info($" Uploading {srcFile}");
await using var memoryStream = new MemoryStream();
await srcStorage.OpenForReadingAsync(srcFile, stream => stream.CopyToAsync(memoryStream));
await myStorage.CreateForWritingAsync(dstFile, TagUtil.IsTagFile(dstFile)?AccessMode.Private : AccessMode.Public, memoryStream);
Interlocked.Add(ref totalSize, memoryStream.Length);
}
await myStorage.InvalidateExternalServicesAsync();
myLogger.Info($"[{DateTime.Now:s}] Done with uploading (size: {totalSize.ToKibibyte()}, files: {uploadFiles.Count})");
}
}
return(0);
}
public async Task <Tuple <Statistics, long> > ValidateAsync(
[NotNull] IEnumerable <KeyValuePair <string, Tag> > items,
[NotNull] IReadOnlyCollection <string> files,
StorageFormat storageFormat,
ValidateMode mode,
bool verifyAcl = false)
{
myLogger.Info($"[{DateTime.Now:s}] Validating storage{myId}...");
var fix = mode == ValidateMode.Fix || mode == ValidateMode.Delete;
var statistics = new Statistics();
ILogger logger = new LoggerWithStatistics(myLogger, statistics);
files = await ValidateDataFilesAsync(logger, files, storageFormat, fix);
var tree = await CreateDirectoryTreeAsync(files);
foreach (var item in items)
{
var tagFile = item.Key;
var tag = item.Value;
logger.Info($" Validating {tagFile}");
var isDirty = false;
if (!tag.Product.ValidateProduct())
{
logger.Error($"Invalid product {tag.Product} in file {tagFile}");
}
if (!tag.Version.ValidateVersion())
{
logger.Error($"Invalid version {tag.Version} in file {tagFile}");
}
if (tag.CreationUtcTime == DateTime.MinValue)
{
logger.Error($"The empty creation time in {tagFile}");
if (fix)
{
var newCreationUtcTime = TryFixCreationTime(tag);
if (newCreationUtcTime != null)
{
logger.Fix($"The creation time will be assigned to tag {tagFile}");
isDirty = true;
tag.CreationUtcTime = newCreationUtcTime.Value;
}
}
}
if (tag.Directories == null || tag.Directories.Length == 0)
{
logger.Error($"The empty directory list in {tagFile}");
if (fix)
{
logger.Fix($"The tag will be deleted {tagFile}");
await myStorage.DeleteAsync(tagFile);
continue;
}
}
else
{
for (var index = 0; index < tag.Directories.Length; index++)
{
var dir = tag.Directories[index];
switch (dir.ValidateAndFixDataPath(StorageFormat.Normal, out var fixedDir))
{
case PathUtil.ValidateAndFixErrors.Ok:
break;
case PathUtil.ValidateAndFixErrors.Error:
logger.Error($"The tag directory {dir} from {tagFile} has invalid format");
break;
case PathUtil.ValidateAndFixErrors.CanBeFixed:
logger.Error($"The tag directory {dir} from {tagFile} has invalid format");
if (fix)
{
isDirty = true;
tag.Directories[index] = fixedDir;
}
break;
default:
throw new ArgumentOutOfRangeException();
}
var dstDir = dir.ValidateAndFixDataPath(storageFormat, out fixedDir) == PathUtil.ValidateAndFixErrors.CanBeFixed ? fixedDir : dir;
var node = tree;
foreach (var part in dstDir.GetPathComponents())
{
node = node?.Lookup(part);
}
if (node == null)
{
logger.Error($"The directory {dir} from {tagFile} id wasn't found");
}
else
{
node.IncrementReferences();
}
}
}
if (isDirty)
{
logger.Info($"The tag file {tagFile} will be overwritten");
await using var stream = new MemoryStream();
await TagUtil.WriteTagScriptAsync(tag, stream);
await myStorage.CreateForWritingAsync(tagFile, AccessMode.Private, stream);
}
}
var deleted = await ValidateUnreachableAsync(logger, tree, mode);
if (verifyAcl)
{
await ValidateAclAsync(logger, files, fix);
}
return(Tuple.Create(statistics, deleted));
}