public async Task <int> ExecuteAsync()
        {
            var srcStorage = (IStorage) new FileSystemStorage(mySource);

            IReadOnlyCollection <string> srcFiles;
            {
                var validator        = new Validator(myLogger, srcStorage, "Source");
                var srcStorageFormat = await validator.ValidateStorageMarkersAsync();

                var tagItems = await validator.LoadTagItemsAsync();

                validator.DumpProducts(tagItems);
                validator.DumpProperties(tagItems);
                var(totalSize, files) = await validator.GatherDataFilesAsync();

                var(statistics, _) = await validator.ValidateAsync(tagItems, files, srcStorageFormat, Validator.ValidateMode.Validate);

                myLogger.Info($"[{DateTime.Now:s}] Done with source validation (size: {totalSize.ToKibibyte()}, files: {files.Count + tagItems.Count}, warnings: {statistics.Warnings}, errors: {statistics.Errors})");
                if (statistics.HasProblems)
                {
                    myLogger.Error("Found some issues in source storage, uploading was interrupted");
                    return(1);
                }

                srcFiles = tagItems.Select(x => x.Key).Concat(files).ToList();
            }

            var dstValidator     = new Validator(myLogger, myStorage, "Destination");
            var dstStorageFormat = await dstValidator.CreateOrValidateStorageMarkersAsync(myNewStorageFormat);

            {
                myLogger.Info($"[{DateTime.Now:s}] Checking file compatibility...");
                var uploadFiles = new List <Tuple <string, string> >();

                {
                    var     statistics = new Statistics();
                    ILogger logger     = new LoggerWithStatistics(myLogger, statistics);
                    var     hash       = SHA256.Create();

                    long existFiles = 0;
                    foreach (var srcFile in srcFiles)
                    {
                        logger.Info($"  Checking {srcFile}");
                        var dstFile = TagUtil.IsDataFile(srcFile) && srcFile.ValidateAndFixDataPath(dstStorageFormat, out var fixedFile) == PathUtil.ValidateAndFixErrors.CanBeFixed
              ? fixedFile
              : srcFile;
                        if (await myStorage.ExistsAsync(dstFile))
                        {
                            Interlocked.Increment(ref existFiles);
                            var dstLen = await myStorage.GetLengthAsync(dstFile);

                            var srcLen = await srcStorage.GetLengthAsync(srcFile);

                            if (srcLen != dstLen)
                            {
                                logger.Error($"The file {srcFile} length {srcLen} differs then the destination length {dstLen}");
                            }
                            else
                            {
                                var dstHash = await myStorage.OpenForReadingAsync(dstFile, stream => hash.ComputeHashAsync(stream));

                                var srcHash = await srcStorage.OpenForReadingAsync(srcFile, stream => hash.ComputeHashAsync(stream));

                                if (!srcHash.SequenceEqual(dstHash))
                                {
                                    logger.Error($"The file {srcFile} hash {srcHash.ToHex()} differs then the destination hash {dstHash.ToHex()}");
                                }
                            }
                        }
                        else
                        {
                            uploadFiles.Add(Tuple.Create(srcFile, dstFile));
                        }
                    }

                    myLogger.Info($"[{DateTime.Now:s}] Done with compatibility (new files: {uploadFiles.Count}, same files: {existFiles}, warnings: {statistics.Warnings}, errors: {statistics.Errors})");
                    if (statistics.HasProblems)
                    {
                        myLogger.Error("Found some issues in source storage, uploading was interrupted");
                        return(1);
                    }
                }

                if (uploadFiles.Count > 0)
                {
                    myLogger.Info($"[{DateTime.Now:s}] Uploading...");
                    long totalSize = 0;
                    foreach (var(srcFile, dstFile) in uploadFiles)
                    {
                        myLogger.Info($"  Uploading {srcFile}");
                        await using var memoryStream = new MemoryStream();
                        await srcStorage.OpenForReadingAsync(srcFile, stream => stream.CopyToAsync(memoryStream));

                        await myStorage.CreateForWritingAsync(dstFile, TagUtil.IsTagFile(dstFile)?AccessMode.Private : AccessMode.Public, memoryStream);

                        Interlocked.Add(ref totalSize, memoryStream.Length);
                    }

                    await myStorage.InvalidateExternalServicesAsync();

                    myLogger.Info($"[{DateTime.Now:s}] Done with uploading (size: {totalSize.ToKibibyte()}, files: {uploadFiles.Count})");
                }
            }

            return(0);
        }
示例#2
0
        public async Task <Tuple <Statistics, long> > ValidateAsync(
            [NotNull] IEnumerable <KeyValuePair <string, Tag> > items,
            [NotNull] IReadOnlyCollection <string> files,
            StorageFormat storageFormat,
            ValidateMode mode,
            bool verifyAcl = false)
        {
            myLogger.Info($"[{DateTime.Now:s}] Validating storage{myId}...");
            var     fix        = mode == ValidateMode.Fix || mode == ValidateMode.Delete;
            var     statistics = new Statistics();
            ILogger logger     = new LoggerWithStatistics(myLogger, statistics);

            files = await ValidateDataFilesAsync(logger, files, storageFormat, fix);

            var tree = await CreateDirectoryTreeAsync(files);

            foreach (var item in items)
            {
                var tagFile = item.Key;
                var tag     = item.Value;
                logger.Info($"  Validating {tagFile}");
                var isDirty = false;

                if (!tag.Product.ValidateProduct())
                {
                    logger.Error($"Invalid product {tag.Product} in file {tagFile}");
                }

                if (!tag.Version.ValidateVersion())
                {
                    logger.Error($"Invalid version {tag.Version} in file {tagFile}");
                }

                if (tag.CreationUtcTime == DateTime.MinValue)
                {
                    logger.Error($"The empty creation time in {tagFile}");
                    if (fix)
                    {
                        var newCreationUtcTime = TryFixCreationTime(tag);
                        if (newCreationUtcTime != null)
                        {
                            logger.Fix($"The creation time will be assigned to tag {tagFile}");
                            isDirty             = true;
                            tag.CreationUtcTime = newCreationUtcTime.Value;
                        }
                    }
                }

                if (tag.Directories == null || tag.Directories.Length == 0)
                {
                    logger.Error($"The empty directory list in {tagFile}");
                    if (fix)
                    {
                        logger.Fix($"The tag will be deleted {tagFile}");
                        await myStorage.DeleteAsync(tagFile);

                        continue;
                    }
                }
                else
                {
                    for (var index = 0; index < tag.Directories.Length; index++)
                    {
                        var dir = tag.Directories[index];

                        switch (dir.ValidateAndFixDataPath(StorageFormat.Normal, out var fixedDir))
                        {
                        case PathUtil.ValidateAndFixErrors.Ok:
                            break;

                        case PathUtil.ValidateAndFixErrors.Error:
                            logger.Error($"The tag directory {dir} from {tagFile} has invalid format");
                            break;

                        case PathUtil.ValidateAndFixErrors.CanBeFixed:
                            logger.Error($"The tag directory {dir} from {tagFile} has invalid format");
                            if (fix)
                            {
                                isDirty = true;
                                tag.Directories[index] = fixedDir;
                            }

                            break;

                        default:
                            throw new ArgumentOutOfRangeException();
                        }

                        var dstDir = dir.ValidateAndFixDataPath(storageFormat, out fixedDir) == PathUtil.ValidateAndFixErrors.CanBeFixed ? fixedDir : dir;
                        var node   = tree;
                        foreach (var part in dstDir.GetPathComponents())
                        {
                            node = node?.Lookup(part);
                        }

                        if (node == null)
                        {
                            logger.Error($"The directory {dir} from {tagFile} id wasn't found");
                        }
                        else
                        {
                            node.IncrementReferences();
                        }
                    }
                }

                if (isDirty)
                {
                    logger.Info($"The tag file {tagFile} will be overwritten");
                    await using var stream = new MemoryStream();
                    await TagUtil.WriteTagScriptAsync(tag, stream);

                    await myStorage.CreateForWritingAsync(tagFile, AccessMode.Private, stream);
                }
            }

            var deleted = await ValidateUnreachableAsync(logger, tree, mode);

            if (verifyAcl)
            {
                await ValidateAclAsync(logger, files, fix);
            }
            return(Tuple.Create(statistics, deleted));
        }