コード例 #1
0
        public async Task TransferLocalDirectoryToAzureStorage(Func <string, UploadDirectoryOptions, DirectoryTransferContext, CancellationToken, Task <TransferStatus> > UploadDirectoryAsync,
                                                               Func <TransferEventArgs, string> getSourceToDestinationInfo,

                                                               CancellationToken cancellationToken)
        {
            UploadDirectoryOptions options = new UploadDirectoryOptions()
            {
                Recursive = _repo.Recursive,
                BlobType  = BlobType.BlockBlob
                            //SearchPattern =
            };

            var internalTokenSource = new CancellationTokenSource();

            using var linkedCts =
                      CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, internalTokenSource.Token);

            using var watcher = new UploadFileSystemWatcher(_repo.UploadFolder);

            try
            {
                var transferCheckpoint = AzCpCheckpoint.Read(_repo.TransferCheckpointFilename);
                if (transferCheckpoint != null)
                {
                    _logger.Information("Resuming upload...");
                }

                while (true)
                {
                    Stopwatch stopWatch = new Stopwatch();// Stopwatch.StartNew();

                    var context = new MyDirectoryTransferContext(_logger, _feedback, _repo.UploadFolder, _repo.ArchiveFolder, _repo.TransferCheckpointFilename, stopWatch, transferCheckpoint, getSourceToDestinationInfo);

                    transferCheckpoint = null;

                    _feedback.WriteProgress("Establishing connection...");

                    watcher.ResetChangeCount();

                    stopWatch.Start();
                    // var transferStatus = await TransferManager.UploadDirectoryAsync(_repo.UploadFolder, blobDirectory, options, context, linkedCts.Token);
                    var transferStatus = await UploadDirectoryAsync(_repo.UploadFolder, options, context, linkedCts.Token);

                    stopWatch.Stop();

                    linkedCts.Token.ThrowIfCancellationRequested();

                    try
                    {
                        if (File.Exists(_repo.TransferCheckpointFilename))
                        {
                            File.Delete(_repo.TransferCheckpointFilename);
                        }
                    }
#pragma warning disable CA1031 // Do not catch general exception types
                    catch (IOException)
                    {
                        // ignore any issues deleting the checkpoint file
                        // e.g. doesn't exist
                    }
#pragma warning restore CA1031 // Do not catch general exception types

                    _logger.Information($"{stopWatch.Elapsed}: {transferStatus.ToUserString(context.InitialBytesTransferred, stopWatch)}");

                    // wait until there are new files to upload
                    // NOTE: Will also be triggered if files are renamed or deleted
                    while (!watcher.AnyChangesInUploadFolder)
                    {
                        _feedback.WriteProgress("Waiting for new files to upload...");

                        if (linkedCts.Token.WaitHandle.WaitOne(TimeSpan.FromMilliseconds(1500 - DateTime.UtcNow.Millisecond)))
                        {
                            linkedCts.Token.ThrowIfCancellationRequested();
                        }
                    }
                }
            }
            catch (Exception ex) when(ex is TaskCanceledException || ex is OperationCanceledException)
            {
                _logger.Information($"The transfer was cancelled");
                throw;
            }
            catch (Exception)
            {
                //_logger.Error(ex, $"UNEXPECTED ERROR: {ex.Message}");
                throw;
            }
        }
コード例 #2
0
        public MyDirectoryTransferContext(ILogger logger, IFeedback feedback,
                                          string uploadFolder, string archiveFolder, string transferCheckpointFilename,
                                          Stopwatch stopwatch,
                                          TransferCheckpoint checkpoint,
                                          Func <TransferEventArgs, string> getSourceToDestinationInfo) : base(checkpoint)
        {
            if (checkpoint == null)
            {
                // if there's no previous checkpoint then set the initial tx to 0 instead of null (resuming)
                InitialBytesTransferred = 0;
            }

            void ArchiveFile(TransferEventArgs e)
            {
                var relPath     = Path.GetRelativePath(uploadFolder, (string)e.Source);
                var archivePath = Path.Combine(archiveFolder, relPath);

                Directory.CreateDirectory(Path.GetDirectoryName(archivePath));
                File.Move((string)e.Source, archivePath, true);
            }

            FileFailed += (sender, e) =>
            {
                lock (feedback)
                {
                    feedback.WriteProgress();
                    logger.Warning($"FAILED: {getSourceToDestinationInfo(e)}", null, IFeedback.Colors.ErrorForegroundColor);
                    e.Exception?.LogIt();
                }
            };
            FileSkipped += (sender, e) =>
            {
                lock (feedback)
                {
                    feedback.WriteProgress();
                    logger.Information($"Skipped: {getSourceToDestinationInfo(e)}");
                    e.Exception?.LogIt();
                }
                ArchiveFile(e);
            };
            FileTransferred += (sender, e) =>
            {
                lock (feedback)
                {
                    feedback.WriteProgress();
                    logger.Information($"Transferred: {getSourceToDestinationInfo(e)}");
                    e.Exception?.LogIt();
                }
                ArchiveFile(e);
            };

            // todo: result.ShouldTransferCallbackAsync

            ProgressHandler = new Progress <TransferStatus>((progress) =>
            {
                if (!InitialBytesTransferred.HasValue)
                {
                    InitialBytesTransferred = progress.BytesTransferred;
                }

                lock (feedback)
                {
                    feedback.WriteProgress(progress.ToUserString((long)InitialBytesTransferred, stopwatch), progress.NumberOfFilesFailed == 0 ? IFeedback.Colors.OkForegroundColor : IFeedback.Colors.WarningForegroundColor);
                }

                AzCpCheckpoint.Write(LastCheckpoint, transferCheckpointFilename);
            });
        }