public async Task TransferLocalDirectoryToAzureStorage(Func <string, UploadDirectoryOptions, DirectoryTransferContext, CancellationToken, Task <TransferStatus> > UploadDirectoryAsync,
Func <TransferEventArgs, string> getSourceToDestinationInfo,
CancellationToken cancellationToken)
{
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
Recursive = _repo.Recursive,
BlobType = BlobType.BlockBlob
//SearchPattern =
};
var internalTokenSource = new CancellationTokenSource();
using var linkedCts =
CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, internalTokenSource.Token);
using var watcher = new UploadFileSystemWatcher(_repo.UploadFolder);
try
{
var transferCheckpoint = AzCpCheckpoint.Read(_repo.TransferCheckpointFilename);
if (transferCheckpoint != null)
{
_logger.Information("Resuming upload...");
}
while (true)
{
Stopwatch stopWatch = new Stopwatch();// Stopwatch.StartNew();
var context = new MyDirectoryTransferContext(_logger, _feedback, _repo.UploadFolder, _repo.ArchiveFolder, _repo.TransferCheckpointFilename, stopWatch, transferCheckpoint, getSourceToDestinationInfo);
transferCheckpoint = null;
_feedback.WriteProgress("Establishing connection...");
watcher.ResetChangeCount();
stopWatch.Start();
// var transferStatus = await TransferManager.UploadDirectoryAsync(_repo.UploadFolder, blobDirectory, options, context, linkedCts.Token);
var transferStatus = await UploadDirectoryAsync(_repo.UploadFolder, options, context, linkedCts.Token);
stopWatch.Stop();
linkedCts.Token.ThrowIfCancellationRequested();
try
{
if (File.Exists(_repo.TransferCheckpointFilename))
{
File.Delete(_repo.TransferCheckpointFilename);
}
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (IOException)
{
// ignore any issues deleting the checkpoint file
// e.g. doesn't exist
}
#pragma warning restore CA1031 // Do not catch general exception types
_logger.Information($"{stopWatch.Elapsed}: {transferStatus.ToUserString(context.InitialBytesTransferred, stopWatch)}");
// wait until there are new files to upload
// NOTE: Will also be triggered if files are renamed or deleted
while (!watcher.AnyChangesInUploadFolder)
{
_feedback.WriteProgress("Waiting for new files to upload...");
if (linkedCts.Token.WaitHandle.WaitOne(TimeSpan.FromMilliseconds(1500 - DateTime.UtcNow.Millisecond)))
{
linkedCts.Token.ThrowIfCancellationRequested();
}
}
}
}
catch (Exception ex) when(ex is TaskCanceledException || ex is OperationCanceledException)
{
_logger.Information($"The transfer was cancelled");
throw;
}
catch (Exception)
{
//_logger.Error(ex, $"UNEXPECTED ERROR: {ex.Message}");
throw;
}
}
public MyDirectoryTransferContext(ILogger logger, IFeedback feedback,
string uploadFolder, string archiveFolder, string transferCheckpointFilename,
Stopwatch stopwatch,
TransferCheckpoint checkpoint,
Func <TransferEventArgs, string> getSourceToDestinationInfo) : base(checkpoint)
{
if (checkpoint == null)
{
// if there's no previous checkpoint then set the initial tx to 0 instead of null (resuming)
InitialBytesTransferred = 0;
}
void ArchiveFile(TransferEventArgs e)
{
var relPath = Path.GetRelativePath(uploadFolder, (string)e.Source);
var archivePath = Path.Combine(archiveFolder, relPath);
Directory.CreateDirectory(Path.GetDirectoryName(archivePath));
File.Move((string)e.Source, archivePath, true);
}
FileFailed += (sender, e) =>
{
lock (feedback)
{
feedback.WriteProgress();
logger.Warning($"FAILED: {getSourceToDestinationInfo(e)}", null, IFeedback.Colors.ErrorForegroundColor);
e.Exception?.LogIt();
}
};
FileSkipped += (sender, e) =>
{
lock (feedback)
{
feedback.WriteProgress();
logger.Information($"Skipped: {getSourceToDestinationInfo(e)}");
e.Exception?.LogIt();
}
ArchiveFile(e);
};
FileTransferred += (sender, e) =>
{
lock (feedback)
{
feedback.WriteProgress();
logger.Information($"Transferred: {getSourceToDestinationInfo(e)}");
e.Exception?.LogIt();
}
ArchiveFile(e);
};
// todo: result.ShouldTransferCallbackAsync
ProgressHandler = new Progress <TransferStatus>((progress) =>
{
if (!InitialBytesTransferred.HasValue)
{
InitialBytesTransferred = progress.BytesTransferred;
}
lock (feedback)
{
feedback.WriteProgress(progress.ToUserString((long)InitialBytesTransferred, stopwatch), progress.NumberOfFilesFailed == 0 ? IFeedback.Colors.OkForegroundColor : IFeedback.Colors.WarningForegroundColor);
}
AzCpCheckpoint.Write(LastCheckpoint, transferCheckpointFilename);
});
}
