private void EnumerateAndTransfer(TransferScheduler scheduler, CancellationToken cancellationToken)
{
Interlocked.Increment(ref this.outstandingTasks);
try
{
foreach (var transfer in this.AllTransfers(cancellationToken))
{
this.CheckAndPauseEnumeration(cancellationToken);
transfer.UpdateProgressLock(this.progressUpdateLock);
this.DoTransfer(transfer, scheduler, cancellationToken);
}
}
catch (StorageException e)
{
throw new TransferException(TransferErrorCode.FailToEnumerateDirectory, e.GetExceptionMessage(), e);
}
finally
{
if (Interlocked.Decrement(ref this.outstandingTasks) == 0)
{
// make sure transfer terminiate when there is no subtransfer.
this.allTransfersCompleteSource.SetResult(null);
}
}
}
private static ServiceSideSyncCopyController CreateServiceSideSyncCopyConstroller(
TransferScheduler transferScheduler,
TransferJob transferJob,
CancellationToken cancellationToken)
{
CloudBlob destinationBlob = transferJob.Destination.Instance as CloudBlob;
if (null == destinationBlob)
{
throw new TransferException(Resources.ServiceSideSyncCopyNotSupportException);
}
if (BlobType.PageBlob == destinationBlob.BlobType)
{
return(new PageBlobServiceSideSyncCopyController(transferScheduler, transferJob, cancellationToken));
}
else if (BlobType.AppendBlob == destinationBlob.BlobType)
{
return(new AppendBlobServiceSideSyncCopyController(transferScheduler, transferJob, cancellationToken));
}
else if (BlobType.BlockBlob == destinationBlob.BlobType)
{
return(new BlockBlobServiceSideSyncCopyController(transferScheduler, transferJob, cancellationToken));
}
else
{
throw new TransferException(string.Format(CultureInfo.CurrentCulture, Resources.NotSupportedBlobType, destinationBlob.BlobType));
}
}
/// <summary>
/// Execute the transfer asynchronously.
/// </summary>
/// <param name="scheduler">Transfer scheduler</param>
/// <param name="cancellationToken">Token that can be used to cancel the transfer.</param>
/// <returns>A task representing the transfer operation.</returns>
public override async Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
{
this.ResetExecutionStatus();
this.Destination.Validate();
try
{
Task listTask = Task.Run(() => this.ListNewTransfers(cancellationToken));
await Task.Run(() => { this.EnumerateAndTransfer(scheduler, cancellationToken); });
await listTask;
}
finally
{
// wait for outstanding transfers to complete
await allTransfersCompleteSource.Task;
}
if (this.enumerateException != null)
{
throw this.enumerateException;
}
this.ProgressTracker.AddBytesTransferred(0);
}
private static TransferControllerBase GenerateTransferConstroller(
TransferScheduler transferScheduler,
TransferJob transferJob,
CancellationToken cancellationToken)
{
TransferControllerBase controller = null;
switch (transferJob.Transfer.TransferMethod)
{
case TransferMethod.SyncCopy:
controller = new SyncTransferController(transferScheduler, transferJob, cancellationToken);
break;
case TransferMethod.ServiceSideAsyncCopy:
controller = CreateAsyncCopyController(transferScheduler, transferJob, cancellationToken);
break;
case TransferMethod.ServiceSideSyncCopy:
controller = CreateServiceSideSyncCopyConstroller(transferScheduler, transferJob, cancellationToken);
break;
case TransferMethod.DummyCopy:
controller = new DummyTransferController(transferScheduler, transferJob, cancellationToken);
break;
}
return(controller);
}
private static AsyncCopyController CreateAsyncCopyController(TransferScheduler transferScheduler, TransferJob transferJob, CancellationToken cancellationToken)
{
if (transferJob.Destination.Type == TransferLocationType.AzureFile)
{
return(new FileAsyncCopyController(transferScheduler, transferJob, cancellationToken));
}
if (transferJob.Destination.Type == TransferLocationType.AzureBlob)
{
return(new BlobAsyncCopyController(transferScheduler, transferJob, cancellationToken));
}
throw new InvalidOperationException(Resources.CanOnlyCopyToFileOrBlobException);
}
public override async Task ExecuteInternalAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
{
try
{
await Task.Yield();
this.ResetExecutionStatus();
cancellationToken.Register(() =>
{
this.cancellationTokenSource?.Cancel();
});
this.currentScheduler = scheduler;
this.currentCancellationToken = cancellationToken;
this.maxConcurrencyControl.Wait();
await DoEnumerationAndTransferAsync(scheduler, cancellationToken);
}
catch (OperationCanceledException)
{
if (null == this.enumerateException)
{
throw;
}
}
finally
{
this.SignalSubDirTaskDecrement();
await this.subDirTransfersCompleteSource.Task;
if (this.maxConcurrency == this.maxConcurrencyControl.Release())
{
this.transfersCompleteSource.TrySetResult(null);
}
await this.transfersCompleteSource.Task;
}
if (null != this.enumerateException)
{
throw this.enumerateException;
}
else
{
Utils.CheckCancellation(cancellationToken);
}
}
/// <summary>
/// Execute the transfer asynchronously.
/// </summary>
/// <param name="scheduler">Transfer scheduler</param>
/// <param name="cancellationToken">Token that can be used to cancel the transfer.</param>
/// <returns>A task representing the transfer operation.</returns>
public override async Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
{
try
{
this.Destination.Validate();
}
catch (StorageException se)
{
throw new TransferException(TransferErrorCode.FailToVadlidateDestination,
Resources.FailedToValidateDestinationException,
se);
}
catch (Exception ex)
{
throw new TransferException(TransferErrorCode.FailToVadlidateDestination,
Resources.FailedToValidateDestinationException,
ex);
}
this.nameResolver = GetNameResolver(this.Source, this.Destination, this.Delimiter);
await this.ExecuteInternalAsync(scheduler, cancellationToken);
}
private async void DoTransfer(Transfer transfer, TransferScheduler scheduler, CancellationToken cancellationToken)
{
using (transfer)
{
bool hasError = false;
Interlocked.Increment(ref this.outstandingTasks);
try
{
await transfer.ExecuteAsync(scheduler, cancellationToken);
}
catch
{
// catch exception thrown from sub-transfer as it's already recorded
hasError = true;
}
finally
{
// Don't keep the failed transferring in memory, if the checkpoint is persist to a streamed journal,
// instead, should only keep them in the streamed journal.
if ((!hasError) ||
(null != this.Journal))
{
this.subTransfers.RemoveTransfer(transfer);
}
this.enumerationResetEvent.Set();
if (Interlocked.Decrement(ref this.outstandingTasks) == 0)
{
// all transfers are done
this.allTransfersCompleteSource.SetResult(null);
}
}
}
}
/// <summary>
/// Execute the transfer asynchronously.
/// </summary>
/// <param name="scheduler">Transfer scheduler</param>
/// <param name="cancellationToken">Token that can be used to cancel the transfer.</param>
/// <returns>A task representing the transfer operation.</returns>
public override async Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
{
if (this.transferJob.Status == TransferJobStatus.Finished ||
this.transferJob.Status == TransferJobStatus.Skipped)
{
return;
}
TransferEventArgs eventArgs = new TransferEventArgs(this.Source.Instance, this.Destination.Instance);
eventArgs.StartTime = DateTime.UtcNow;
if (this.transferJob.Status == TransferJobStatus.Failed)
{
// Resuming a failed transfer job
if (string.IsNullOrEmpty(this.transferJob.CopyId))
{
this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Transfer);
}
else
{
this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Monitor);
}
}
try
{
await scheduler.ExecuteJobAsync(this.transferJob, cancellationToken);
if (TransferJobStatus.SkippedDueToShouldNotTransfer != this.transferJob.Status)
{
eventArgs.EndTime = DateTime.UtcNow;
this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Finished);
if (this.Context != null)
{
this.Context.OnTransferSuccess(eventArgs);
}
}
}
catch (TransferException exception)
{
eventArgs.EndTime = DateTime.UtcNow;
eventArgs.Exception = exception;
if (exception.ErrorCode == TransferErrorCode.NotOverwriteExistingDestination)
{
// transfer skipped
this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Skipped);
if (this.Context != null)
{
this.Context.OnTransferSkipped(eventArgs);
}
throw;
}
else if (exception.ErrorCode == TransferErrorCode.FailedCheckingShouldTransfer)
{
throw;
}
else
{
this.OnTransferFailed(eventArgs);
throw;
}
}
catch (Exception ex)
{
eventArgs.EndTime = DateTime.UtcNow;
eventArgs.Exception = ex;
this.OnTransferFailed(eventArgs);
throw;
}
this.Journal?.RemoveTransfer(this);
}
public abstract Task ExecuteInternalAsync(TransferScheduler scheduler, CancellationToken cancellationToken);
private async Task DoEnumerationAndTransferAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
{
await Task.Yield();
this.Resume(scheduler, cancellationToken);
if (0 == this.enumerationStarted)
{
// Got nothing from checkpoint, start directory transfer from the very beginning.
var subDirTransfer = new SubDirectoryTransfer(this, "");
lock (this.continuationTokenLock)
{
if (null == this.Journal)
{
this.ongoingSubDirTransfers.TryAdd(subDirTransfer, new object());
}
this.enumerationStarted = 1;
}
if (null != this.Journal)
{
this.Journal.AddOngoingSubDirTransfer(subDirTransfer);
this.Journal?.UpdateJournalItem(this);
}
this.ScheduleSubDirectoryTransfer(
subDirTransfer,
this.cancellationTokenSource.Token,
null,
-1);
}
bool gotDirectory = true;
while (true)
{
Utils.CheckCancellation(this.cancellationTokenSource.Token);
if (!gotDirectory)
{
newAddSubDirResetEventSlim.Wait(cancellationToken);
newAddSubDirResetEventSlim.Reset();
}
// Check whether theres ongoing subdirectory listing thread.
bool listCompleted = (1 == Interlocked.Read(ref this.outstandingTasks));
string subDirRelativePath = null;
if (null != this.Journal)
{
subDirRelativePath = this.Journal.PeekSubDirTransfer();
}
else
{
this.subDirectories.TryPeek(out subDirRelativePath);
}
if (string.IsNullOrEmpty(subDirRelativePath))
{
if (listCompleted)
{
// There's no ongoing subdirectory listing thread,
// and no subdirectory pending on listing
// This means that the whole listing is completed.
break;
}
gotDirectory = false;
continue;
}
else
{
gotDirectory = true;
Utils.CheckCancellation(this.cancellationTokenSource.Token);
SubDirectoryTransfer subDirTransfer = new SubDirectoryTransfer(this, subDirRelativePath);
this.ScheduleSubDirectoryTransfer(
subDirTransfer,
this.cancellationTokenSource.Token,
() =>
{
if (null != this.Journal)
{
this.Journal.AddOngoingSubDirTransfer(subDirTransfer);
this.Journal.RemoveFirstSubDirTransfer();
}
else
{
this.ongoingSubDirTransfers.TryAdd(subDirTransfer, new object());
this.subDirectories.TryDequeue(out subDirRelativePath);
}
},
-1);
}
}
}
private bool Resume(TransferScheduler scheduler, CancellationToken cancellationToken)
{
bool isResume = false;
if (null != this.Journal)
{
foreach (var transfer in this.Journal.ListSubTransfers())
{
isResume = true;
Utils.CheckCancellation(this.cancellationTokenSource.Token);
transfer.Context = this.Context;
this.UpdateTransfer(transfer);
this.subTransfers.AddTransfer(transfer, false);
this.TransferFile((transfer as SingleObjectTransfer), scheduler, cancellationToken);
}
foreach (var subDirTransfer in this.Journal.ListSubDirTransfers())
{
isResume = true;
Utils.CheckCancellation(this.cancellationTokenSource.Token);
subDirTransfer.Update(this);
this.ScheduleSubDirectoryTransfer(
subDirTransfer,
this.cancellationTokenSource.Token,
() =>
{ },
-1);
}
}
else
{
// return all existing transfers in subTransfers
foreach (var transfer in this.subTransfers.GetEnumerator())
{
isResume = true;
Utils.CheckCancellation(this.cancellationTokenSource.Token);
transfer.Context = this.Context;
this.UpdateTransfer(transfer);
this.TransferFile((transfer as SingleObjectTransfer), scheduler, cancellationToken);
}
foreach (var subDirTransferPair in this.ongoingSubDirTransfers)
{
isResume = true;
var subDirTransfer = subDirTransferPair.Key;
subDirTransfer.Update(this);
this.ScheduleSubDirectoryTransfer(
subDirTransfer,
this.cancellationTokenSource.Token,
() =>
{
},
-1);
}
}
return(isResume);
}
internal async void TransferFile(SingleObjectTransfer transferItem, TransferScheduler scheduler, CancellationToken cancellationToken)
{
try
{
this.maxConcurrencyControl.Wait(cancellationToken);
}
catch (OperationCanceledException)
{
// No need to report exception here, OperationCanceledException will be handled in other place.
return;
}
catch (ObjectDisposedException)
{
return;
}
await Task.Yield();
bool hasError = false;
bool shouldStopTransfer = false;
transferItem.UpdateProgressLock(this.progressUpdateLock);
try
{
using (transferItem)
{
await transferItem.ExecuteAsync(scheduler, cancellationToken);
}
}
catch (TransferException ex)
{
if (ex.ErrorCode == TransferErrorCode.FailedCheckingShouldTransfer)
{
shouldStopTransfer = true;
this.enumerateException = new TransferException(
TransferErrorCode.FailToEnumerateDirectory,
string.Format(CultureInfo.CurrentCulture,
Resources.EnumerateDirectoryException,
this.Destination.Instance.ConvertToString()),
ex.InnerException);
}
hasError = true;
}
catch
{
hasError = true;
}
finally
{
if ((!hasError) ||
(null != this.Journal))
{
this.subTransfers.RemoveTransfer(transferItem);
}
if (this.maxConcurrency == this.maxConcurrencyControl.Release())
{
this.transfersCompleteSource.TrySetResult(null);
}
}
if (shouldStopTransfer)
{
this.cancellationTokenSource?.Cancel();
}
}
