/// <summary>
/// Execute the transfer asynchronously.
/// </summary>
/// <param name="scheduler">Transfer scheduler</param>
/// <param name="cancellationToken">Token that can be used to cancel the transfer.</param>
/// <returns>A task representing the transfer operation.</returns>
public override async Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
{
try
{
this.Destination.Validate();
}
catch (StorageException se)
{
throw new TransferException(TransferErrorCode.FailToVadlidateDestination,
string.Format(Resources.FailedToValidateDestinationException,
se.ToErrorDetail(),
CultureInfo.CurrentCulture),
se);
}
catch (Exception ex)
{
throw new TransferException(TransferErrorCode.FailToVadlidateDestination,
string.Format(Resources.FailedToValidateDestinationException,
ex.Message,
CultureInfo.CurrentCulture),
ex);
}
this.nameResolver = GetNameResolver(this.Source, this.Destination, this.Delimiter);
await base.ExecuteAsync(scheduler, cancellationToken);
}
private async Task DoTransfer(Transfer transfer, TransferScheduler scheduler, CancellationToken cancellationToken)
{
using (transfer)
{
bool hasError = false;
Interlocked.Increment(ref this.outstandingTasks);
try
{
await transfer.ExecuteAsync(scheduler, cancellationToken);
}
catch
{
// catch exception thrown from sub-transfer as it's already recorded
hasError = true;
}
finally
{
if (!hasError)
{
this.subTransfers.RemoveTransfer(transfer);
}
if (Interlocked.Decrement(ref this.outstandingTasks) == 0)
{
// all transfers are done
this.allTransfersCompleteSource.SetResult(null);
}
this.enumerationResetEvent.Set();
}
}
}
/// <summary>
/// Execute the transfer asynchronously.
/// </summary>
/// <param name="scheduler">Transfer scheduler</param>
/// <param name="cancellationToken">Token that can be used to cancel the transfer.</param>
/// <returns>A task representing the transfer operation.</returns>
public override async Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
{
this.ResetExecutionStatus();
this.Destination.Validate();
Task enumerateTask = Task.Run(() => { this.EnumerateAndTransfer(scheduler, cancellationToken); });
await enumerateTask;
// wait for outstanding transfers to complete
await allTransfersCompleteSource.Task;
if (this.enumerateException != null)
{
throw this.enumerateException;
}
else if (this.subTransfers.Count != 0)
{
string errorMessage = string.Format(
CultureInfo.CurrentCulture,
Resources.SubTransferFailsException,
this.subTransfers.Count);
throw new TransferException(TransferErrorCode.SubTransferFails, errorMessage);
}
}
private void EnumerateAndTransfer(TransferScheduler scheduler, CancellationToken cancellationToken)
{
Interlocked.Increment(ref this.outstandingTasks);
try
{
foreach (var transfer in this.AllTransfers(cancellationToken))
{
this.CheckAndPauseEnumeration(cancellationToken);
Task unused = this.DoTransfer(transfer, scheduler, cancellationToken);
}
}
catch (Exception ex)
{
// exception happens when enumerating source
this.enumerateException = ex;
}
if (Interlocked.Decrement(ref this.outstandingTasks) == 0)
{
// make sure transfer terminiate when there is no subtransfer.
this.allTransfersCompleteSource.SetResult(null);
}
}
/// <summary>
/// Execute the transfer asynchronously.
/// </summary>
/// <param name="scheduler">Transfer scheduler</param>
/// <param name="cancellationToken">Token that can be used to cancel the transfer.</param>
/// <returns>A task representing the transfer operation.</returns>
public override async Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
{
this.ResetExecutionStatus();
this.Destination.Validate();
try
{
Task listTask = Task.Run(() => this.ListNewTransfers(cancellationToken));
await Task.Run(() => { this.EnumerateAndTransfer(scheduler, cancellationToken); }).ConfigureAwait(false);
await listTask.ConfigureAwait(false);
}
finally
{
// wait for outstanding transfers to complete
await allTransfersCompleteSource.Task.ConfigureAwait(false);
}
if (this.enumerateException != null)
{
throw this.enumerateException;
}
this.ProgressTracker.AddBytesTransferred(0);
}
private void EnumerateAndTransfer(TransferScheduler scheduler, CancellationToken cancellationToken)
{
Interlocked.Increment(ref this.outstandingTasks);
try
{
foreach (var transfer in this.AllTransfers(cancellationToken))
{
this.CheckAndPauseEnumeration(cancellationToken);
transfer.UpdateProgressLock(this.progressUpdateLock);
this.DoTransfer(transfer, scheduler, cancellationToken);
}
}
catch (StorageException e)
{
throw new TransferException(TransferErrorCode.FailToEnumerateDirectory, e.GetExceptionMessage(), e);
}
finally
{
if (Interlocked.Decrement(ref this.outstandingTasks) == 0)
{
// make sure transfer terminiate when there is no subtransfer.
this.allTransfersCompleteSource.SetResult(null);
}
}
}
public FileAsyncCopyController(
TransferScheduler transferScheduler,
TransferJob transferJob,
CancellationToken cancellationToken)
: base(transferScheduler, transferJob, cancellationToken)
{
if (transferJob.Destination.Type != TransferLocationType.AzureFile)
{
throw new ArgumentException(
string.Format(
CultureInfo.CurrentCulture,
Resources.ParameterCannotBeNullException,
"Dest.AzureFile"),
"transferJob");
}
if (transferJob.Source.Type != TransferLocationType.SourceUri &&
transferJob.Source.Type != TransferLocationType.AzureBlob &&
transferJob.Source.Type != TransferLocationType.AzureFile)
{
throw new ArgumentException(
string.Format(
CultureInfo.CurrentCulture,
Resources.ProvideExactlyOneOfThreeParameters,
"Source.SourceUri",
"Source.Blob",
"Source.AzureFile"),
"transferJob");
}
this.destLocation = this.TransferJob.Destination as AzureFileLocation;
this.destFile = this.destLocation.AzureFile;
}
/// <summary>
/// Execute the transfer asynchronously.
/// </summary>
/// <param name="scheduler">Transfer scheduler</param>
/// <param name="cancellationToken">Token that can be used to cancel the transfer.</param>
/// <returns>A task representing the transfer operation.</returns>
public override async Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
{
try
{
this.Destination.Validate();
}
catch (Exception ex)
{
throw new TransferException(TransferErrorCode.FailToVadlidateDestination, Resources.FailedToValidateDestinationException, ex);
}
await base.ExecuteAsync(scheduler, cancellationToken);
}
/// <summary>
/// Execute the transfer asynchronously.
/// </summary>
/// <param name="scheduler">Transfer scheduler</param>
/// <param name="cancellationToken">Token that can be used to cancel the transfer.</param>
/// <returns>A task representing the transfer operation.</returns>
public override async Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
{
this.ResetExecutionStatus();
this.Destination.Validate();
Task enumerateTask = Task.Run(() => { this.EnumerateAndTransfer(scheduler, cancellationToken); });
await enumerateTask;
// wait for outstanding transfers to complete
await allTransfersCompleteSource.Task;
if (this.enumerateException != null)
{
throw this.enumerateException;
}
this.ProgressTracker.AddBytesTransferred(0);
}
protected TransferControllerBase(TransferScheduler transferScheduler, TransferJob transferJob, CancellationToken userCancellationToken)
{
if (null == transferScheduler)
{
throw new ArgumentNullException("transferScheduler");
}
if (null == transferJob)
{
throw new ArgumentNullException("transferJob");
}
this.Scheduler = transferScheduler;
this.TransferJob = transferJob;
this.transferSchedulerCancellationTokenRegistration =
this.Scheduler.CancellationTokenSource.Token.Register(this.CancelWork);
this.userCancellationTokenRegistration = userCancellationToken.Register(this.CancelWork);
this.TaskCompletionSource = new TaskCompletionSource<object>();
}
public BlobAsyncCopyController(
TransferScheduler transferScheduler,
TransferJob transferJob,
CancellationToken cancellationToken)
: base(transferScheduler, transferJob, cancellationToken)
{
this.destLocation = transferJob.Destination as AzureBlobLocation;
CloudBlob transferDestBlob = this.destLocation.Blob;
if (null == transferDestBlob)
{
throw new ArgumentException(
string.Format(
CultureInfo.CurrentCulture,
Resources.ParameterCannotBeNullException,
"Dest.Blob"),
"transferJob");
}
if (transferDestBlob.IsSnapshot)
{
throw new ArgumentException(Resources.DestinationMustBeBaseBlob, "transferJob");
}
AzureBlobLocation sourceBlobLocation = transferJob.Source as AzureBlobLocation;
if (sourceBlobLocation != null)
{
if (sourceBlobLocation.Blob.BlobType != transferDestBlob.BlobType)
{
throw new ArgumentException(Resources.SourceAndDestinationBlobTypeDifferent, "transferJob");
}
if (StorageExtensions.Equals(sourceBlobLocation.Blob, transferDestBlob))
{
throw new InvalidOperationException(Resources.SourceAndDestinationLocationCannotBeEqualException);
}
}
this.destBlob = transferDestBlob;
}
/// <summary>
/// Execute the transfer asynchronously.
/// </summary>
/// <param name="scheduler">Transfer scheduler</param>
/// <param name="cancellationToken">Token that can be used to cancel the transfer.</param>
/// <returns>A task representing the transfer operation.</returns>
public override async Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
{
try
{
this.Destination.Validate();
}
catch (StorageException se)
{
throw new TransferException(TransferErrorCode.FailToVadlidateDestination,
Resources.FailedToValidateDestinationException,
se);
}
catch (Exception ex)
{
throw new TransferException(TransferErrorCode.FailToVadlidateDestination,
Resources.FailedToValidateDestinationException,
ex);
}
this.nameResolver = GetNameResolver(this.Source, this.Destination, this.Delimiter);
await base.ExecuteAsync(scheduler, cancellationToken);
}
private async void DoTransfer(Transfer transfer, TransferScheduler scheduler, CancellationToken cancellationToken)
{
using (transfer)
{
bool hasError = false;
Interlocked.Increment(ref this.outstandingTasks);
try
{
await transfer.ExecuteAsync(scheduler, cancellationToken).ConfigureAwait(false);
}
catch
{
// catch exception thrown from sub-transfer as it's already recorded
hasError = true;
}
finally
{
// Don't keep the failed transferring in memory, if the checkpoint is persist to a streamed journal,
// instead, should only keep them in the streamed journal.
if ((!hasError) ||
(null != this.Journal))
{
this.subTransfers.RemoveTransfer(transfer);
}
this.enumerationResetEvent.Set();
if (Interlocked.Decrement(ref this.outstandingTasks) == 0)
{
// all transfers are done
this.allTransfersCompleteSource.SetResult(null);
}
}
}
}
/// <summary>
/// Execute the transfer asynchronously.
/// </summary>
/// <param name="scheduler">Transfer scheduler</param>
/// <param name="cancellationToken">Token that can be used to cancel the transfer.</param>
/// <returns>A task representing the transfer operation.</returns>
public override async Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
{
if (this.transferJob.Status == TransferJobStatus.Finished ||
this.transferJob.Status == TransferJobStatus.Skipped)
{
return;
}
if (transferJob.Status == TransferJobStatus.Failed)
{
// Resuming a failed transfer job
this.UpdateTransferJobStatus(transferJob, TransferJobStatus.Transfer);
}
try
{
await scheduler.ExecuteJobAsync(transferJob, cancellationToken);
this.UpdateTransferJobStatus(transferJob, TransferJobStatus.Finished);
}
catch (TransferException exception)
{
if (exception.ErrorCode == TransferErrorCode.NotOverwriteExistingDestination)
{
// transfer skipped
this.UpdateTransferJobStatus(transferJob, TransferJobStatus.Skipped);
}
else
{
// transfer failed
this.UpdateTransferJobStatus(transferJob, TransferJobStatus.Failed);
}
throw;
}
}
/// <summary>
/// Execute the transfer asynchronously.
/// </summary>
/// <param name="scheduler">Transfer scheduler</param>
/// <param name="cancellationToken">Token that can be used to cancel the transfer.</param>
/// <returns>A task representing the transfer operation.</returns>
public override async Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
{
try
{
this.Destination.Validate();
}
catch(Exception ex)
{
throw new TransferException(TransferErrorCode.FailToVadlidateDestination, Resources.FailedToValidateDestinationException, ex);
}
await base.ExecuteAsync(scheduler, cancellationToken);
}
/// <summary>
/// Execute the transfer asynchronously.
/// </summary>
/// <param name="scheduler">Transfer scheduler</param>
/// <param name="cancellationToken">Token that can be used to cancel the transfer.</param>
/// <returns>A task representing the transfer operation.</returns>
public override async Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
{
if (this.transferJob.Status == TransferJobStatus.Finished ||
this.transferJob.Status == TransferJobStatus.Skipped)
{
return;
}
TransferEventArgs eventArgs = new TransferEventArgs(this.Source.ToString(), this.Destination.ToString());
eventArgs.StartTime = DateTime.UtcNow;
if (this.transferJob.Status == TransferJobStatus.Failed)
{
// Resuming a failed transfer job
if (string.IsNullOrEmpty(this.transferJob.CopyId))
{
this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Transfer);
}
else
{
this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Monitor);
}
}
try
{
await scheduler.ExecuteJobAsync(this.transferJob, cancellationToken);
this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Finished);
eventArgs.EndTime = DateTime.UtcNow;
if(this.Context != null)
{
this.Context.OnTransferSuccess(eventArgs);
}
}
catch (TransferException exception)
{
eventArgs.EndTime = DateTime.UtcNow;
eventArgs.Exception = exception;
if (exception.ErrorCode == TransferErrorCode.NotOverwriteExistingDestination)
{
// transfer skipped
this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Skipped);
if (this.Context != null)
{
this.Context.OnTransferSkipped(eventArgs);
}
throw;
}
else
{
// transfer failed
this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Failed);
if (this.Context != null)
{
this.Context.OnTransferFailed(eventArgs);
}
throw;
}
}
catch (Exception ex)
{
eventArgs.EndTime = DateTime.UtcNow;
eventArgs.Exception = ex;
// transfer failed
this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Failed);
if (this.Context != null)
{
this.Context.OnTransferFailed(eventArgs);
}
throw;
}
}
/// <summary>
/// Execute the transfer asynchronously.
/// </summary>
/// <param name="scheduler">Transfer scheduler</param>
/// <param name="cancellationToken">Token that can be used to cancel the transfer.</param>
/// <returns>A task representing the transfer operation.</returns>
public override async Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
{
if (this.transferJob.Status == TransferJobStatus.Finished ||
this.transferJob.Status == TransferJobStatus.Skipped)
{
return;
}
TransferEventArgs eventArgs = new TransferEventArgs(this.Source.ToString(), this.Destination.ToString());
eventArgs.StartTime = DateTime.UtcNow;
if (this.transferJob.Status == TransferJobStatus.Failed)
{
// Resuming a failed transfer job
if (string.IsNullOrEmpty(this.transferJob.CopyId))
{
this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Transfer);
}
else
{
this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Monitor);
}
}
try
{
await scheduler.ExecuteJobAsync(this.transferJob, cancellationToken);
this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Finished);
eventArgs.EndTime = DateTime.UtcNow;
if (this.Context != null)
{
this.Context.OnTransferSuccess(eventArgs);
}
}
catch (TransferException exception)
{
eventArgs.EndTime = DateTime.UtcNow;
eventArgs.Exception = exception;
if (exception.ErrorCode == TransferErrorCode.NotOverwriteExistingDestination)
{
// transfer skipped
this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Skipped);
if (this.Context != null)
{
this.Context.OnTransferSkipped(eventArgs);
}
throw;
}
else
{
// transfer failed
this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Failed);
if (this.Context != null)
{
this.Context.OnTransferFailed(eventArgs);
}
throw;
}
}
catch (Exception ex)
{
eventArgs.EndTime = DateTime.UtcNow;
eventArgs.Exception = ex;
// transfer failed
this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Failed);
if (this.Context != null)
{
this.Context.OnTransferFailed(eventArgs);
}
throw;
}
}
/// <summary> /// Execute the transfer asynchronously. /// </summary> /// <param name="scheduler">Transfer scheduler</param> /// <param name="cancellationToken">Token that can be used to cancel the transfer.</param> /// <returns>A task representing the transfer operation.</returns> public abstract Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken);
private async Task DoTransfer(Transfer transfer, TransferScheduler scheduler, CancellationToken cancellationToken)
{
using (transfer)
{
bool hasError = false;
Interlocked.Increment(ref this.outstandingTasks);
try
{
await transfer.ExecuteAsync(scheduler, cancellationToken);
}
catch
{
// catch exception thrown from sub-transfer as it's already recorded
hasError = true;
}
finally
{
if (!hasError)
{
this.subTransfers.RemoveTransfer(transfer);
}
if (Interlocked.Decrement(ref this.outstandingTasks) == 0)
{
// all transfers are done
this.allTransfersCompleteSource.SetResult(null);
}
this.enumerationResetEvent.Set();
}
}
}
private void EnumerateAndTransfer(TransferScheduler scheduler, CancellationToken cancellationToken)
{
Interlocked.Increment(ref this.outstandingTasks);
try
{
foreach (var transfer in this.AllTransfers(cancellationToken))
{
this.CheckAndPauseEnumeration(cancellationToken);
Task unused = this.DoTransfer(transfer, scheduler, cancellationToken);
}
}
catch (Exception ex)
{
// exception happens when enumerating source
this.enumerateException = ex;
}
if (Interlocked.Decrement(ref this.outstandingTasks) == 0)
{
// make sure transfer terminiate when there is no subtransfer.
this.allTransfersCompleteSource.SetResult(null);
}
}
/// <summary>
/// Execute the transfer asynchronously.
/// </summary>
/// <param name="scheduler">Transfer scheduler</param>
/// <param name="cancellationToken">Token that can be used to cancel the transfer.</param>
/// <returns>A task representing the transfer operation.</returns>
public override async Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
{
this.ResetExecutionStatus();
this.Destination.Validate();
Task enumerateTask = Task.Run(() => { this.EnumerateAndTransfer(scheduler, cancellationToken); });
await enumerateTask;
// wait for outstanding transfers to complete
await allTransfersCompleteSource.Task;
if (this.enumerateException != null)
{
throw this.enumerateException;
}
else if (this.subTransfers.Count != 0)
{
string errorMessage = string.Format(
CultureInfo.CurrentCulture,
Resources.SubTransferFailsException,
this.subTransfers.Count);
throw new TransferException(TransferErrorCode.SubTransferFails, errorMessage);
}
}
/// <summary> /// Execute the transfer asynchronously. /// </summary> /// <param name="scheduler">Transfer scheduler</param> /// <param name="cancellationToken">Token that can be used to cancel the transfer.</param> /// <returns>A task representing the transfer operation.</returns> public abstract Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken);
/// <summary>
/// Execute the transfer asynchronously.
/// </summary>
/// <param name="scheduler">Transfer scheduler</param>
/// <param name="cancellationToken">Token that can be used to cancel the transfer.</param>
/// <returns>A task representing the transfer operation.</returns>
public override async Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
{
if (this.transferJob.Status == TransferJobStatus.Finished ||
this.transferJob.Status == TransferJobStatus.Skipped)
{
return;
}
if (transferJob.Status == TransferJobStatus.Failed)
{
// Resuming a failed transfer job
this.UpdateTransferJobStatus(transferJob, TransferJobStatus.Transfer);
}
try
{
await scheduler.ExecuteJobAsync(transferJob, cancellationToken);
this.UpdateTransferJobStatus(transferJob, TransferJobStatus.Finished);
}
catch (TransferException exception)
{
if (exception.ErrorCode == TransferErrorCode.NotOverwriteExistingDestination)
{
// transfer skipped
this.UpdateTransferJobStatus(transferJob, TransferJobStatus.Skipped);
}
else
{
// transfer failed
this.UpdateTransferJobStatus(transferJob, TransferJobStatus.Failed);
}
throw;
}
}