private void EnumerateAndTransfer(TransferScheduler scheduler, CancellationToken cancellationToken) { Interlocked.Increment(ref this.outstandingTasks); try { foreach (var transfer in this.AllTransfers(cancellationToken)) { this.CheckAndPauseEnumeration(cancellationToken); transfer.UpdateProgressLock(this.progressUpdateLock); this.DoTransfer(transfer, scheduler, cancellationToken); } } catch (StorageException e) { throw new TransferException(TransferErrorCode.FailToEnumerateDirectory, e.GetExceptionMessage(), e); } finally { if (Interlocked.Decrement(ref this.outstandingTasks) == 0) { // make sure transfer terminiate when there is no subtransfer. this.allTransfersCompleteSource.SetResult(null); } } }
private static ServiceSideSyncCopyController CreateServiceSideSyncCopyConstroller( TransferScheduler transferScheduler, TransferJob transferJob, CancellationToken cancellationToken) { CloudBlob destinationBlob = transferJob.Destination.Instance as CloudBlob; if (null == destinationBlob) { throw new TransferException(Resources.ServiceSideSyncCopyNotSupportException); } if (BlobType.PageBlob == destinationBlob.BlobType) { return(new PageBlobServiceSideSyncCopyController(transferScheduler, transferJob, cancellationToken)); } else if (BlobType.AppendBlob == destinationBlob.BlobType) { return(new AppendBlobServiceSideSyncCopyController(transferScheduler, transferJob, cancellationToken)); } else if (BlobType.BlockBlob == destinationBlob.BlobType) { return(new BlockBlobServiceSideSyncCopyController(transferScheduler, transferJob, cancellationToken)); } else { throw new TransferException(string.Format(CultureInfo.CurrentCulture, Resources.NotSupportedBlobType, destinationBlob.BlobType)); } }
/// <summary> /// Execute the transfer asynchronously. /// </summary> /// <param name="scheduler">Transfer scheduler</param> /// <param name="cancellationToken">Token that can be used to cancel the transfer.</param> /// <returns>A task representing the transfer operation.</returns> public override async Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken) { this.ResetExecutionStatus(); this.Destination.Validate(); try { Task listTask = Task.Run(() => this.ListNewTransfers(cancellationToken)); await Task.Run(() => { this.EnumerateAndTransfer(scheduler, cancellationToken); }); await listTask; } finally { // wait for outstanding transfers to complete await allTransfersCompleteSource.Task; } if (this.enumerateException != null) { throw this.enumerateException; } this.ProgressTracker.AddBytesTransferred(0); }
private static TransferControllerBase GenerateTransferConstroller( TransferScheduler transferScheduler, TransferJob transferJob, CancellationToken cancellationToken) { TransferControllerBase controller = null; switch (transferJob.Transfer.TransferMethod) { case TransferMethod.SyncCopy: controller = new SyncTransferController(transferScheduler, transferJob, cancellationToken); break; case TransferMethod.ServiceSideAsyncCopy: controller = CreateAsyncCopyController(transferScheduler, transferJob, cancellationToken); break; case TransferMethod.ServiceSideSyncCopy: controller = CreateServiceSideSyncCopyConstroller(transferScheduler, transferJob, cancellationToken); break; case TransferMethod.DummyCopy: controller = new DummyTransferController(transferScheduler, transferJob, cancellationToken); break; } return(controller); }
private static AsyncCopyController CreateAsyncCopyController(TransferScheduler transferScheduler, TransferJob transferJob, CancellationToken cancellationToken) { if (transferJob.Destination.Type == TransferLocationType.AzureFile) { return(new FileAsyncCopyController(transferScheduler, transferJob, cancellationToken)); } if (transferJob.Destination.Type == TransferLocationType.AzureBlob) { return(new BlobAsyncCopyController(transferScheduler, transferJob, cancellationToken)); } throw new InvalidOperationException(Resources.CanOnlyCopyToFileOrBlobException); }
public override async Task ExecuteInternalAsync(TransferScheduler scheduler, CancellationToken cancellationToken) { try { await Task.Yield(); this.ResetExecutionStatus(); cancellationToken.Register(() => { this.cancellationTokenSource?.Cancel(); }); this.currentScheduler = scheduler; this.currentCancellationToken = cancellationToken; this.maxConcurrencyControl.Wait(); await DoEnumerationAndTransferAsync(scheduler, cancellationToken); } catch (OperationCanceledException) { if (null == this.enumerateException) { throw; } } finally { this.SignalSubDirTaskDecrement(); await this.subDirTransfersCompleteSource.Task; if (this.maxConcurrency == this.maxConcurrencyControl.Release()) { this.transfersCompleteSource.TrySetResult(null); } await this.transfersCompleteSource.Task; } if (null != this.enumerateException) { throw this.enumerateException; } else { Utils.CheckCancellation(cancellationToken); } }
/// <summary> /// Execute the transfer asynchronously. /// </summary> /// <param name="scheduler">Transfer scheduler</param> /// <param name="cancellationToken">Token that can be used to cancel the transfer.</param> /// <returns>A task representing the transfer operation.</returns> public override async Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken) { try { this.Destination.Validate(); } catch (StorageException se) { throw new TransferException(TransferErrorCode.FailToVadlidateDestination, Resources.FailedToValidateDestinationException, se); } catch (Exception ex) { throw new TransferException(TransferErrorCode.FailToVadlidateDestination, Resources.FailedToValidateDestinationException, ex); } this.nameResolver = GetNameResolver(this.Source, this.Destination, this.Delimiter); await this.ExecuteInternalAsync(scheduler, cancellationToken); }
private async void DoTransfer(Transfer transfer, TransferScheduler scheduler, CancellationToken cancellationToken) { using (transfer) { bool hasError = false; Interlocked.Increment(ref this.outstandingTasks); try { await transfer.ExecuteAsync(scheduler, cancellationToken); } catch { // catch exception thrown from sub-transfer as it's already recorded hasError = true; } finally { // Don't keep the failed transferring in memory, if the checkpoint is persist to a streamed journal, // instead, should only keep them in the streamed journal. if ((!hasError) || (null != this.Journal)) { this.subTransfers.RemoveTransfer(transfer); } this.enumerationResetEvent.Set(); if (Interlocked.Decrement(ref this.outstandingTasks) == 0) { // all transfers are done this.allTransfersCompleteSource.SetResult(null); } } } }
/// <summary> /// Execute the transfer asynchronously. /// </summary> /// <param name="scheduler">Transfer scheduler</param> /// <param name="cancellationToken">Token that can be used to cancel the transfer.</param> /// <returns>A task representing the transfer operation.</returns> public override async Task ExecuteAsync(TransferScheduler scheduler, CancellationToken cancellationToken) { if (this.transferJob.Status == TransferJobStatus.Finished || this.transferJob.Status == TransferJobStatus.Skipped) { return; } TransferEventArgs eventArgs = new TransferEventArgs(this.Source.Instance, this.Destination.Instance); eventArgs.StartTime = DateTime.UtcNow; if (this.transferJob.Status == TransferJobStatus.Failed) { // Resuming a failed transfer job if (string.IsNullOrEmpty(this.transferJob.CopyId)) { this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Transfer); } else { this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Monitor); } } try { await scheduler.ExecuteJobAsync(this.transferJob, cancellationToken); if (TransferJobStatus.SkippedDueToShouldNotTransfer != this.transferJob.Status) { eventArgs.EndTime = DateTime.UtcNow; this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Finished); if (this.Context != null) { this.Context.OnTransferSuccess(eventArgs); } } } catch (TransferException exception) { eventArgs.EndTime = DateTime.UtcNow; eventArgs.Exception = exception; if (exception.ErrorCode == TransferErrorCode.NotOverwriteExistingDestination) { // transfer skipped this.UpdateTransferJobStatus(this.transferJob, TransferJobStatus.Skipped); if (this.Context != null) { this.Context.OnTransferSkipped(eventArgs); } throw; } else if (exception.ErrorCode == TransferErrorCode.FailedCheckingShouldTransfer) { throw; } else { this.OnTransferFailed(eventArgs); throw; } } catch (Exception ex) { eventArgs.EndTime = DateTime.UtcNow; eventArgs.Exception = ex; this.OnTransferFailed(eventArgs); throw; } this.Journal?.RemoveTransfer(this); }
public abstract Task ExecuteInternalAsync(TransferScheduler scheduler, CancellationToken cancellationToken);
private async Task DoEnumerationAndTransferAsync(TransferScheduler scheduler, CancellationToken cancellationToken) { await Task.Yield(); this.Resume(scheduler, cancellationToken); if (0 == this.enumerationStarted) { // Got nothing from checkpoint, start directory transfer from the very beginning. var subDirTransfer = new SubDirectoryTransfer(this, ""); lock (this.continuationTokenLock) { if (null == this.Journal) { this.ongoingSubDirTransfers.TryAdd(subDirTransfer, new object()); } this.enumerationStarted = 1; } if (null != this.Journal) { this.Journal.AddOngoingSubDirTransfer(subDirTransfer); this.Journal?.UpdateJournalItem(this); } this.ScheduleSubDirectoryTransfer( subDirTransfer, this.cancellationTokenSource.Token, null, -1); } bool gotDirectory = true; while (true) { Utils.CheckCancellation(this.cancellationTokenSource.Token); if (!gotDirectory) { newAddSubDirResetEventSlim.Wait(cancellationToken); newAddSubDirResetEventSlim.Reset(); } // Check whether theres ongoing subdirectory listing thread. bool listCompleted = (1 == Interlocked.Read(ref this.outstandingTasks)); string subDirRelativePath = null; if (null != this.Journal) { subDirRelativePath = this.Journal.PeekSubDirTransfer(); } else { this.subDirectories.TryPeek(out subDirRelativePath); } if (string.IsNullOrEmpty(subDirRelativePath)) { if (listCompleted) { // There's no ongoing subdirectory listing thread, // and no subdirectory pending on listing // This means that the whole listing is completed. break; } gotDirectory = false; continue; } else { gotDirectory = true; Utils.CheckCancellation(this.cancellationTokenSource.Token); SubDirectoryTransfer subDirTransfer = new SubDirectoryTransfer(this, subDirRelativePath); this.ScheduleSubDirectoryTransfer( subDirTransfer, this.cancellationTokenSource.Token, () => { if (null != this.Journal) { this.Journal.AddOngoingSubDirTransfer(subDirTransfer); this.Journal.RemoveFirstSubDirTransfer(); } else { this.ongoingSubDirTransfers.TryAdd(subDirTransfer, new object()); this.subDirectories.TryDequeue(out subDirRelativePath); } }, -1); } } }
private bool Resume(TransferScheduler scheduler, CancellationToken cancellationToken) { bool isResume = false; if (null != this.Journal) { foreach (var transfer in this.Journal.ListSubTransfers()) { isResume = true; Utils.CheckCancellation(this.cancellationTokenSource.Token); transfer.Context = this.Context; this.UpdateTransfer(transfer); this.subTransfers.AddTransfer(transfer, false); this.TransferFile((transfer as SingleObjectTransfer), scheduler, cancellationToken); } foreach (var subDirTransfer in this.Journal.ListSubDirTransfers()) { isResume = true; Utils.CheckCancellation(this.cancellationTokenSource.Token); subDirTransfer.Update(this); this.ScheduleSubDirectoryTransfer( subDirTransfer, this.cancellationTokenSource.Token, () => { }, -1); } } else { // return all existing transfers in subTransfers foreach (var transfer in this.subTransfers.GetEnumerator()) { isResume = true; Utils.CheckCancellation(this.cancellationTokenSource.Token); transfer.Context = this.Context; this.UpdateTransfer(transfer); this.TransferFile((transfer as SingleObjectTransfer), scheduler, cancellationToken); } foreach (var subDirTransferPair in this.ongoingSubDirTransfers) { isResume = true; var subDirTransfer = subDirTransferPair.Key; subDirTransfer.Update(this); this.ScheduleSubDirectoryTransfer( subDirTransfer, this.cancellationTokenSource.Token, () => { }, -1); } } return(isResume); }
internal async void TransferFile(SingleObjectTransfer transferItem, TransferScheduler scheduler, CancellationToken cancellationToken) { try { this.maxConcurrencyControl.Wait(cancellationToken); } catch (OperationCanceledException) { // No need to report exception here, OperationCanceledException will be handled in other place. return; } catch (ObjectDisposedException) { return; } await Task.Yield(); bool hasError = false; bool shouldStopTransfer = false; transferItem.UpdateProgressLock(this.progressUpdateLock); try { using (transferItem) { await transferItem.ExecuteAsync(scheduler, cancellationToken); } } catch (TransferException ex) { if (ex.ErrorCode == TransferErrorCode.FailedCheckingShouldTransfer) { shouldStopTransfer = true; this.enumerateException = new TransferException( TransferErrorCode.FailToEnumerateDirectory, string.Format(CultureInfo.CurrentCulture, Resources.EnumerateDirectoryException, this.Destination.Instance.ConvertToString()), ex.InnerException); } hasError = true; } catch { hasError = true; } finally { if ((!hasError) || (null != this.Journal)) { this.subTransfers.RemoveTransfer(transferItem); } if (this.maxConcurrency == this.maxConcurrencyControl.Release()) { this.transfersCompleteSource.TrySetResult(null); } } if (shouldStopTransfer) { this.cancellationTokenSource?.Cancel(); } }