protected TransferContext GetTransferContext(ProgressRecord record, long totalTransferLength) { TransferContext transferContext = new TransferContext(); transferContext.ClientRequestId = CmdletOperationContext.ClientRequestId; transferContext.OverwriteCallback = ConfirmOverwrite; transferContext.ProgressHandler = new TransferProgressHandler((transferProgress) => { if (record != null) { record.PercentComplete = (int)(transferProgress.BytesTransferred * 100 / totalTransferLength); record.StatusDescription = string.Format(CultureInfo.CurrentCulture, Resources.FileTransmitStatus, record.PercentComplete); this.OutputStream.WriteProgress(record); } }); return transferContext; }
public async Task UploadFileToBlobAsync(string sourceFilePath, BlobOptions options) { Validate.BlobContainerName(options.ContainerName, "containerName"); Validate.BlobName(options.BlobName, "blobName"); var destinationBlob = DataManagerUtility.GetCloudBlob(options); TransferManager.Configurations.ParallelOperations = 64; var context = new TransferContext { OverwriteCallback = (path, destinationPath) => options.OverwriteDestination }; ProgressRecorder = new ProgressRecorder(); context.ProgressHandler = ProgressRecorder; await TransferManager.UploadAsync(sourceFilePath, destinationBlob, null, context, CancellationToken.None); }
protected TransferContext GetTransferContext(DataMovementUserData userData) { TransferContext transferContext = new TransferContext(); transferContext.ClientRequestId = CmdletOperationContext.ClientRequestId; transferContext.OverwriteCallback = ConfirmOverwrite; transferContext.ProgressHandler = new TransferProgressHandler((transferProgress) => { if (userData.Record != null) { // Size of the source file might be 0, when it is, directly treat the progress as 100 percent. userData.Record.PercentComplete = 0 == userData.TotalSize ? 100 : (int)(transferProgress.BytesTransferred * 100 / userData.TotalSize); userData.Record.StatusDescription = string.Format(CultureInfo.CurrentCulture, Resources.FileTransmitStatus, userData.Record.PercentComplete); this.OutputStream.WriteProgress(userData.Record); } }); return transferContext; }
/// <summary> /// Generate an OperationContext from the the specified TransferContext. /// </summary> /// <param name="transferContext">Transfer context</param> /// <returns>An <see cref="OperationContext"/> object.</returns> public static OperationContext GenerateOperationContext( TransferContext transferContext) { if (transferContext == null) { return(null); } OperationContext operationContext = new OperationContext() { LogLevel = transferContext.LogLevel }; if (transferContext.ClientRequestId != null) { operationContext.ClientRequestID = transferContext.ClientRequestId; } return(operationContext); }
public void Apply(TransferContext context) { context.FileTransferred += (sender, transferEventArgs) => { Test.Info("Transfer succeeds: {0} -> {1}", transferEventArgs.Source, transferEventArgs.Destination); this.Increase(TransferEventType.Transferred); }; context.FileSkipped += (sender, transferEventArgs) => { Test.Info("Transfer skips: {0} -> {1}", transferEventArgs.Source, transferEventArgs.Destination); this.Increase(TransferEventType.Skippied); }; context.FileFailed += (sender, transferEventArgs) => { Test.Info("Transfer fails: {0} -> {1}", transferEventArgs.Source, transferEventArgs.Destination); Test.Info("Exception: {0}", transferEventArgs.Exception.ToString()); this.Increase(TransferEventType.Failed); }; }
public void TestProgressHandlerTest() { long fileSize = 10 * 1024 * 1024; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSize); var options = new TestExecutionOptions<DMLibDataInfo>(); options.TransferItemModifier = (fileNode, transferItem) => { TransferContext transferContext = new TransferContext(); ProgressChecker progressChecker = new ProgressChecker(1, fileNode.SizeInByte); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); transferItem.TransferContext = transferContext; }; var result = this.ExecuteTestCase(sourceDataInfo, options); Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown."); Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); }
/// <summary> /// Generate an OperationContext from the the specified TransferContext. /// </summary> /// <param name="transferContext">Transfer context</param> /// <returns>An <see cref="OperationContext"/> object.</returns> public static OperationContext GenerateOperationContext( TransferContext transferContext) { OperationContext operationContext = new OperationContext() { CustomUserAgent = string.Format(CultureInfo.InvariantCulture, "{0} {1}", TransferManager.Configurations.UserAgentPrefix, Constants.UserAgent) }; if (transferContext == null) { return(operationContext); } operationContext.LogLevel = transferContext.LogLevel; if (transferContext.ClientRequestId != null) { operationContext.ClientRequestID = transferContext.ClientRequestId; } return(operationContext); }
public virtual Task DownloadAsync(CloudBlob sourceBlob, string destPath, DownloadOptions options, TransferContext context, CancellationToken cancellationToken) { throw new NotImplementedException(); }
public virtual Task UploadAsync(string sourcePath, CloudBlob destBlob, UploadOptions options, TransferContext context, CancellationToken cancellationToken) { throw new NotImplementedException(); }
/// <summary> /// Upload a file to Azure Blob Storage. /// </summary> /// <param name="sourcePath">Path to the source file.</param> /// <param name="destBlob">The <see cref="CloudBlob"/> that is the destination Azure blob.</param> /// <param name="options">An <see cref="UploadOptions"/> object that specifies additional options for the operation.</param> /// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param> /// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns> /// <param name="cancellationToken">A <see cref="CancellationToken"/> object to observe while waiting for a task to complete.</param> public static Task UploadAsync(string sourcePath, CloudBlob destBlob, UploadOptions options, TransferContext context, CancellationToken cancellationToken) { TransferLocation sourceLocation = new TransferLocation(sourcePath); TransferLocation destLocation = new TransferLocation(destBlob); return UploadInternalAsync(sourceLocation, destLocation, options, context, cancellationToken); }
/// <summary> /// Download an Azure blob from Azure Blob Storage. /// </summary> /// <param name="sourceBlob">The <see cref="CloudBlob"/> that is the source Azure blob.</param> /// <param name="destPath">Path to the destination file.</param> /// <param name="options">A <see cref="DownloadOptions"/> object that specifies additional options for the operation.</param> /// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param> /// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns> public static Task DownloadAsync(CloudBlob sourceBlob, string destPath, DownloadOptions options, TransferContext context) { return DownloadAsync(sourceBlob, destPath, options, context, CancellationToken.None); }
/// <summary> /// Upload a file to Azure File Storage. /// </summary> /// <param name="sourceStream">A <see cref="System.IO.Stream"/> object providing the file content.</param> /// <param name="destFile">The <see cref="CloudFile"/> that is the destination Azure file.</param> /// <param name="options">An <see cref="UploadOptions"/> object that specifies additional options for the operation.</param> /// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param> /// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns> public static Task UploadAsync(Stream sourceStream, CloudFile destFile, UploadOptions options, TransferContext context) { return UploadAsync(sourceStream, destFile, options, context, CancellationToken.None); }
/// <summary> /// Download an Azure file from Azure File Storage. /// </summary> /// <param name="sourceFile">The <see cref="CloudFile"/> that is the source Azure file.</param> /// <param name="destStream">A <see cref="System.IO.Stream"/> object representing the destination stream.</param> /// <param name="options">A <see cref="DownloadOptions"/> object that specifies additional options for the operation.</param> /// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param> /// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns> public static Task DownloadAsync(CloudFile sourceFile, Stream destStream, DownloadOptions options, TransferContext context) { return DownloadAsync(sourceFile, destStream, options, context, CancellationToken.None); }
/// <summary> /// Upload a file to Azure File Storage. /// </summary> /// <param name="sourceStream">A <see cref="System.IO.Stream"/> object providing the file content.</param> /// <param name="destFile">The <see cref="CloudFile"/> that is the destination Azure file.</param> /// <param name="options">An <see cref="UploadOptions"/> object that specifies additional options for the operation.</param> /// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param> /// <param name="cancellationToken">A <see cref="CancellationToken"/> object to observe while waiting for a task to complete.</param> /// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns> public static Task UploadAsync(Stream sourceStream, CloudFile destFile, UploadOptions options, TransferContext context, CancellationToken cancellationToken) { TransferLocation sourceLocation = new TransferLocation(sourceStream); TransferLocation destLocation = new TransferLocation(destFile); return UploadInternalAsync(sourceLocation, destLocation, options, context, cancellationToken); }
/// <summary> /// Copy content, properties and metadata of an Azure file to an Azure blob. /// </summary> /// <param name="sourceFile">The <see cref="CloudFile"/> that is the source Azure file.</param> /// <param name="destBlob">The <see cref="CloudBlob"/> that is the destination Azure blob.</param> /// <param name="isServiceCopy">A flag indicating whether the copy is service-side asynchronous copy or not. /// If this flag is set to true, service-side asychronous copy will be used; if this flag is set to false, /// file is downloaded from source first, then uploaded to destination.</param> /// <param name="options">A <see cref="CopyOptions"/> object that specifies additional options for the operation.</param> /// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param> /// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns> public static Task CopyAsync(CloudFile sourceFile, CloudBlob destBlob, bool isServiceCopy, CopyOptions options, TransferContext context) { return CopyAsync(sourceFile, destBlob, isServiceCopy, options, context, CancellationToken.None); }
private static Task UploadInternalAsync(TransferLocation sourceLocation, TransferLocation destLocation, UploadOptions options, TransferContext context, CancellationToken cancellationToken) { if (options != null) { destLocation.AccessCondition = options.DestinationAccessCondition; } Transfer transfer = CreateSingleObjectTransfer(sourceLocation, destLocation, TransferMethod.SyncCopy, context); if (options != null) { transfer.ContentType = options.ContentType; } return DoTransfer(transfer, cancellationToken); }
private static Task DownloadInternalAsync(TransferLocation sourceLocation, TransferLocation destLocation, DownloadOptions options, TransferContext context, CancellationToken cancellationToken) { if (options != null) { sourceLocation.AccessCondition = options.SourceAccessCondition; } Transfer transfer = CreateSingleObjectTransfer(sourceLocation, destLocation, TransferMethod.SyncCopy, context); return DoTransfer(transfer, cancellationToken); }
/// <summary> /// Copy file from an specified URI to an Azure file. /// </summary> /// <param name="sourceUri">The <see cref="System.Uri"/> of the source file.</param> /// <param name="destFile">The <see cref="CloudFile"/> that is the destination Azure file.</param> /// <param name="isServiceCopy">A flag indicating whether the copy is service-side asynchronous copy or not. /// If this flag is set to true, service-side asychronous copy will be used; if this flag is set to false, /// file is downloaded from source first, then uploaded to destination.</param> /// <param name="options">A <see cref="CopyOptions"/> object that specifies additional options for the operation.</param> /// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param> /// <param name="cancellationToken">A <see cref="CancellationToken"/> object to observe while waiting for a task to complete.</param> /// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns> /// <remarks>Copying from an URI to Azure file synchronously is not supported yet.</remarks> public static Task CopyAsync(Uri sourceUri, CloudFile destFile, bool isServiceCopy, CopyOptions options, TransferContext context, CancellationToken cancellationToken) { if (!isServiceCopy) { throw new NotSupportedException(Resources.SyncCopyFromUriToAzureFileNotSupportedException); } TransferLocation sourceLocation = new TransferLocation(sourceUri); TransferLocation destLocation = new TransferLocation(destFile); return CopyInternalAsync(sourceLocation, destLocation, isServiceCopy, options, context, cancellationToken); }
/// <summary> /// Copy file from an specified URI to an Azure file. /// </summary> /// <param name="sourceUri">The <see cref="System.Uri"/> of the source file.</param> /// <param name="destFile">The <see cref="CloudFile"/> that is the destination Azure file.</param> /// <param name="isServiceCopy">A flag indicating whether the copy is service-side asynchronous copy or not. /// If this flag is set to true, service-side asychronous copy will be used; if this flag is set to false, /// file is downloaded from source first, then uploaded to destination.</param> /// <param name="options">A <see cref="CopyOptions"/> object that specifies additional options for the operation.</param> /// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param> /// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns> /// <remarks>Copying from an URI to Azure file synchronously is not supported yet.</remarks> public static Task CopyAsync(Uri sourceUri, CloudFile destFile, bool isServiceCopy, CopyOptions options, TransferContext context) { return CopyAsync(sourceUri, destFile, isServiceCopy, options, context, CancellationToken.None); }
/// <summary> /// Copy content, properties and metadata of an Azure file to another. /// </summary> /// <param name="sourceFile">The <see cref="CloudFile"/> that is the source Azure file.</param> /// <param name="destFile">The <see cref="CloudFile"/> that is the destination Azure file.</param> /// <param name="isServiceCopy">A flag indicating whether the copy is service-side asynchronous copy or not. /// If this flag is set to true, service-side asychronous copy will be used; if this flag is set to false, /// file is downloaded from source first, then uploaded to destination.</param> /// <param name="options">A <see cref="CopyOptions"/> object that specifies additional options for the operation.</param> /// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param> /// <param name="cancellationToken">A <see cref="CancellationToken"/> object to observe while waiting for a task to complete.</param> /// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns> public static Task CopyAsync(CloudFile sourceFile, CloudFile destFile, bool isServiceCopy, CopyOptions options, TransferContext context, CancellationToken cancellationToken) { TransferLocation sourceLocation = new TransferLocation(sourceFile); TransferLocation destLocation = new TransferLocation(destFile); return CopyInternalAsync(sourceLocation, destLocation, isServiceCopy, options, context, cancellationToken); }
public void TestResume() { int fileSizeInKB = 100 * 1024; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSizeInKB); CancellationTokenSource tokenSource = new CancellationTokenSource(); TransferItem transferItem = null; var options = new TestExecutionOptions<DMLibDataInfo>(); options.LimitSpeed = true; var transferContext = new TransferContext(); var progressChecker = new ProgressChecker(1, fileSizeInKB * 1024); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); options.TransferItemModifier = (fileName, item) => { item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; transferItem = item; }; TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null; options.AfterAllItemAdded = () => { // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); // Store the first checkpoint firstCheckpoint = transferContext.LastCheckpoint; Thread.Sleep(1000); // Cancel the transfer and store the second checkpoint tokenSource.Cancel(); secondCheckpoint = transferContext.LastCheckpoint; }; // Cancel and store checkpoint for resume var result = this.ExecuteTestCase(sourceDataInfo, options); Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled"); Exception exception = result.Exceptions[0]; VerificationHelper.VerifyExceptionErrorMessage(exception, "A task was canceled."); TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null; // DMLib doesn't support to resume transfer from a checkpoint which is inconsistent with // the actual transfer progress when the destination is an append blob. if (Helper.RandomBoolean() && DMLibTestContext.DestType != DMLibDataType.AppendBlob) { Test.Info("Resume with the first checkpoint first."); firstResumeCheckpoint = firstCheckpoint; secondResumeCheckpoint = secondCheckpoint; } else { Test.Info("Resume with the second checkpoint first."); firstResumeCheckpoint = secondCheckpoint; secondResumeCheckpoint = firstCheckpoint; } // resume with firstResumeCheckpoint TransferItem resumeItem = transferItem.Clone(); progressChecker.Reset(); TransferContext resumeContext = new TransferContext(firstResumeCheckpoint) { ProgressHandler = progressChecker.GetProgressHandler() }; resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List<TransferItem>() { resumeItem }, new TestExecutionOptions<DMLibDataInfo>()); VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo); // resume with secondResumeCheckpoint resumeItem = transferItem.Clone(); progressChecker.Reset(); resumeContext = new TransferContext(secondResumeCheckpoint) { ProgressHandler = progressChecker.GetProgressHandler() }; resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List<TransferItem>() { resumeItem }, new TestExecutionOptions<DMLibDataInfo>()); if (DMLibTestContext.DestType != DMLibDataType.AppendBlob || DMLibTestContext.SourceType == DMLibDataType.Stream) { VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo); } else { Test.Assert(result.Exceptions.Count == 1, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob."); exception = result.Exceptions[0]; Test.Assert(exception is InvalidOperationException, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob."); VerificationHelper.VerifyExceptionErrorMessage(exception, "Destination might be changed by other process or application."); } }
public void DirectoryOverwriteDestination() { string destExistYName = "destExistY"; string destExistNName = "destExistN"; string destNotExistYName = "destNotExistY"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistNName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destNotExistYName, 1024); DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistNName, 1024); TransferContext transferContext = new TransferContext(); transferContext.OverwriteCallback = (string sourcePath, string destinationPath) => { if (sourcePath.EndsWith(destExistNName)) { return false; } else { return true; } }; int skipCount = 0; int successCount = 0; transferContext.FileSkipped += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref skipCount); TransferException transferException = args.Exception as TransferException; Test.Assert(transferException != null, "Verify the exception is a TransferException"); VerificationHelper.VerifyTransferException(transferException, TransferErrorCode.NotOverwriteExistingDestination, "Skiped file", destExistNName); }; transferContext.FileTransferred += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref successCount); }; var options = new TestExecutionOptions<DMLibDataInfo>(); options.IsDirectoryTransfer = true; if (DMLibTestContext.DestType != DMLibDataType.Stream) { options.DestTransferDataInfo = destDataInfo; } options.TransferItemModifier = (fileNode, transferItem) => { transferItem.TransferContext = transferContext; dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; transferItem.Options = transferOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); DMLibDataInfo expectedDataInfo = new DMLibDataInfo(string.Empty); if (DMLibTestContext.DestType != DMLibDataType.Stream) { expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destExistYName)); expectedDataInfo.RootNode.AddFileNode(destDataInfo.RootNode.GetFileNode(destExistNName)); expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destNotExistYName)); } else { expectedDataInfo = sourceDataInfo; } // Verify transfer result Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, result.DataInfo), "Verify transfer result."); // Verify exception if (DMLibTestContext.DestType != DMLibDataType.Stream) { Test.Assert(successCount == 2, "Verify success transfers"); Test.Assert(skipCount == 1, "Verify skipped transfer"); } else { Test.Assert(successCount == 3, "Very all transfers are success"); Test.Assert(skipCount == 0, "Very no transfer is skipped"); } }
private static Task CopyInternalAsync(TransferLocation sourceLocation, TransferLocation destLocation, bool isServiceCopy, CopyOptions options, TransferContext context, CancellationToken cancellationToken) { if (options != null) { sourceLocation.AccessCondition = options.SourceAccessCondition; destLocation.AccessCondition = options.DestinationAccessCondition; } Transfer transfer = CreateSingleObjectTransfer(sourceLocation, destLocation, isServiceCopy ? TransferMethod.AsyncCopy : TransferMethod.SyncCopy, context); return DoTransfer(transfer, cancellationToken); }
/// <summary> /// Download an Azure file from Azure File Storage. /// </summary> /// <param name="sourceFile">The <see cref="CloudFile"/> that is the source Azure file.</param> /// <param name="destStream">A <see cref="System.IO.Stream"/> object representing the destination stream.</param> /// <param name="options">A <see cref="DownloadOptions"/> object that specifies additional options for the operation.</param> /// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param> /// <param name="cancellationToken">A <see cref="CancellationToken"/> object to observe while waiting for a task to complete.</param> /// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns> public static Task DownloadAsync(CloudFile sourceFile, Stream destStream, DownloadOptions options, TransferContext context, CancellationToken cancellationToken) { TransferLocation sourceLocation = new TransferLocation(sourceFile); TransferLocation destLocation = new TransferLocation(destStream); if (options != null) { FileRequestOptions requestOptions = Transfer_RequestOptions.DefaultFileRequestOptions; requestOptions.DisableContentMD5Validation = options.DisableContentMD5Validation; sourceLocation.RequestOptions = requestOptions; } return DownloadInternalAsync(sourceLocation, destLocation, options, context, cancellationToken); }
/// <summary> /// Generate an OperationContext from the the specified TransferContext. /// </summary> /// <param name="transferContext">Transfer context</param> /// <returns>An <see cref="OperationContext"/> object.</returns> public static OperationContext GenerateOperationContext( TransferContext transferContext) { if (transferContext == null) { return null; } return new OperationContext() { ClientRequestID = transferContext.ClientRequestId, LogLevel = transferContext.LogLevel, }; }
public async static Task ProcessMessage([QueueTrigger("backupqueue")] CopyItem copyItem, TextWriter log, CancellationToken cancelToken) { // Copy TextWrite into Log Helper class Logger.log = log; // Log Job Start await Logger.JobStartAsync(copyItem.JobName); // This class accumulates transfer data during the copy ProgressRecorder progressRecorder = new ProgressRecorder(); try { // OpContext to track PreCopy Retries on Azure Storage // DML has its own context object and retry _opContext = new OperationContext(); _opContext.Retrying += StorageRequest_Retrying; // Define Blob Request Options _blobRequestOptions = new BlobRequestOptions { // Defined Exponential Retry Policy above RetryPolicy = _retryPolicy }; // Set the number of parallel tasks in DML. // This allows it to copy multiple items at once when copying a container or directory // The best (and default value) is Environment.ProcessorCount * 8 int parallelTasks = Environment.ProcessorCount * 8; TransferManager.Configurations.ParallelOperations = parallelTasks; // Set the number of connections. // This should match ParallelOperations so each DML copy task has its own connection to Azure Storage ServicePointManager.DefaultConnectionLimit = parallelTasks; // Short circuit additional request round trips. We are not chunking and // uploading large amounts of data where we'd send 100's so set to false ServicePointManager.Expect100Continue = false; // User Agent for tracing TransferManager.Configurations.UserAgentPrefix = "AzureDmlBackup"; // CancellationTokenSource used to cancel the transfer CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); // Open connections to both storage accounts CloudStorageAccount sourceAccount = GetAccount(copyItem.SourceAccountToken); CloudStorageAccount destinationAccount = GetAccount(copyItem.DestinationAccountToken); // Represents a checkpoint from which a transfer may be resumed and continued. // This is initalized as null first time then hydrated within CopyDirectoryAsync(). // However if this job is being resumed from a previous failure this function will hydrate // from a serialized checkpoint saved to blob storage. TransferCheckpoint transferCheckpoint = await GetTransferCheckpoint(copyItem.JobId); // Context object for the transfer, provides additional runtime information about its execution // If this is a resumed copy operation then pass the checkpoint to the TransferContext so it can resume the copy TransferContext transferContext = new TransferContext(transferCheckpoint) { // Pipe transfer progress data to ProgressRecorder // ProgressRecorder is used to log the results of the copy operation ProgressHandler = progressRecorder, // If the destination already exists this delegate is called. // Return true to overwrite or false to skip the file during the transfer OverwriteCallback = (source, destination) => { return OverwriteFile(source, destination, sourceAccount, destinationAccount, copyItem.IsIncremental); } }; // This event is used to log files skipped during the transfer transferContext.FileSkipped += TransferContext_FileSkipped; // This event is used to catch exceptions for files that fail during a transfer transferContext.FileFailed += TransferContext_FileFailed; // Set Options for copying the container such as search patterns, recursive, etc. CopyDirectoryOptions copyDirectoryOptions = new CopyDirectoryOptions { IncludeSnapshots = true, Recursive = true }; // Get the root source and destination directories for the two containers to be copied CloudBlobDirectory sourceDirectory = await GetDirectoryAsync(sourceAccount, copyItem.SourceContainer, copyItem.SourceDirectory); CloudBlobDirectory destinationDirectory = await GetDirectoryAsync(destinationAccount, copyItem.DestinationContainer, copyItem.DestinationDirectory); // Copy the container await CopyDirectoryAsync(copyItem.JobId, sourceDirectory, destinationDirectory, copyDirectoryOptions, transferContext, transferCheckpoint, cancellationTokenSource); // Check if any files failed during transfer if (_failedFiles.Count > 0) { // Save a Checkpoint so we can restart the transfer transferCheckpoint = transferContext.LastCheckpoint; SaveTransferCheckpoint(copyItem.JobId, transferCheckpoint); // Throw an exception to fail the job so WebJobs will rerun it throw new Exception("One or more errors occurred during the transfer."); } // Log job completion await Logger.JobCompleteAsync(copyItem.JobName, progressRecorder, _skippedFiles); } catch (Exception ex) { // Log Job Error await Logger.JobErrorAsync(copyItem.JobName, ex.Message, progressRecorder, _failedFiles, _skippedFiles); // Rethrow the error to fail the web job throw ex; } }
public void OverwriteDestination() { string destExistYName = "destExistY"; string destExistNName = "destExistN"; string destNotExistYName = "destNotExistY"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistNName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destNotExistYName, 1024); DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistNName, 1024); var options = new TestExecutionOptions<DMLibDataInfo>(); if (DMLibTestContext.DestType != DMLibDataType.Stream) { options.DestTransferDataInfo = destDataInfo; } options.TransferItemModifier = (fileNode, transferItem) => { string fileName = fileNode.Name; TransferContext transferContext = new TransferContext(); if (fileName.Equals(destExistYName)) { transferContext.OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY(); } else if (fileName.Equals(destExistNName)) { transferContext.OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackN(); } else if (fileName.Equals(destNotExistYName)) { transferContext.OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY(); } transferItem.TransferContext = transferContext; }; var result = this.ExecuteTestCase(sourceDataInfo, options); DMLibDataInfo expectedDataInfo = new DMLibDataInfo(string.Empty); if (DMLibTestContext.DestType != DMLibDataType.Stream) { expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destExistYName)); expectedDataInfo.RootNode.AddFileNode(destDataInfo.RootNode.GetFileNode(destExistNName)); expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destNotExistYName)); } else { expectedDataInfo = sourceDataInfo; } // Verify transfer result Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, result.DataInfo), "Verify transfer result."); // Verify exception if (DMLibTestContext.DestType != DMLibDataType.Stream) { Test.Assert(result.Exceptions.Count == 1, "Verify there's only one exceptions."); TransferException transferException = result.Exceptions[0] as TransferException; Test.Assert(transferException != null, "Verify the exception is a TransferException"); VerificationHelper.VerifyTransferException(transferException, TransferErrorCode.NotOverwriteExistingDestination, "Skiped file", destExistNName); } }
/// <summary> /// Upload a file to Azure Blob Storage. /// </summary> /// <param name="sourcePath">Path to the source file.</param> /// <param name="destBlob">The <see cref="CloudBlob"/> that is the destination Azure blob.</param> /// <param name="options">An <see cref="UploadOptions"/> object that specifies additional options for the operation.</param> /// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param> /// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns> public static Task UploadAsync(string sourcePath, CloudBlob destBlob, UploadOptions options, TransferContext context) { return UploadAsync(sourcePath, destBlob, options, context, CancellationToken.None); }
private static Transfer CreateSingleObjectTransfer(TransferLocation sourceLocation, TransferLocation destLocation, TransferMethod transferMethod, TransferContext transferContext) { Transfer transfer = GetTransfer(sourceLocation, destLocation, transferMethod, transferContext); if (transfer == null) { transfer = new SingleObjectTransfer(sourceLocation, destLocation, transferMethod); if (transferContext != null) { transferContext.Checkpoint.AddTransfer(transfer); } } if (transferContext != null) { transfer.ProgressTracker.Parent = transferContext.OverallProgressTracker; transfer.Context = transferContext; } return transfer; }
/// <summary> /// Download an Azure blob from Azure Blob Storage. /// </summary> /// <param name="sourceBlob">The <see cref="CloudBlob"/> that is the source Azure blob.</param> /// <param name="destPath">Path to the destination file.</param> /// <param name="options">A <see cref="DownloadOptions"/> object that specifies additional options for the operation.</param> /// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param> /// <param name="cancellationToken">A <see cref="CancellationToken"/> object to observe while waiting for a task to complete.</param> /// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns> public static Task DownloadAsync(CloudBlob sourceBlob, string destPath, DownloadOptions options, TransferContext context, CancellationToken cancellationToken) { TransferLocation sourceLocation = new TransferLocation(sourceBlob); TransferLocation destLocation = new TransferLocation(destPath); if (options != null) { BlobRequestOptions requestOptions = Transfer_RequestOptions.DefaultBlobRequestOptions; requestOptions.DisableContentMD5Validation = options.DisableContentMD5Validation; sourceLocation.RequestOptions = requestOptions; } return DownloadInternalAsync(sourceLocation, destLocation, options, context, cancellationToken); }
public void ResumeInAllDirections() { List<TransferItem> allItems = AllTransferDirectionTest.GetTransformItemsForAllDirections(resume: true); int fileCount = expectedFileNodes.Keys.Count; // Execution and store checkpoints CancellationTokenSource tokenSource = new CancellationTokenSource(); var transferContext = new TransferContext(); var progressChecker = new ProgressChecker(fileCount, 1024 * fileCount); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); allItems.ForEach(item => { item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; }); var options = new TestExecutionOptions<DMLibDataInfo>(); options.DisableDestinationFetch = true; // Checkpoint names const string PartialStarted = "PartialStarted", AllStarted = "AllStarted", AllStartedAndWait = "AllStartedAndWait", BeforeCancel = "BeforeCancel", AfterCancel = "AfterCancel"; Dictionary<string, TransferCheckpoint> checkpoints = new Dictionary<string, TransferCheckpoint>(); TransferItem randomItem = allItems[random.Next(0, allItems.Count)]; randomItem.AfterStarted = () => { Test.Info("Store check point after transfer item: {0}.", randomItem.ToString()); checkpoints.Add(PartialStarted, transferContext.LastCheckpoint); }; options.AfterAllItemAdded = () => { progressChecker.DataTransferred.WaitOne(); checkpoints.Add(AllStarted, transferContext.LastCheckpoint); Thread.Sleep(1000); checkpoints.Add(AllStartedAndWait, transferContext.LastCheckpoint); Thread.Sleep(1000); checkpoints.Add(BeforeCancel, transferContext.LastCheckpoint); tokenSource.Cancel(); checkpoints.Add(AfterCancel, transferContext.LastCheckpoint); }; var result = this.RunTransferItems(allItems, options); // Resume with stored checkpoints in random order var checkpointList = new List<KeyValuePair<string,TransferCheckpoint>>(); checkpointList.AddRange(checkpoints); checkpointList.Shuffle(); foreach(var pair in checkpointList) { Test.Info("===Resume with checkpoint '{0}'===", pair.Key); options = new TestExecutionOptions<DMLibDataInfo>(); options.DisableDestinationFetch = true; progressChecker.Reset(); transferContext = new TransferContext(DMLibTestHelper.RandomReloadCheckpoint(pair.Value)) { ProgressHandler = progressChecker.GetProgressHandler(), // The checkpoint can be stored when DMLib doesn't check overwrite callback yet. // So it will case an skip file error if the desination file already exists and // We don't have overwrite callback here. OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY() }; TransferEventChecker eventChecker = new TransferEventChecker(); eventChecker.Apply(transferContext); List<TransferItem> itemsToResume = allItems.Select(item => { TransferItem itemToResume = item.Clone(); itemToResume.TransferContext = transferContext; return itemToResume; }).ToList(); result = this.RunTransferItems(itemsToResume, options); foreach (DMLibDataType destDataType in DataTypes) { DataAdaptor<DMLibDataInfo> destAdaptor = GetSourceAdaptor(destDataType); DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty); foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes()) { string fileName = destFileNode.Name; FileNode sourceFileNode = expectedFileNodes[fileName]; Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result."); } } Test.Assert(result.Exceptions.Count == 0, "Verify no error happens. Actual: {0}", result.Exceptions.Count); } }
private static Transfer GetTransfer(TransferLocation sourceLocation, TransferLocation destLocation, TransferMethod transferMethod, TransferContext transferContext) { Transfer transfer = null; if (transferContext != null) { transfer = transferContext.Checkpoint.GetTransfer(sourceLocation, destLocation, transferMethod); if (transfer != null) { // update transfer location information UpdateTransferLocation(transfer.Source, sourceLocation); UpdateTransferLocation(transfer.Destination, destLocation); } } return transfer; }