public async static Task ProcessMessage([QueueTrigger("backupqueue")] CopyItem copyItem, TextWriter log, CancellationToken cancelToken) { // Copy TextWrite into Log Helper class Logger.log = log; // Log Job Start await Logger.JobStartAsync(copyItem.JobName); // This class accumulates transfer data during the copy ProgressRecorder progressRecorder = new ProgressRecorder(); try { // OpContext to track PreCopy Retries on Azure Storage // DML has its own context object and retry _opContext = new OperationContext(); _opContext.Retrying += StorageRequest_Retrying; // Define Blob Request Options _blobRequestOptions = new BlobRequestOptions { // Defined Exponential Retry Policy above RetryPolicy = _retryPolicy }; // Set the number of parallel tasks in DML. // This allows it to copy multiple items at once when copying a container or directory // The best (and default value) is Environment.ProcessorCount * 8 int parallelTasks = Environment.ProcessorCount * 8; TransferManager.Configurations.ParallelOperations = parallelTasks; // Set the number of connections. // This should match ParallelOperations so each DML copy task has its own connection to Azure Storage ServicePointManager.DefaultConnectionLimit = parallelTasks; // Short circuit additional request round trips. We are not chunking and // uploading large amounts of data where we'd send 100's so set to false ServicePointManager.Expect100Continue = false; // User Agent for tracing TransferManager.Configurations.UserAgentPrefix = "AzureDmlBackup"; // CancellationTokenSource used to cancel the transfer CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); // Open connections to both storage accounts CloudStorageAccount sourceAccount = GetAccount(copyItem.SourceAccountToken); CloudStorageAccount destinationAccount = GetAccount(copyItem.DestinationAccountToken); // Represents a checkpoint from which a transfer may be resumed and continued. // This is initalized as null first time then hydrated within CopyDirectoryAsync(). // However if this job is being resumed from a previous failure this function will hydrate // from a serialized checkpoint saved to blob storage. TransferCheckpoint transferCheckpoint = await GetTransferCheckpoint(copyItem.JobId); // Context object for the transfer, provides additional runtime information about its execution // If this is a resumed copy operation then pass the checkpoint to the TransferContext so it can resume the copy TransferContext transferContext = new TransferContext(transferCheckpoint) { // Pipe transfer progress data to ProgressRecorder // ProgressRecorder is used to log the results of the copy operation ProgressHandler = progressRecorder, // If the destination already exists this delegate is called. // Return true to overwrite or false to skip the file during the transfer OverwriteCallback = (source, destination) => { return OverwriteFile(source, destination, sourceAccount, destinationAccount, copyItem.IsIncremental); } }; // This event is used to log files skipped during the transfer transferContext.FileSkipped += TransferContext_FileSkipped; // This event is used to catch exceptions for files that fail during a transfer transferContext.FileFailed += TransferContext_FileFailed; // Set Options for copying the container such as search patterns, recursive, etc. CopyDirectoryOptions copyDirectoryOptions = new CopyDirectoryOptions { IncludeSnapshots = true, Recursive = true }; // Get the root source and destination directories for the two containers to be copied CloudBlobDirectory sourceDirectory = await GetDirectoryAsync(sourceAccount, copyItem.SourceContainer, copyItem.SourceDirectory); CloudBlobDirectory destinationDirectory = await GetDirectoryAsync(destinationAccount, copyItem.DestinationContainer, copyItem.DestinationDirectory); // Copy the container await CopyDirectoryAsync(copyItem.JobId, sourceDirectory, destinationDirectory, copyDirectoryOptions, transferContext, transferCheckpoint, cancellationTokenSource); // Check if any files failed during transfer if (_failedFiles.Count > 0) { // Save a Checkpoint so we can restart the transfer transferCheckpoint = transferContext.LastCheckpoint; SaveTransferCheckpoint(copyItem.JobId, transferCheckpoint); // Throw an exception to fail the job so WebJobs will rerun it throw new Exception("One or more errors occurred during the transfer."); } // Log job completion await Logger.JobCompleteAsync(copyItem.JobName, progressRecorder, _skippedFiles); } catch (Exception ex) { // Log Job Error await Logger.JobErrorAsync(copyItem.JobName, ex.Message, progressRecorder, _failedFiles, _skippedFiles); // Rethrow the error to fail the web job throw ex; } }
private async static Task CopyDirectoryAsync(string jobId, CloudBlobDirectory sourceDirectory, CloudBlobDirectory destinationDirectory, CopyDirectoryOptions copyDirectoryOptions, TransferContext transferContext, TransferCheckpoint transferCheckpoint, CancellationTokenSource cancellationTokenSource) { // Start the transfer try { await TransferManager.CopyDirectoryAsync( sourceBlobDir: sourceDirectory, destBlobDir: destinationDirectory, isServiceCopy: false, options: copyDirectoryOptions, context: transferContext, cancellationToken: cancellationTokenSource.Token); // Store the transfer checkpoint to record the completed copy operation transferCheckpoint = transferContext.LastCheckpoint; } catch(TransferException) { // Swallow all transfer exceptions here. Files skipped in the OverwriteCallback throw an exception here // even in an Incremental copy where the source is skipped because it and destination are identical // Instead all exceptions from transfers are handled in the FileFailed event handler. } catch (Exception ex) { // Fatal or other exceptions resulting in the transfer being cancelled will still appear here // Save a Checkpoint so we can restart the transfer transferCheckpoint = transferContext.LastCheckpoint; SaveTransferCheckpoint(jobId, transferCheckpoint); throw new Exception("Error in CopyDirectoryAsync(): " + ex.Message); } }
private static Task CopyDirectoryInternalAsync(TransferLocation sourceLocation, TransferLocation destLocation, bool isServiceCopy, ITransferEnumerator sourceEnumerator, CopyDirectoryOptions options, TransferContext context, CancellationToken cancellationToken) { DirectoryTransfer transfer = GetOrCreateDirectoryTransfer(sourceLocation, destLocation, isServiceCopy ? TransferMethod.AsyncCopy : TransferMethod.SyncCopy, context); if (transfer.SourceEnumerator == null || !AreSameTransferEnumerators(transfer.SourceEnumerator, sourceEnumerator)) { transfer.SourceEnumerator = sourceEnumerator; } if (options != null) { transfer.BlobType = options.BlobType; } return DoTransfer(transfer, context, cancellationToken); }
/// <summary> /// Copy an Azure file directory to an Azure blob directory. /// </summary> /// <param name="sourceFileDir">The <see cref="CloudFileDirectory"/> that is the source Azure file directory.</param> /// <param name="destBlobDir">The <see cref="CloudBlobDirectory"/> that is the destination Azure blob directory.</param> /// <param name="isServiceCopy">A flag indicating whether the copy is service-side asynchronous copy or not. /// If this flag is set to true, service-side asychronous copy will be used; if this flag is set to false, /// file is downloaded from source first, then uploaded to destination.</param> /// <param name="options">A <see cref="CopyDirectoryOptions"/> object that specifies additional options for the operation.</param> /// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param> /// <param name="cancellationToken">A <see cref="CancellationToken"/> object to observe while waiting for a task to complete.</param> /// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns> public static Task CopyDirectoryAsync(CloudFileDirectory sourceFileDir, CloudBlobDirectory destBlobDir, bool isServiceCopy, CopyDirectoryOptions options, TransferContext context, CancellationToken cancellationToken) { AzureFileDirectoryLocation sourceLocation = new AzureFileDirectoryLocation(sourceFileDir); AzureBlobDirectoryLocation destLocation = new AzureBlobDirectoryLocation(destBlobDir); AzureFileEnumerator sourceEnumerator = new AzureFileEnumerator(sourceLocation); if (options != null) { TransferManager.CheckSearchPatternOfAzureFileSource(options); sourceEnumerator.SearchPattern = options.SearchPattern; sourceEnumerator.Recursive = options.Recursive; } return CopyDirectoryInternalAsync(sourceLocation, destLocation, isServiceCopy, sourceEnumerator, options, context, cancellationToken); }
/// <summary> /// Copy an Azure file directory to an Azure blob directory. /// </summary> /// <param name="sourceFileDir">The <see cref="CloudFileDirectory"/> that is the source Azure file directory.</param> /// <param name="destBlobDir">The <see cref="CloudBlobDirectory"/> that is the destination Azure blob directory.</param> /// <param name="isServiceCopy">A flag indicating whether the copy is service-side asynchronous copy or not. /// If this flag is set to true, service-side asychronous copy will be used; if this flag is set to false, /// file is downloaded from source first, then uploaded to destination.</param> /// <param name="options">A <see cref="CopyDirectoryOptions"/> object that specifies additional options for the operation.</param> /// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param> /// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns> public static Task CopyDirectoryAsync(CloudFileDirectory sourceFileDir, CloudBlobDirectory destBlobDir, bool isServiceCopy, CopyDirectoryOptions options, TransferContext context) { return CopyDirectoryAsync(sourceFileDir, destBlobDir, isServiceCopy, options, context, CancellationToken.None); }
/// <summary> /// Copy an Azure blob directory to another Azure blob directory. /// </summary> /// <param name="sourceBlobDir">The <see cref="CloudBlobDirectory"/> that is the source Azure blob directory.</param> /// <param name="destBlobDir">The <see cref="CloudBlobDirectory"/> that is the destination Azure blob directory.</param> /// <param name="isServiceCopy">A flag indicating whether the copy is service-side asynchronous copy or not. /// If this flag is set to true, service-side asychronous copy will be used; if this flag is set to false, /// file is downloaded from source first, then uploaded to destination.</param> /// <param name="options">A <see cref="CopyDirectoryOptions"/> object that specifies additional options for the operation.</param> /// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param> /// <param name="cancellationToken">A <see cref="CancellationToken"/> object to observe while waiting for a task to complete.</param> /// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns> public static Task CopyDirectoryAsync(CloudBlobDirectory sourceBlobDir, CloudBlobDirectory destBlobDir, bool isServiceCopy, CopyDirectoryOptions options, TransferContext context, CancellationToken cancellationToken) { AzureBlobDirectoryLocation sourceLocation = new AzureBlobDirectoryLocation(sourceBlobDir); AzureBlobDirectoryLocation destLocation = new AzureBlobDirectoryLocation(destBlobDir); AzureBlobEnumerator sourceEnumerator = new AzureBlobEnumerator(sourceLocation); if (options != null) { sourceEnumerator.SearchPattern = options.SearchPattern; sourceEnumerator.Recursive = options.Recursive; sourceEnumerator.IncludeSnapshots = options.IncludeSnapshots; } return CopyDirectoryInternalAsync(sourceLocation, destLocation, isServiceCopy, sourceEnumerator, options, context, cancellationToken); }