public async static Task ProcessMessage([QueueTrigger("backupqueue")] CopyItem copyItem, TextWriter log, CancellationToken cancelToken)
{
// Copy TextWrite into Log Helper class
Logger.log = log;
// Log Job Start
await Logger.JobStartAsync(copyItem.JobName);
// This class accumulates transfer data during the copy
ProgressRecorder progressRecorder = new ProgressRecorder();
try
{
// OpContext to track PreCopy Retries on Azure Storage
// DML has its own context object and retry
_opContext = new OperationContext();
_opContext.Retrying += StorageRequest_Retrying;
// Define Blob Request Options
_blobRequestOptions = new BlobRequestOptions
{
// Defined Exponential Retry Policy above
RetryPolicy = _retryPolicy
};
// Set the number of parallel tasks in DML.
// This allows it to copy multiple items at once when copying a container or directory
// The best (and default value) is Environment.ProcessorCount * 8
int parallelTasks = Environment.ProcessorCount * 8;
TransferManager.Configurations.ParallelOperations = parallelTasks;
// Set the number of connections.
// This should match ParallelOperations so each DML copy task has its own connection to Azure Storage
ServicePointManager.DefaultConnectionLimit = parallelTasks;
// Short circuit additional request round trips. We are not chunking and
// uploading large amounts of data where we'd send 100's so set to false
ServicePointManager.Expect100Continue = false;
// User Agent for tracing
TransferManager.Configurations.UserAgentPrefix = "AzureDmlBackup";
// CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
// Open connections to both storage accounts
CloudStorageAccount sourceAccount = GetAccount(copyItem.SourceAccountToken);
CloudStorageAccount destinationAccount = GetAccount(copyItem.DestinationAccountToken);
// Represents a checkpoint from which a transfer may be resumed and continued.
// This is initalized as null first time then hydrated within CopyDirectoryAsync().
// However if this job is being resumed from a previous failure this function will hydrate
// from a serialized checkpoint saved to blob storage.
TransferCheckpoint transferCheckpoint = await GetTransferCheckpoint(copyItem.JobId);
// Context object for the transfer, provides additional runtime information about its execution
// If this is a resumed copy operation then pass the checkpoint to the TransferContext so it can resume the copy
TransferContext transferContext = new TransferContext(transferCheckpoint)
{
// Pipe transfer progress data to ProgressRecorder
// ProgressRecorder is used to log the results of the copy operation
ProgressHandler = progressRecorder,
// If the destination already exists this delegate is called.
// Return true to overwrite or false to skip the file during the transfer
OverwriteCallback = (source, destination) =>
{
return OverwriteFile(source, destination, sourceAccount, destinationAccount, copyItem.IsIncremental);
}
};
// This event is used to log files skipped during the transfer
transferContext.FileSkipped += TransferContext_FileSkipped;
// This event is used to catch exceptions for files that fail during a transfer
transferContext.FileFailed += TransferContext_FileFailed;
// Set Options for copying the container such as search patterns, recursive, etc.
CopyDirectoryOptions copyDirectoryOptions = new CopyDirectoryOptions
{
IncludeSnapshots = true,
Recursive = true
};
// Get the root source and destination directories for the two containers to be copied
CloudBlobDirectory sourceDirectory = await GetDirectoryAsync(sourceAccount, copyItem.SourceContainer, copyItem.SourceDirectory);
CloudBlobDirectory destinationDirectory = await GetDirectoryAsync(destinationAccount, copyItem.DestinationContainer, copyItem.DestinationDirectory);
// Copy the container
await CopyDirectoryAsync(copyItem.JobId, sourceDirectory, destinationDirectory, copyDirectoryOptions, transferContext, transferCheckpoint, cancellationTokenSource);
// Check if any files failed during transfer
if (_failedFiles.Count > 0)
{
// Save a Checkpoint so we can restart the transfer
transferCheckpoint = transferContext.LastCheckpoint;
SaveTransferCheckpoint(copyItem.JobId, transferCheckpoint);
// Throw an exception to fail the job so WebJobs will rerun it
throw new Exception("One or more errors occurred during the transfer.");
}
// Log job completion
await Logger.JobCompleteAsync(copyItem.JobName, progressRecorder, _skippedFiles);
}
catch (Exception ex)
{
// Log Job Error
await Logger.JobErrorAsync(copyItem.JobName, ex.Message, progressRecorder, _failedFiles, _skippedFiles);
// Rethrow the error to fail the web job
throw ex;
}
}
private async static Task CopyDirectoryAsync(string jobId, CloudBlobDirectory sourceDirectory, CloudBlobDirectory destinationDirectory, CopyDirectoryOptions copyDirectoryOptions, TransferContext transferContext, TransferCheckpoint transferCheckpoint, CancellationTokenSource cancellationTokenSource)
{
// Start the transfer
try
{
await TransferManager.CopyDirectoryAsync(
sourceBlobDir: sourceDirectory,
destBlobDir: destinationDirectory,
isServiceCopy: false,
options: copyDirectoryOptions,
context: transferContext,
cancellationToken: cancellationTokenSource.Token);
// Store the transfer checkpoint to record the completed copy operation
transferCheckpoint = transferContext.LastCheckpoint;
}
catch(TransferException)
{
// Swallow all transfer exceptions here. Files skipped in the OverwriteCallback throw an exception here
// even in an Incremental copy where the source is skipped because it and destination are identical
// Instead all exceptions from transfers are handled in the FileFailed event handler.
}
catch (Exception ex)
{
// Fatal or other exceptions resulting in the transfer being cancelled will still appear here
// Save a Checkpoint so we can restart the transfer
transferCheckpoint = transferContext.LastCheckpoint;
SaveTransferCheckpoint(jobId, transferCheckpoint);
throw new Exception("Error in CopyDirectoryAsync(): " + ex.Message);
}
}
private static Task CopyDirectoryInternalAsync(TransferLocation sourceLocation, TransferLocation destLocation, bool isServiceCopy, ITransferEnumerator sourceEnumerator, CopyDirectoryOptions options, TransferContext context, CancellationToken cancellationToken)
{
DirectoryTransfer transfer = GetOrCreateDirectoryTransfer(sourceLocation, destLocation, isServiceCopy ? TransferMethod.AsyncCopy : TransferMethod.SyncCopy, context);
if (transfer.SourceEnumerator == null || !AreSameTransferEnumerators(transfer.SourceEnumerator, sourceEnumerator))
{
transfer.SourceEnumerator = sourceEnumerator;
}
if (options != null)
{
transfer.BlobType = options.BlobType;
}
return DoTransfer(transfer, context, cancellationToken);
}
/// <summary>
/// Copy an Azure file directory to an Azure blob directory.
/// </summary>
/// <param name="sourceFileDir">The <see cref="CloudFileDirectory"/> that is the source Azure file directory.</param>
/// <param name="destBlobDir">The <see cref="CloudBlobDirectory"/> that is the destination Azure blob directory.</param>
/// <param name="isServiceCopy">A flag indicating whether the copy is service-side asynchronous copy or not.
/// If this flag is set to true, service-side asychronous copy will be used; if this flag is set to false,
/// file is downloaded from source first, then uploaded to destination.</param>
/// <param name="options">A <see cref="CopyDirectoryOptions"/> object that specifies additional options for the operation.</param>
/// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> object to observe while waiting for a task to complete.</param>
/// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns>
public static Task CopyDirectoryAsync(CloudFileDirectory sourceFileDir, CloudBlobDirectory destBlobDir, bool isServiceCopy, CopyDirectoryOptions options, TransferContext context, CancellationToken cancellationToken)
{
AzureFileDirectoryLocation sourceLocation = new AzureFileDirectoryLocation(sourceFileDir);
AzureBlobDirectoryLocation destLocation = new AzureBlobDirectoryLocation(destBlobDir);
AzureFileEnumerator sourceEnumerator = new AzureFileEnumerator(sourceLocation);
if (options != null)
{
TransferManager.CheckSearchPatternOfAzureFileSource(options);
sourceEnumerator.SearchPattern = options.SearchPattern;
sourceEnumerator.Recursive = options.Recursive;
}
return CopyDirectoryInternalAsync(sourceLocation, destLocation, isServiceCopy, sourceEnumerator, options, context, cancellationToken);
}
/// <summary>
/// Copy an Azure file directory to an Azure blob directory.
/// </summary>
/// <param name="sourceFileDir">The <see cref="CloudFileDirectory"/> that is the source Azure file directory.</param>
/// <param name="destBlobDir">The <see cref="CloudBlobDirectory"/> that is the destination Azure blob directory.</param>
/// <param name="isServiceCopy">A flag indicating whether the copy is service-side asynchronous copy or not.
/// If this flag is set to true, service-side asychronous copy will be used; if this flag is set to false,
/// file is downloaded from source first, then uploaded to destination.</param>
/// <param name="options">A <see cref="CopyDirectoryOptions"/> object that specifies additional options for the operation.</param>
/// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param>
/// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns>
public static Task CopyDirectoryAsync(CloudFileDirectory sourceFileDir, CloudBlobDirectory destBlobDir, bool isServiceCopy, CopyDirectoryOptions options, TransferContext context)
{
return CopyDirectoryAsync(sourceFileDir, destBlobDir, isServiceCopy, options, context, CancellationToken.None);
}
/// <summary>
/// Copy an Azure blob directory to another Azure blob directory.
/// </summary>
/// <param name="sourceBlobDir">The <see cref="CloudBlobDirectory"/> that is the source Azure blob directory.</param>
/// <param name="destBlobDir">The <see cref="CloudBlobDirectory"/> that is the destination Azure blob directory.</param>
/// <param name="isServiceCopy">A flag indicating whether the copy is service-side asynchronous copy or not.
/// If this flag is set to true, service-side asychronous copy will be used; if this flag is set to false,
/// file is downloaded from source first, then uploaded to destination.</param>
/// <param name="options">A <see cref="CopyDirectoryOptions"/> object that specifies additional options for the operation.</param>
/// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> object to observe while waiting for a task to complete.</param>
/// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns>
public static Task CopyDirectoryAsync(CloudBlobDirectory sourceBlobDir, CloudBlobDirectory destBlobDir, bool isServiceCopy, CopyDirectoryOptions options, TransferContext context, CancellationToken cancellationToken)
{
AzureBlobDirectoryLocation sourceLocation = new AzureBlobDirectoryLocation(sourceBlobDir);
AzureBlobDirectoryLocation destLocation = new AzureBlobDirectoryLocation(destBlobDir);
AzureBlobEnumerator sourceEnumerator = new AzureBlobEnumerator(sourceLocation);
if (options != null)
{
sourceEnumerator.SearchPattern = options.SearchPattern;
sourceEnumerator.Recursive = options.Recursive;
sourceEnumerator.IncludeSnapshots = options.IncludeSnapshots;
}
return CopyDirectoryInternalAsync(sourceLocation, destLocation, isServiceCopy, sourceEnumerator, options, context, cancellationToken);
}
