public static void JobError(string jobName, string message, ProgressRecorder progressRecorder, List<TransferDetail> failedFiles, List<TransferDetail> skippedFiles)
{
log.WriteLine("Error for Job: " + jobName);
log.WriteLine("Error: " + message);
log.WriteLine("WebJobs will make 5 attempts to rerun and complete");
log.WriteLine(progressRecorder.ToString());
if(failedFiles.Count > 0)
{
log.WriteLine("Detailed File Transfer Errors");
foreach (TransferDetail td in failedFiles)
{
log.WriteLine("Source File: " + td.Source);
log.WriteLine("Destination File: " + td.Destination);
log.WriteLine("Error Message: " + td.Error);
}
}
if (skippedFiles.Count > 0)
{
log.WriteLine("Skipped File Details");
foreach (TransferDetail td in skippedFiles)
{
log.WriteLine("Source File: " + td.Source);
log.WriteLine("Destination File: " + td.Destination);
log.WriteLine("Error Message: " + td.Error);
}
}
}
private static void CopyContainer(CloudBlobDirectory sourceDirectory, CloudBlobDirectory destDirectory)
{
// This function copies an entire Storage container using the data movement library.
// See https://docs.microsoft.com/en-us/dotnet/api/microsoft.azure.storage.datamovement.copydirectoryoptions?view=azure-dotnet for the API
// and https://docs.microsoft.com/en-us/azure/storage/common/storage-use-data-movement-library for a general overview.
ProgressRecorder recorder = new ProgressRecorder();
Stopwatch stopWatch = Stopwatch.StartNew();
CopyDirectoryOptions copyDirectoryOptions = new CopyDirectoryOptions
{
// Needs to be true to copy the contents of the directory / container. If this is false the contents will
// not be copied.
Recursive = true
};
// Copy all files under root folder
DirectoryTransferContext context = new DirectoryTransferContext
{
ProgressHandler = recorder
};
TransferStatus copyStatus = TransferManager.CopyDirectoryAsync(sourceDirectory, destDirectory, CopyMethod.ServiceSideAsyncCopy, copyDirectoryOptions, context).Result;
stopWatch.Stop();
// Get the elapsed time as a TimeSpan value.
TimeSpan ts = stopWatch.Elapsed;
// Format and display the TimeSpan value.
string elapsedTime = string.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds,
ts.Milliseconds / 10);
Console.WriteLine($"Elapsed:{elapsedTime} T File transferred: {copyStatus.NumberOfFilesTransferred} total bytes：{copyStatus.BytesTransferred}, failed: {copyStatus.NumberOfFilesFailed}, skipped:{copyStatus.NumberOfFilesSkipped}");
}
/// <summary>
/// Download data from Azure storage.
/// 1. Download a CloudBlob to a Stream instance
/// 2. Download the same CloudBlob with step #1 to a different Stream instance
/// 3. Download another CloudBlob to a different stream with content MD5 validation disabled
/// 4. Show the overall progress of all transfers
/// </summary>
private static async Task BlobDownloadToStreamSample()
{
string sourceBlobName1 = "azure_blockblob.png";
string sourceBlobName2 = "azure_blockblob2.png";
// Create the source CloudBlob instances
CloudBlob sourceBlob1 = await Util.GetCloudBlobAsync(ContainerName, sourceBlobName1, BlobType.BlockBlob);
CloudBlob sourceBlob2 = await Util.GetCloudBlobAsync(ContainerName, sourceBlobName2, BlobType.BlockBlob);
// Create a TransferContext shared by both transfers
SingleTransferContext sharedTransferContext = new SingleTransferContext();
// Record the overall progress
ProgressRecorder recorder = new ProgressRecorder();
sharedTransferContext.ProgressHandler = recorder;
MemoryStream memoryStream1_1 = new MemoryStream();
MemoryStream memoryStream1_2 = new MemoryStream();
MemoryStream memoryStream2 = new MemoryStream();
try
{
// Start the blob download
Task task1 = TransferManager.DownloadAsync(sourceBlob1, memoryStream1_1, null /* options */, sharedTransferContext);
// Start to download the same blob to another Stream
// Please note, DataMovement Library will download blob once for each of downloads.
// For example, if you start two downloads from the same source blob to two different Stream instance,
// DataMovement Library will download the blob content twice.
Task task2 = TransferManager.DownloadAsync(sourceBlob1, memoryStream1_2, null /* options */, sharedTransferContext);
// Create a DownloadOptions to disable md5 check after data is downloaded. Otherwise, data movement
// library will check the md5 checksum stored in the ContentMD5 property of the source CloudFile/CloudBlob
// You can uncomment following codes, enable ContentMD5Validation and have a try.
// sourceBlob2.Properties.ContentMD5 = "WrongMD5";
// sourceBlob2.SetProperties();
DownloadOptions options = new DownloadOptions();
options.DisableContentMD5Validation = true;
// Start the download
Task task3 = TransferManager.DownloadAsync(sourceBlob2, memoryStream2, options, sharedTransferContext);
// Wait for all transfers to finish
await task1;
await task2;
await task3;
// Print out the final transfer state
Console.WriteLine("Final transfer state: {0}", recorder.ToString());
}
finally
{
memoryStream1_1.Dispose();
memoryStream1_2.Dispose();
memoryStream2.Dispose();
}
}
public virtual void before_each()
{
autoSubstitute = new AutoSubstitute();
testExecutionRecorder = autoSubstitute.Resolve <ITestExecutionRecorder>();
recorder = autoSubstitute.Resolve <ProgressRecorder>();
}
public static async Task JobCompleteAsync(string jobName, ProgressRecorder progressRecorder, List<TransferDetail> skippedFiles)
{
await log.WriteLineAsync("Job Complete: " + jobName);
await log.WriteLineAsync(progressRecorder.ToString());
if (skippedFiles.Count > 0)
{
log.WriteLine("Skipped File Details");
foreach (TransferDetail td in skippedFiles)
{
await log.WriteLineAsync("Source File: " + td.Source);
await log.WriteLineAsync("Destination File: " + td.Destination);
await log.WriteLineAsync("Error Message: " + td.Error);
}
}
}
public async Task UploadFileToBlobAsync(string sourceFilePath, BlobOptions options)
{
Validate.BlobContainerName(options.ContainerName, "containerName");
Validate.BlobName(options.BlobName, "blobName");
var destinationBlob = DataManagerUtility.GetCloudBlob(options);
TransferManager.Configurations.ParallelOperations = 64;
var context = new TransferContext
{
OverwriteCallback = (path, destinationPath) => options.OverwriteDestination
};
ProgressRecorder = new ProgressRecorder();
context.ProgressHandler = ProgressRecorder;
await TransferManager.UploadAsync(sourceFilePath, destinationBlob, null, context, CancellationToken.None);
}
/// <summary>
/// Copy data between Azure storage.
/// 1. Copy a CloudBlobDirectory
/// 2. Cancel the transfer before it finishes with a CancellationToken
/// 3. Store the transfer checkpoint into a file after transfer being cancelled
/// 4. Reload checkpoint from the file
/// 4. Resume the transfer with the loaded checkpoint
/// </summary>
private static async Task BlobDirectoryCopySample()
{
CloudBlobDirectory sourceBlobDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir");
CloudBlobDirectory destBlobDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir2");
// When source is CloudBlobDirectory:
// 1. If recursive is set to true, data movement library matches the source blob name against SearchPattern as prefix.
// 2. Otherwise, data movement library matches the blob with the exact name specified by SearchPattern.
//
// You can also replace the source directory with a CloudFileDirectory instance to copy data from Azure File Storage. If so:
// 1. If recursive is set to true, SearchPattern is not supported. Data movement library simply transfer all azure files
// under the source CloudFileDirectory and its sub-directories.
// 2. Otherwise, data movement library matches the azure file with the exact name specified by SearchPattern.
//
// In the following case, data movement library will copy all blobs with the prefix "azure" in source blob directory.
CopyDirectoryOptions options = new CopyDirectoryOptions()
{
SearchPattern = "azure",
Recursive = true,
};
DirectoryTransferContext context = new DirectoryTransferContext();
context.FileTransferred += FileTransferredCallback;
context.FileFailed += FileFailedCallback;
context.FileSkipped += FileSkippedCallback;
// Create CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationSource = new CancellationTokenSource();
TransferCheckpoint checkpoint = null;
TransferStatus trasferStatus = null;
try
{
Task <TransferStatus> task = TransferManager.CopyDirectoryAsync(sourceBlobDir, destBlobDir, false /* isServiceCopy */, options, context, cancellationSource.Token);
// Sleep for 1 seconds and cancel the transfer.
// It may fail to cancel the transfer if transfer is done in 1 second. If so, no file will be copied after resume.
Thread.Sleep(1000);
Console.WriteLine("Cancel the transfer.");
cancellationSource.Cancel();
trasferStatus = await task;
}
catch (Exception e)
{
Console.WriteLine("The transfer is cancelled: {0}", e.Message);
}
// Store the transfer checkpoint
checkpoint = context.LastCheckpoint;
// Serialize the checkpoint into a file
#if DOTNET5_4
var formatter = new DataContractSerializer(typeof(TransferCheckpoint));
#else
IFormatter formatter = new BinaryFormatter();
#endif
string tempFileName = Guid.NewGuid().ToString();
using (var stream = new FileStream(tempFileName, FileMode.Create, FileAccess.Write, FileShare.None))
{
formatter.Serialize(stream, checkpoint);
}
// Deserialize the checkpoint from the file
using (var stream = new FileStream(tempFileName, FileMode.Open, FileAccess.Read, FileShare.None))
{
checkpoint = formatter.Deserialize(stream) as TransferCheckpoint;
}
File.Delete(tempFileName);
// Create a new TransferContext with the store checkpoint
DirectoryTransferContext resumeContext = new DirectoryTransferContext(checkpoint);
resumeContext.FileTransferred += FileTransferredCallback;
resumeContext.FileFailed += FileFailedCallback;
resumeContext.FileSkipped += FileSkippedCallback;
// Record the overall progress
ProgressRecorder recorder = new ProgressRecorder();
resumeContext.ProgressHandler = recorder;
// Resume transfer from the stored checkpoint
Console.WriteLine("Resume the cancelled transfer.");
trasferStatus = await TransferManager.CopyDirectoryAsync(sourceBlobDir, destBlobDir, false /* isServiceCopy */, options, resumeContext);
// Print out the final transfer state
Console.WriteLine("Final transfer state: {0}", TransferStatusToString(trasferStatus));
}
/// <summary>
/// Download data from Azure storage.
/// 1. Download a CloudBlob to an exsiting local file
/// 2. Query the user to overwrite the local file or not in the OverwriteCallback
/// 3. Download another CloudBlob to local with content MD5 validation disabled
/// 4. Show the overall progress of both transfers
/// </summary>
private static async Task BlobDownloadSample()
{
string sourceBlobName1 = "azure_blockblob.png";
string sourceBlobName2 = "azure_blockblob2.png";
string destinationFileName1 = "azure.png";
string destinationFileName2 = "azure_new.png";
// Create the source CloudBlob instances
CloudBlob sourceBlob1 = await Util.GetCloudBlobAsync(ContainerName, sourceBlobName1, BlobType.BlockBlob);
CloudBlob sourceBlob2 = await Util.GetCloudBlobAsync(ContainerName, sourceBlobName2, BlobType.BlockBlob);
// Create a TransferContext shared by both transfers
SingleTransferContext sharedTransferContext = new SingleTransferContext();
// Show overwrite prompt in console when OverwriteCallback is triggered
sharedTransferContext.ShouldOverwriteCallbackAsync = async(source, destination) =>
{
Console.WriteLine("{0} already exists. Do you want to overwrite it with {1}? (Y/N)", destination, source);
while (true)
{
ConsoleKeyInfo keyInfo = Console.ReadKey(true);
char key = keyInfo.KeyChar;
if (key == 'y' || key == 'Y')
{
Console.WriteLine("User choose to overwrite the destination.");
return(true);
}
else if (key == 'n' || key == 'N')
{
Console.WriteLine("User choose NOT to overwrite the destination.");
return(false);
}
Console.WriteLine("Please press 'y' or 'n'.");
}
};
// Record the overall progress
ProgressRecorder recorder = new ProgressRecorder();
sharedTransferContext.ProgressHandler = recorder;
// Start the blob download
Task task1 = TransferManager.DownloadAsync(sourceBlob1, destinationFileName1, null /* options */, sharedTransferContext);
// Create a DownloadOptions to disable md5 check after data is downloaded. Otherwise, data movement
// library will check the md5 checksum stored in the ContentMD5 property of the source CloudFile/CloudBlob
// You can uncomment following codes, enable ContentMD5Validation and have a try.
// sourceBlob2.Properties.ContentMD5 = "WrongMD5";
// sourceBlob2.SetProperties();
DownloadOptions options = new DownloadOptions();
options.DisableContentMD5Validation = true;
// Start the download
Task task2 = TransferManager.DownloadAsync(sourceBlob2, destinationFileName2, options, sharedTransferContext);
// Wait for both transfers to finish
try
{
await task1;
}
catch (Exception e)
{
// Data movement library will throw a TransferException when user choose to not overwrite the existing destination
Console.WriteLine(e.Message);
}
await task2;
// Print out the final transfer state
Console.WriteLine("Final transfer state: {0}", recorder.ToString());
}
public async static Task ProcessMessage([QueueTrigger("backupqueue")] CopyItem copyItem, TextWriter log, CancellationToken cancelToken)
{
// Copy TextWrite into Log Helper class
Logger.log = log;
// Log Job Start
await Logger.JobStartAsync(copyItem.JobName);
// This class accumulates transfer data during the copy
ProgressRecorder progressRecorder = new ProgressRecorder();
try
{
// OpContext to track PreCopy Retries on Azure Storage
// DML has its own context object and retry
_opContext = new OperationContext();
_opContext.Retrying += StorageRequest_Retrying;
// Define Blob Request Options
_blobRequestOptions = new BlobRequestOptions
{
// Defined Exponential Retry Policy above
RetryPolicy = _retryPolicy
};
// Set the number of parallel tasks in DML.
// This allows it to copy multiple items at once when copying a container or directory
// The best (and default value) is Environment.ProcessorCount * 8
int parallelTasks = Environment.ProcessorCount * 8;
TransferManager.Configurations.ParallelOperations = parallelTasks;
// Set the number of connections.
// This should match ParallelOperations so each DML copy task has its own connection to Azure Storage
ServicePointManager.DefaultConnectionLimit = parallelTasks;
// Short circuit additional request round trips. We are not chunking and
// uploading large amounts of data where we'd send 100's so set to false
ServicePointManager.Expect100Continue = false;
// User Agent for tracing
TransferManager.Configurations.UserAgentPrefix = "AzureDmlBackup";
// CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
// Open connections to both storage accounts
CloudStorageAccount sourceAccount = GetAccount(copyItem.SourceAccountToken);
CloudStorageAccount destinationAccount = GetAccount(copyItem.DestinationAccountToken);
// Represents a checkpoint from which a transfer may be resumed and continued.
// This is initalized as null first time then hydrated within CopyDirectoryAsync().
// However if this job is being resumed from a previous failure this function will hydrate
// from a serialized checkpoint saved to blob storage.
TransferCheckpoint transferCheckpoint = await GetTransferCheckpoint(copyItem.JobId);
// Context object for the transfer, provides additional runtime information about its execution
// If this is a resumed copy operation then pass the checkpoint to the TransferContext so it can resume the copy
TransferContext transferContext = new TransferContext(transferCheckpoint)
{
// Pipe transfer progress data to ProgressRecorder
// ProgressRecorder is used to log the results of the copy operation
ProgressHandler = progressRecorder,
// If the destination already exists this delegate is called.
// Return true to overwrite or false to skip the file during the transfer
OverwriteCallback = (source, destination) =>
{
return OverwriteFile(source, destination, sourceAccount, destinationAccount, copyItem.IsIncremental);
}
};
// This event is used to log files skipped during the transfer
transferContext.FileSkipped += TransferContext_FileSkipped;
// This event is used to catch exceptions for files that fail during a transfer
transferContext.FileFailed += TransferContext_FileFailed;
// Set Options for copying the container such as search patterns, recursive, etc.
CopyDirectoryOptions copyDirectoryOptions = new CopyDirectoryOptions
{
IncludeSnapshots = true,
Recursive = true
};
// Get the root source and destination directories for the two containers to be copied
CloudBlobDirectory sourceDirectory = await GetDirectoryAsync(sourceAccount, copyItem.SourceContainer, copyItem.SourceDirectory);
CloudBlobDirectory destinationDirectory = await GetDirectoryAsync(destinationAccount, copyItem.DestinationContainer, copyItem.DestinationDirectory);
// Copy the container
await CopyDirectoryAsync(copyItem.JobId, sourceDirectory, destinationDirectory, copyDirectoryOptions, transferContext, transferCheckpoint, cancellationTokenSource);
// Check if any files failed during transfer
if (_failedFiles.Count > 0)
{
// Save a Checkpoint so we can restart the transfer
transferCheckpoint = transferContext.LastCheckpoint;
SaveTransferCheckpoint(copyItem.JobId, transferCheckpoint);
// Throw an exception to fail the job so WebJobs will rerun it
throw new Exception("One or more errors occurred during the transfer.");
}
// Log job completion
await Logger.JobCompleteAsync(copyItem.JobName, progressRecorder, _skippedFiles);
}
catch (Exception ex)
{
// Log Job Error
await Logger.JobErrorAsync(copyItem.JobName, ex.Message, progressRecorder, _failedFiles, _skippedFiles);
// Rethrow the error to fail the web job
throw ex;
}
}
