protected TransferContext GetTransferContext(ProgressRecord record, long totalTransferLength)
{
TransferContext transferContext = new TransferContext();
transferContext.ClientRequestId = CmdletOperationContext.ClientRequestId;
transferContext.OverwriteCallback = ConfirmOverwrite;
transferContext.ProgressHandler = new TransferProgressHandler((transferProgress) =>
{
if (record != null)
{
record.PercentComplete = (int)(transferProgress.BytesTransferred * 100 / totalTransferLength);
record.StatusDescription = string.Format(CultureInfo.CurrentCulture, Resources.FileTransmitStatus, record.PercentComplete);
this.OutputStream.WriteProgress(record);
}
});
return transferContext;
}
public async Task UploadFileToBlobAsync(string sourceFilePath, BlobOptions options)
{
Validate.BlobContainerName(options.ContainerName, "containerName");
Validate.BlobName(options.BlobName, "blobName");
var destinationBlob = DataManagerUtility.GetCloudBlob(options);
TransferManager.Configurations.ParallelOperations = 64;
var context = new TransferContext
{
OverwriteCallback = (path, destinationPath) => options.OverwriteDestination
};
ProgressRecorder = new ProgressRecorder();
context.ProgressHandler = ProgressRecorder;
await TransferManager.UploadAsync(sourceFilePath, destinationBlob, null, context, CancellationToken.None);
}
protected TransferContext GetTransferContext(DataMovementUserData userData)
{
TransferContext transferContext = new TransferContext();
transferContext.ClientRequestId = CmdletOperationContext.ClientRequestId;
transferContext.OverwriteCallback = ConfirmOverwrite;
transferContext.ProgressHandler = new TransferProgressHandler((transferProgress) =>
{
if (userData.Record != null)
{
// Size of the source file might be 0, when it is, directly treat the progress as 100 percent.
userData.Record.PercentComplete = 0 == userData.TotalSize ? 100 : (int)(transferProgress.BytesTransferred * 100 / userData.TotalSize);
userData.Record.StatusDescription = string.Format(CultureInfo.CurrentCulture, Resources.FileTransmitStatus, userData.Record.PercentComplete);
this.OutputStream.WriteProgress(userData.Record);
}
});
return transferContext;
}
/// <summary>
/// Generate an OperationContext from the the specified TransferContext.
/// </summary>
/// <param name="transferContext">Transfer context</param>
/// <returns>An <see cref="OperationContext"/> object.</returns>
public static OperationContext GenerateOperationContext(
TransferContext transferContext)
{
if (transferContext == null)
{
return(null);
}
OperationContext operationContext = new OperationContext()
{
LogLevel = transferContext.LogLevel
};
if (transferContext.ClientRequestId != null)
{
operationContext.ClientRequestID = transferContext.ClientRequestId;
}
return(operationContext);
}
public void Apply(TransferContext context)
{
context.FileTransferred += (sender, transferEventArgs) =>
{
Test.Info("Transfer succeeds: {0} -> {1}", transferEventArgs.Source, transferEventArgs.Destination);
this.Increase(TransferEventType.Transferred);
};
context.FileSkipped += (sender, transferEventArgs) =>
{
Test.Info("Transfer skips: {0} -> {1}", transferEventArgs.Source, transferEventArgs.Destination);
this.Increase(TransferEventType.Skippied);
};
context.FileFailed += (sender, transferEventArgs) =>
{
Test.Info("Transfer fails: {0} -> {1}", transferEventArgs.Source, transferEventArgs.Destination);
Test.Info("Exception: {0}", transferEventArgs.Exception.ToString());
this.Increase(TransferEventType.Failed);
};
}
public void TestProgressHandlerTest()
{
long fileSize = 10 * 1024 * 1024;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSize);
var options = new TestExecutionOptions<DMLibDataInfo>();
options.TransferItemModifier = (fileNode, transferItem) =>
{
TransferContext transferContext = new TransferContext();
ProgressChecker progressChecker = new ProgressChecker(1, fileNode.SizeInByte);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
transferItem.TransferContext = transferContext;
};
var result = this.ExecuteTestCase(sourceDataInfo, options);
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
}
/// <summary>
/// Generate an OperationContext from the the specified TransferContext.
/// </summary>
/// <param name="transferContext">Transfer context</param>
/// <returns>An <see cref="OperationContext"/> object.</returns>
public static OperationContext GenerateOperationContext(
TransferContext transferContext)
{
OperationContext operationContext = new OperationContext()
{
CustomUserAgent = string.Format(CultureInfo.InvariantCulture, "{0} {1}", TransferManager.Configurations.UserAgentPrefix, Constants.UserAgent)
};
if (transferContext == null)
{
return(operationContext);
}
operationContext.LogLevel = transferContext.LogLevel;
if (transferContext.ClientRequestId != null)
{
operationContext.ClientRequestID = transferContext.ClientRequestId;
}
return(operationContext);
}
public virtual Task DownloadAsync(CloudBlob sourceBlob, string destPath, DownloadOptions options, TransferContext context, CancellationToken cancellationToken)
{
throw new NotImplementedException();
}
public virtual Task UploadAsync(string sourcePath, CloudBlob destBlob, UploadOptions options, TransferContext context, CancellationToken cancellationToken)
{
throw new NotImplementedException();
}
/// <summary>
/// Upload a file to Azure Blob Storage.
/// </summary>
/// <param name="sourcePath">Path to the source file.</param>
/// <param name="destBlob">The <see cref="CloudBlob"/> that is the destination Azure blob.</param>
/// <param name="options">An <see cref="UploadOptions"/> object that specifies additional options for the operation.</param>
/// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param>
/// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> object to observe while waiting for a task to complete.</param>
public static Task UploadAsync(string sourcePath, CloudBlob destBlob, UploadOptions options, TransferContext context, CancellationToken cancellationToken)
{
TransferLocation sourceLocation = new TransferLocation(sourcePath);
TransferLocation destLocation = new TransferLocation(destBlob);
return UploadInternalAsync(sourceLocation, destLocation, options, context, cancellationToken);
}
/// <summary>
/// Download an Azure blob from Azure Blob Storage.
/// </summary>
/// <param name="sourceBlob">The <see cref="CloudBlob"/> that is the source Azure blob.</param>
/// <param name="destPath">Path to the destination file.</param>
/// <param name="options">A <see cref="DownloadOptions"/> object that specifies additional options for the operation.</param>
/// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param>
/// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns>
public static Task DownloadAsync(CloudBlob sourceBlob, string destPath, DownloadOptions options, TransferContext context)
{
return DownloadAsync(sourceBlob, destPath, options, context, CancellationToken.None);
}
/// <summary>
/// Upload a file to Azure File Storage.
/// </summary>
/// <param name="sourceStream">A <see cref="System.IO.Stream"/> object providing the file content.</param>
/// <param name="destFile">The <see cref="CloudFile"/> that is the destination Azure file.</param>
/// <param name="options">An <see cref="UploadOptions"/> object that specifies additional options for the operation.</param>
/// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param>
/// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns>
public static Task UploadAsync(Stream sourceStream, CloudFile destFile, UploadOptions options, TransferContext context)
{
return UploadAsync(sourceStream, destFile, options, context, CancellationToken.None);
}
/// <summary>
/// Download an Azure file from Azure File Storage.
/// </summary>
/// <param name="sourceFile">The <see cref="CloudFile"/> that is the source Azure file.</param>
/// <param name="destStream">A <see cref="System.IO.Stream"/> object representing the destination stream.</param>
/// <param name="options">A <see cref="DownloadOptions"/> object that specifies additional options for the operation.</param>
/// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param>
/// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns>
public static Task DownloadAsync(CloudFile sourceFile, Stream destStream, DownloadOptions options, TransferContext context)
{
return DownloadAsync(sourceFile, destStream, options, context, CancellationToken.None);
}
/// <summary>
/// Upload a file to Azure File Storage.
/// </summary>
/// <param name="sourceStream">A <see cref="System.IO.Stream"/> object providing the file content.</param>
/// <param name="destFile">The <see cref="CloudFile"/> that is the destination Azure file.</param>
/// <param name="options">An <see cref="UploadOptions"/> object that specifies additional options for the operation.</param>
/// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> object to observe while waiting for a task to complete.</param>
/// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns>
public static Task UploadAsync(Stream sourceStream, CloudFile destFile, UploadOptions options, TransferContext context, CancellationToken cancellationToken)
{
TransferLocation sourceLocation = new TransferLocation(sourceStream);
TransferLocation destLocation = new TransferLocation(destFile);
return UploadInternalAsync(sourceLocation, destLocation, options, context, cancellationToken);
}
/// <summary>
/// Copy content, properties and metadata of an Azure file to an Azure blob.
/// </summary>
/// <param name="sourceFile">The <see cref="CloudFile"/> that is the source Azure file.</param>
/// <param name="destBlob">The <see cref="CloudBlob"/> that is the destination Azure blob.</param>
/// <param name="isServiceCopy">A flag indicating whether the copy is service-side asynchronous copy or not.
/// If this flag is set to true, service-side asychronous copy will be used; if this flag is set to false,
/// file is downloaded from source first, then uploaded to destination.</param>
/// <param name="options">A <see cref="CopyOptions"/> object that specifies additional options for the operation.</param>
/// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param>
/// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns>
public static Task CopyAsync(CloudFile sourceFile, CloudBlob destBlob, bool isServiceCopy, CopyOptions options, TransferContext context)
{
return CopyAsync(sourceFile, destBlob, isServiceCopy, options, context, CancellationToken.None);
}
private static Task UploadInternalAsync(TransferLocation sourceLocation, TransferLocation destLocation, UploadOptions options, TransferContext context, CancellationToken cancellationToken)
{
if (options != null)
{
destLocation.AccessCondition = options.DestinationAccessCondition;
}
Transfer transfer = CreateSingleObjectTransfer(sourceLocation, destLocation, TransferMethod.SyncCopy, context);
if (options != null)
{
transfer.ContentType = options.ContentType;
}
return DoTransfer(transfer, cancellationToken);
}
private static Task DownloadInternalAsync(TransferLocation sourceLocation, TransferLocation destLocation, DownloadOptions options, TransferContext context, CancellationToken cancellationToken)
{
if (options != null)
{
sourceLocation.AccessCondition = options.SourceAccessCondition;
}
Transfer transfer = CreateSingleObjectTransfer(sourceLocation, destLocation, TransferMethod.SyncCopy, context);
return DoTransfer(transfer, cancellationToken);
}
/// <summary>
/// Copy file from an specified URI to an Azure file.
/// </summary>
/// <param name="sourceUri">The <see cref="System.Uri"/> of the source file.</param>
/// <param name="destFile">The <see cref="CloudFile"/> that is the destination Azure file.</param>
/// <param name="isServiceCopy">A flag indicating whether the copy is service-side asynchronous copy or not.
/// If this flag is set to true, service-side asychronous copy will be used; if this flag is set to false,
/// file is downloaded from source first, then uploaded to destination.</param>
/// <param name="options">A <see cref="CopyOptions"/> object that specifies additional options for the operation.</param>
/// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> object to observe while waiting for a task to complete.</param>
/// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns>
/// <remarks>Copying from an URI to Azure file synchronously is not supported yet.</remarks>
public static Task CopyAsync(Uri sourceUri, CloudFile destFile, bool isServiceCopy, CopyOptions options, TransferContext context, CancellationToken cancellationToken)
{
if (!isServiceCopy)
{
throw new NotSupportedException(Resources.SyncCopyFromUriToAzureFileNotSupportedException);
}
TransferLocation sourceLocation = new TransferLocation(sourceUri);
TransferLocation destLocation = new TransferLocation(destFile);
return CopyInternalAsync(sourceLocation, destLocation, isServiceCopy, options, context, cancellationToken);
}
/// <summary>
/// Copy file from an specified URI to an Azure file.
/// </summary>
/// <param name="sourceUri">The <see cref="System.Uri"/> of the source file.</param>
/// <param name="destFile">The <see cref="CloudFile"/> that is the destination Azure file.</param>
/// <param name="isServiceCopy">A flag indicating whether the copy is service-side asynchronous copy or not.
/// If this flag is set to true, service-side asychronous copy will be used; if this flag is set to false,
/// file is downloaded from source first, then uploaded to destination.</param>
/// <param name="options">A <see cref="CopyOptions"/> object that specifies additional options for the operation.</param>
/// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param>
/// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns>
/// <remarks>Copying from an URI to Azure file synchronously is not supported yet.</remarks>
public static Task CopyAsync(Uri sourceUri, CloudFile destFile, bool isServiceCopy, CopyOptions options, TransferContext context)
{
return CopyAsync(sourceUri, destFile, isServiceCopy, options, context, CancellationToken.None);
}
/// <summary>
/// Copy content, properties and metadata of an Azure file to another.
/// </summary>
/// <param name="sourceFile">The <see cref="CloudFile"/> that is the source Azure file.</param>
/// <param name="destFile">The <see cref="CloudFile"/> that is the destination Azure file.</param>
/// <param name="isServiceCopy">A flag indicating whether the copy is service-side asynchronous copy or not.
/// If this flag is set to true, service-side asychronous copy will be used; if this flag is set to false,
/// file is downloaded from source first, then uploaded to destination.</param>
/// <param name="options">A <see cref="CopyOptions"/> object that specifies additional options for the operation.</param>
/// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> object to observe while waiting for a task to complete.</param>
/// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns>
public static Task CopyAsync(CloudFile sourceFile, CloudFile destFile, bool isServiceCopy, CopyOptions options, TransferContext context, CancellationToken cancellationToken)
{
TransferLocation sourceLocation = new TransferLocation(sourceFile);
TransferLocation destLocation = new TransferLocation(destFile);
return CopyInternalAsync(sourceLocation, destLocation, isServiceCopy, options, context, cancellationToken);
}
public void TestResume()
{
int fileSizeInKB = 100 * 1024;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSizeInKB);
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
var options = new TestExecutionOptions<DMLibDataInfo>();
options.LimitSpeed = true;
var transferContext = new TransferContext();
var progressChecker = new ProgressChecker(1, fileSizeInKB * 1024);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
options.TransferItemModifier = (fileName, item) =>
{
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
transferItem = item;
};
TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
// Store the first checkpoint
firstCheckpoint = transferContext.LastCheckpoint;
Thread.Sleep(1000);
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
secondCheckpoint = transferContext.LastCheckpoint;
};
// Cancel and store checkpoint for resume
var result = this.ExecuteTestCase(sourceDataInfo, options);
Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled");
Exception exception = result.Exceptions[0];
VerificationHelper.VerifyExceptionErrorMessage(exception, "A task was canceled.");
TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null;
// DMLib doesn't support to resume transfer from a checkpoint which is inconsistent with
// the actual transfer progress when the destination is an append blob.
if (Helper.RandomBoolean() && DMLibTestContext.DestType != DMLibDataType.AppendBlob)
{
Test.Info("Resume with the first checkpoint first.");
firstResumeCheckpoint = firstCheckpoint;
secondResumeCheckpoint = secondCheckpoint;
}
else
{
Test.Info("Resume with the second checkpoint first.");
firstResumeCheckpoint = secondCheckpoint;
secondResumeCheckpoint = firstCheckpoint;
}
// resume with firstResumeCheckpoint
TransferItem resumeItem = transferItem.Clone();
progressChecker.Reset();
TransferContext resumeContext = new TransferContext(firstResumeCheckpoint)
{
ProgressHandler = progressChecker.GetProgressHandler()
};
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List<TransferItem>() { resumeItem }, new TestExecutionOptions<DMLibDataInfo>());
VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo);
// resume with secondResumeCheckpoint
resumeItem = transferItem.Clone();
progressChecker.Reset();
resumeContext = new TransferContext(secondResumeCheckpoint)
{
ProgressHandler = progressChecker.GetProgressHandler()
};
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List<TransferItem>() { resumeItem }, new TestExecutionOptions<DMLibDataInfo>());
if (DMLibTestContext.DestType != DMLibDataType.AppendBlob || DMLibTestContext.SourceType == DMLibDataType.Stream)
{
VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo);
}
else
{
Test.Assert(result.Exceptions.Count == 1, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob.");
exception = result.Exceptions[0];
Test.Assert(exception is InvalidOperationException, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob.");
VerificationHelper.VerifyExceptionErrorMessage(exception, "Destination might be changed by other process or application.");
}
}
public void DirectoryOverwriteDestination()
{
string destExistYName = "destExistY";
string destExistNName = "destExistN";
string destNotExistYName = "destNotExistY";
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistYName, 1024);
DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistNName, 1024);
DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destNotExistYName, 1024);
DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistYName, 1024);
DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistNName, 1024);
TransferContext transferContext = new TransferContext();
transferContext.OverwriteCallback = (string sourcePath, string destinationPath) =>
{
if (sourcePath.EndsWith(destExistNName))
{
return false;
}
else
{
return true;
}
};
int skipCount = 0;
int successCount = 0;
transferContext.FileSkipped += (object sender, TransferEventArgs args) =>
{
Interlocked.Increment(ref skipCount);
TransferException transferException = args.Exception as TransferException;
Test.Assert(transferException != null, "Verify the exception is a TransferException");
VerificationHelper.VerifyTransferException(transferException, TransferErrorCode.NotOverwriteExistingDestination,
"Skiped file", destExistNName);
};
transferContext.FileTransferred += (object sender, TransferEventArgs args) =>
{
Interlocked.Increment(ref successCount);
};
var options = new TestExecutionOptions<DMLibDataInfo>();
options.IsDirectoryTransfer = true;
if (DMLibTestContext.DestType != DMLibDataType.Stream)
{
options.DestTransferDataInfo = destDataInfo;
}
options.TransferItemModifier = (fileNode, transferItem) =>
{
transferItem.TransferContext = transferContext;
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferItem.Options = transferOptions;
};
var result = this.ExecuteTestCase(sourceDataInfo, options);
DMLibDataInfo expectedDataInfo = new DMLibDataInfo(string.Empty);
if (DMLibTestContext.DestType != DMLibDataType.Stream)
{
expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destExistYName));
expectedDataInfo.RootNode.AddFileNode(destDataInfo.RootNode.GetFileNode(destExistNName));
expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destNotExistYName));
}
else
{
expectedDataInfo = sourceDataInfo;
}
// Verify transfer result
Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, result.DataInfo), "Verify transfer result.");
// Verify exception
if (DMLibTestContext.DestType != DMLibDataType.Stream)
{
Test.Assert(successCount == 2, "Verify success transfers");
Test.Assert(skipCount == 1, "Verify skipped transfer");
}
else
{
Test.Assert(successCount == 3, "Very all transfers are success");
Test.Assert(skipCount == 0, "Very no transfer is skipped");
}
}
private static Task CopyInternalAsync(TransferLocation sourceLocation, TransferLocation destLocation, bool isServiceCopy, CopyOptions options, TransferContext context, CancellationToken cancellationToken)
{
if (options != null)
{
sourceLocation.AccessCondition = options.SourceAccessCondition;
destLocation.AccessCondition = options.DestinationAccessCondition;
}
Transfer transfer = CreateSingleObjectTransfer(sourceLocation, destLocation, isServiceCopy ? TransferMethod.AsyncCopy : TransferMethod.SyncCopy, context);
return DoTransfer(transfer, cancellationToken);
}
/// <summary>
/// Download an Azure file from Azure File Storage.
/// </summary>
/// <param name="sourceFile">The <see cref="CloudFile"/> that is the source Azure file.</param>
/// <param name="destStream">A <see cref="System.IO.Stream"/> object representing the destination stream.</param>
/// <param name="options">A <see cref="DownloadOptions"/> object that specifies additional options for the operation.</param>
/// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> object to observe while waiting for a task to complete.</param>
/// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns>
public static Task DownloadAsync(CloudFile sourceFile, Stream destStream, DownloadOptions options, TransferContext context, CancellationToken cancellationToken)
{
TransferLocation sourceLocation = new TransferLocation(sourceFile);
TransferLocation destLocation = new TransferLocation(destStream);
if (options != null)
{
FileRequestOptions requestOptions = Transfer_RequestOptions.DefaultFileRequestOptions;
requestOptions.DisableContentMD5Validation = options.DisableContentMD5Validation;
sourceLocation.RequestOptions = requestOptions;
}
return DownloadInternalAsync(sourceLocation, destLocation, options, context, cancellationToken);
}
/// <summary>
/// Generate an OperationContext from the the specified TransferContext.
/// </summary>
/// <param name="transferContext">Transfer context</param>
/// <returns>An <see cref="OperationContext"/> object.</returns>
public static OperationContext GenerateOperationContext(
TransferContext transferContext)
{
if (transferContext == null)
{
return null;
}
return new OperationContext()
{
ClientRequestID = transferContext.ClientRequestId,
LogLevel = transferContext.LogLevel,
};
}
public async static Task ProcessMessage([QueueTrigger("backupqueue")] CopyItem copyItem, TextWriter log, CancellationToken cancelToken)
{
// Copy TextWrite into Log Helper class
Logger.log = log;
// Log Job Start
await Logger.JobStartAsync(copyItem.JobName);
// This class accumulates transfer data during the copy
ProgressRecorder progressRecorder = new ProgressRecorder();
try
{
// OpContext to track PreCopy Retries on Azure Storage
// DML has its own context object and retry
_opContext = new OperationContext();
_opContext.Retrying += StorageRequest_Retrying;
// Define Blob Request Options
_blobRequestOptions = new BlobRequestOptions
{
// Defined Exponential Retry Policy above
RetryPolicy = _retryPolicy
};
// Set the number of parallel tasks in DML.
// This allows it to copy multiple items at once when copying a container or directory
// The best (and default value) is Environment.ProcessorCount * 8
int parallelTasks = Environment.ProcessorCount * 8;
TransferManager.Configurations.ParallelOperations = parallelTasks;
// Set the number of connections.
// This should match ParallelOperations so each DML copy task has its own connection to Azure Storage
ServicePointManager.DefaultConnectionLimit = parallelTasks;
// Short circuit additional request round trips. We are not chunking and
// uploading large amounts of data where we'd send 100's so set to false
ServicePointManager.Expect100Continue = false;
// User Agent for tracing
TransferManager.Configurations.UserAgentPrefix = "AzureDmlBackup";
// CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
// Open connections to both storage accounts
CloudStorageAccount sourceAccount = GetAccount(copyItem.SourceAccountToken);
CloudStorageAccount destinationAccount = GetAccount(copyItem.DestinationAccountToken);
// Represents a checkpoint from which a transfer may be resumed and continued.
// This is initalized as null first time then hydrated within CopyDirectoryAsync().
// However if this job is being resumed from a previous failure this function will hydrate
// from a serialized checkpoint saved to blob storage.
TransferCheckpoint transferCheckpoint = await GetTransferCheckpoint(copyItem.JobId);
// Context object for the transfer, provides additional runtime information about its execution
// If this is a resumed copy operation then pass the checkpoint to the TransferContext so it can resume the copy
TransferContext transferContext = new TransferContext(transferCheckpoint)
{
// Pipe transfer progress data to ProgressRecorder
// ProgressRecorder is used to log the results of the copy operation
ProgressHandler = progressRecorder,
// If the destination already exists this delegate is called.
// Return true to overwrite or false to skip the file during the transfer
OverwriteCallback = (source, destination) =>
{
return OverwriteFile(source, destination, sourceAccount, destinationAccount, copyItem.IsIncremental);
}
};
// This event is used to log files skipped during the transfer
transferContext.FileSkipped += TransferContext_FileSkipped;
// This event is used to catch exceptions for files that fail during a transfer
transferContext.FileFailed += TransferContext_FileFailed;
// Set Options for copying the container such as search patterns, recursive, etc.
CopyDirectoryOptions copyDirectoryOptions = new CopyDirectoryOptions
{
IncludeSnapshots = true,
Recursive = true
};
// Get the root source and destination directories for the two containers to be copied
CloudBlobDirectory sourceDirectory = await GetDirectoryAsync(sourceAccount, copyItem.SourceContainer, copyItem.SourceDirectory);
CloudBlobDirectory destinationDirectory = await GetDirectoryAsync(destinationAccount, copyItem.DestinationContainer, copyItem.DestinationDirectory);
// Copy the container
await CopyDirectoryAsync(copyItem.JobId, sourceDirectory, destinationDirectory, copyDirectoryOptions, transferContext, transferCheckpoint, cancellationTokenSource);
// Check if any files failed during transfer
if (_failedFiles.Count > 0)
{
// Save a Checkpoint so we can restart the transfer
transferCheckpoint = transferContext.LastCheckpoint;
SaveTransferCheckpoint(copyItem.JobId, transferCheckpoint);
// Throw an exception to fail the job so WebJobs will rerun it
throw new Exception("One or more errors occurred during the transfer.");
}
// Log job completion
await Logger.JobCompleteAsync(copyItem.JobName, progressRecorder, _skippedFiles);
}
catch (Exception ex)
{
// Log Job Error
await Logger.JobErrorAsync(copyItem.JobName, ex.Message, progressRecorder, _failedFiles, _skippedFiles);
// Rethrow the error to fail the web job
throw ex;
}
}
public void OverwriteDestination()
{
string destExistYName = "destExistY";
string destExistNName = "destExistN";
string destNotExistYName = "destNotExistY";
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistYName, 1024);
DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistNName, 1024);
DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destNotExistYName, 1024);
DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistYName, 1024);
DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistNName, 1024);
var options = new TestExecutionOptions<DMLibDataInfo>();
if (DMLibTestContext.DestType != DMLibDataType.Stream)
{
options.DestTransferDataInfo = destDataInfo;
}
options.TransferItemModifier = (fileNode, transferItem) =>
{
string fileName = fileNode.Name;
TransferContext transferContext = new TransferContext();
if (fileName.Equals(destExistYName))
{
transferContext.OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY();
}
else if (fileName.Equals(destExistNName))
{
transferContext.OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackN();
}
else if (fileName.Equals(destNotExistYName))
{
transferContext.OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY();
}
transferItem.TransferContext = transferContext;
};
var result = this.ExecuteTestCase(sourceDataInfo, options);
DMLibDataInfo expectedDataInfo = new DMLibDataInfo(string.Empty);
if (DMLibTestContext.DestType != DMLibDataType.Stream)
{
expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destExistYName));
expectedDataInfo.RootNode.AddFileNode(destDataInfo.RootNode.GetFileNode(destExistNName));
expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destNotExistYName));
}
else
{
expectedDataInfo = sourceDataInfo;
}
// Verify transfer result
Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, result.DataInfo), "Verify transfer result.");
// Verify exception
if (DMLibTestContext.DestType != DMLibDataType.Stream)
{
Test.Assert(result.Exceptions.Count == 1, "Verify there's only one exceptions.");
TransferException transferException = result.Exceptions[0] as TransferException;
Test.Assert(transferException != null, "Verify the exception is a TransferException");
VerificationHelper.VerifyTransferException(transferException, TransferErrorCode.NotOverwriteExistingDestination,
"Skiped file", destExistNName);
}
}
/// <summary>
/// Upload a file to Azure Blob Storage.
/// </summary>
/// <param name="sourcePath">Path to the source file.</param>
/// <param name="destBlob">The <see cref="CloudBlob"/> that is the destination Azure blob.</param>
/// <param name="options">An <see cref="UploadOptions"/> object that specifies additional options for the operation.</param>
/// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param>
/// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns>
public static Task UploadAsync(string sourcePath, CloudBlob destBlob, UploadOptions options, TransferContext context)
{
return UploadAsync(sourcePath, destBlob, options, context, CancellationToken.None);
}
private static Transfer CreateSingleObjectTransfer(TransferLocation sourceLocation, TransferLocation destLocation, TransferMethod transferMethod, TransferContext transferContext)
{
Transfer transfer = GetTransfer(sourceLocation, destLocation, transferMethod, transferContext);
if (transfer == null)
{
transfer = new SingleObjectTransfer(sourceLocation, destLocation, transferMethod);
if (transferContext != null)
{
transferContext.Checkpoint.AddTransfer(transfer);
}
}
if (transferContext != null)
{
transfer.ProgressTracker.Parent = transferContext.OverallProgressTracker;
transfer.Context = transferContext;
}
return transfer;
}
/// <summary>
/// Download an Azure blob from Azure Blob Storage.
/// </summary>
/// <param name="sourceBlob">The <see cref="CloudBlob"/> that is the source Azure blob.</param>
/// <param name="destPath">Path to the destination file.</param>
/// <param name="options">A <see cref="DownloadOptions"/> object that specifies additional options for the operation.</param>
/// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> object to observe while waiting for a task to complete.</param>
/// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns>
public static Task DownloadAsync(CloudBlob sourceBlob, string destPath, DownloadOptions options, TransferContext context, CancellationToken cancellationToken)
{
TransferLocation sourceLocation = new TransferLocation(sourceBlob);
TransferLocation destLocation = new TransferLocation(destPath);
if (options != null)
{
BlobRequestOptions requestOptions = Transfer_RequestOptions.DefaultBlobRequestOptions;
requestOptions.DisableContentMD5Validation = options.DisableContentMD5Validation;
sourceLocation.RequestOptions = requestOptions;
}
return DownloadInternalAsync(sourceLocation, destLocation, options, context, cancellationToken);
}
public void ResumeInAllDirections()
{
List<TransferItem> allItems = AllTransferDirectionTest.GetTransformItemsForAllDirections(resume: true);
int fileCount = expectedFileNodes.Keys.Count;
// Execution and store checkpoints
CancellationTokenSource tokenSource = new CancellationTokenSource();
var transferContext = new TransferContext();
var progressChecker = new ProgressChecker(fileCount, 1024 * fileCount);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
allItems.ForEach(item =>
{
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
});
var options = new TestExecutionOptions<DMLibDataInfo>();
options.DisableDestinationFetch = true;
// Checkpoint names
const string PartialStarted = "PartialStarted",
AllStarted = "AllStarted",
AllStartedAndWait = "AllStartedAndWait",
BeforeCancel = "BeforeCancel",
AfterCancel = "AfterCancel";
Dictionary<string, TransferCheckpoint> checkpoints = new Dictionary<string, TransferCheckpoint>();
TransferItem randomItem = allItems[random.Next(0, allItems.Count)];
randomItem.AfterStarted = () =>
{
Test.Info("Store check point after transfer item: {0}.", randomItem.ToString());
checkpoints.Add(PartialStarted, transferContext.LastCheckpoint);
};
options.AfterAllItemAdded = () =>
{
progressChecker.DataTransferred.WaitOne();
checkpoints.Add(AllStarted, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(AllStartedAndWait, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(BeforeCancel, transferContext.LastCheckpoint);
tokenSource.Cancel();
checkpoints.Add(AfterCancel, transferContext.LastCheckpoint);
};
var result = this.RunTransferItems(allItems, options);
// Resume with stored checkpoints in random order
var checkpointList = new List<KeyValuePair<string,TransferCheckpoint>>();
checkpointList.AddRange(checkpoints);
checkpointList.Shuffle();
foreach(var pair in checkpointList)
{
Test.Info("===Resume with checkpoint '{0}'===", pair.Key);
options = new TestExecutionOptions<DMLibDataInfo>();
options.DisableDestinationFetch = true;
progressChecker.Reset();
transferContext = new TransferContext(DMLibTestHelper.RandomReloadCheckpoint(pair.Value))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// The checkpoint can be stored when DMLib doesn't check overwrite callback yet.
// So it will case an skip file error if the desination file already exists and
// We don't have overwrite callback here.
OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY()
};
TransferEventChecker eventChecker = new TransferEventChecker();
eventChecker.Apply(transferContext);
List<TransferItem> itemsToResume = allItems.Select(item =>
{
TransferItem itemToResume = item.Clone();
itemToResume.TransferContext = transferContext;
return itemToResume;
}).ToList();
result = this.RunTransferItems(itemsToResume, options);
foreach (DMLibDataType destDataType in DataTypes)
{
DataAdaptor<DMLibDataInfo> destAdaptor = GetSourceAdaptor(destDataType);
DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty);
foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
{
string fileName = destFileNode.Name;
FileNode sourceFileNode = expectedFileNodes[fileName];
Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result.");
}
}
Test.Assert(result.Exceptions.Count == 0, "Verify no error happens. Actual: {0}", result.Exceptions.Count);
}
}
private static Transfer GetTransfer(TransferLocation sourceLocation, TransferLocation destLocation, TransferMethod transferMethod, TransferContext transferContext)
{
Transfer transfer = null;
if (transferContext != null)
{
transfer = transferContext.Checkpoint.GetTransfer(sourceLocation, destLocation, transferMethod);
if (transfer != null)
{
// update transfer location information
UpdateTransferLocation(transfer.Source, sourceLocation);
UpdateTransferLocation(transfer.Destination, destLocation);
}
}
return transfer;
}
