private void SetRequestOptionsInDataMovement(BlobTransferOptions opts)
{
BlobRequestOptions cmdletOptions = RequestOptions;
if (cmdletOptions == null)
{
return;
}
foreach (BlobRequestOperation operation in Enum.GetValues(typeof(BlobRequestOperation)))
{
BlobRequestOptions requestOptions = opts.GetBlobRequestOptions(operation);
if (cmdletOptions.MaximumExecutionTime != null)
{
requestOptions.MaximumExecutionTime = cmdletOptions.MaximumExecutionTime;
}
if (cmdletOptions.ServerTimeout != null)
{
requestOptions.ServerTimeout = cmdletOptions.ServerTimeout;
}
opts.SetBlobRequestOptions(operation, requestOptions);
}
}
/// <summary>
/// Cmdlet begin processing
/// </summary>
protected override void BeginProcessing()
{
base.BeginProcessing();
BlobTransferOptions opts = new BlobTransferOptions();
opts.Concurrency = GetCmdletConcurrency();
opts.AppendToClientRequestId(CmdletOperationContext.ClientRequestId);
opts.OverwritePromptCallback = ConfirmOverwrite;
SetRequestOptionsInDataMovement(opts);
transferManager = new BlobTransferManager(opts);
CmdletCancellationToken.Register(() => transferManager.CancelWork());
}
/// <summary>
/// Cmdlet begin processing
/// </summary>
protected override void BeginProcessing()
{
if (concurrentTaskCount == 0)
{
concurrentTaskCount = Environment.ProcessorCount * asyncTasksPerCoreMultiplier;
}
ConfigureServicePointManager();
BlobTransferOptions opts = new BlobTransferOptions();
opts.Concurrency = concurrentTaskCount;
transferManager = new BlobTransferManager(opts);
base.BeginProcessing();
}
/// <summary>
/// Cmdlet begin processing
/// </summary>
protected override void BeginProcessing()
{
if (concurrentTaskCount == 0)
{
concurrentTaskCount = Environment.ProcessorCount * asyncTasksPerCoreMultiplier;
}
ConfigureServicePointManager();
BlobTransferOptions opts = new BlobTransferOptions();
opts.Concurrency = concurrentTaskCount;
transferManager = new BlobTransferManager(opts);
base.BeginProcessing();
}
private void SetRequestOptionsInDataMovement(BlobTransferOptions opts)
{
BlobRequestOptions cmdletOptions = RequestOptions;
if (cmdletOptions == null)
{
return;
}
foreach (BlobRequestOperation operation in Enum.GetValues(typeof(BlobRequestOperation)))
{
BlobRequestOptions requestOptions = opts.GetBlobRequestOptions(operation);
if (cmdletOptions.MaximumExecutionTime != null)
{
requestOptions.MaximumExecutionTime = cmdletOptions.MaximumExecutionTime;
}
if (cmdletOptions.ServerTimeout != null)
{
requestOptions.ServerTimeout = cmdletOptions.ServerTimeout;
}
opts.SetBlobRequestOptions(operation, requestOptions);
}
}
/// <summary>
/// Cmdlet begin processing
/// </summary>
protected override void BeginProcessing()
{
base.BeginProcessing();
BlobTransferOptions opts = new BlobTransferOptions();
opts.Concurrency = GetCmdletConcurrency();
opts.AppendToClientRequestId(CmdletOperationContext.ClientRequestId);
opts.OverwritePromptCallback = ConfirmOverwrite;
SetRequestOptionsInDataMovement(opts);
transferManager = new BlobTransferManager(opts);
CmdletCancellationToken.Register(() => transferManager.CancelWork());
}
/// <summary>
/// Gets data content from the data container
/// </summary>
/// <returns>data content in the data container</returns>
public byte[] GetData()
{
TraceHelper.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BlobDataContainer] .GetData: blob url={0}", this.dataBlob.Uri.AbsoluteUri);
WaitUntilTransferComplete(this.dataBlob);
Exception transferException = null;
byte[] data = new byte[this.dataBlob.Properties.Length];
using (MemoryStream ms = new MemoryStream(data))
{
TraceHelper.TraceSource.TraceEvent(
TraceEventType.Verbose,
0,
"[BlobDataContainer] .GetData: download blob. threadCount={0} minBackoffInSeconds={1} maxBackoffInSeconds={2} retryCount={3} timeoutInSeconds={4}",
downloadBlobThreadCount,
downloadBlobMinBackoffInSeconds,
downloadBlobMaxBackoffInSeconds,
downloadBlobRetryCount,
downloadBlobTimeoutInSeconds);
ExponentialRetry downloadRetryPolicy =
new ExponentialRetry(
TimeSpan.FromSeconds(downloadBlobMinBackoffInSeconds),
downloadBlobRetryCount);
BlobRequestOptions blobRequestOptions = new BlobRequestOptions()
{
RetryPolicy = downloadRetryPolicy,
MaximumExecutionTime = TimeSpan.FromSeconds(downloadBlobTimeoutInSeconds),
};
BlobTransferOptions options = new BlobTransferOptions();
options.Concurrency = downloadBlobThreadCount;
options.SetBlobRequestOptions(BlobRequestOperation.OpenRead, blobRequestOptions);
using (BlobTransferManager transferManager = new BlobTransferManager(options))
{
transferManager.QueueDownload(
this.dataBlob,
ms,
true,
null,
delegate(object userData, double speed, double progress)
{
TraceHelper.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BlobDataContainer] .GetData: progress={0}%", progress);
},
delegate(object userData, Exception ex)
{
if (ex != null)
{
transferException = ex;
}
},
null);
transferManager.WaitForCompletion();
}
}
TraceHelper.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BlobDataContainer] .GetData: data transfer done");
if (transferException == null)
{
return(data);
}
TraceHelper.TraceSource.TraceEvent(TraceEventType.Error, 0, "[BlobDataContainer] .GetData: received exception={0}", transferException);
throw TranslateTransferExceptionToDataException(transferException);
}
/// <summary>
/// Write a data item into data container and flush
/// </summary>
/// <param name="data">data content to be written</param>
public void AddDataAndFlush(DataContent data)
{
TraceHelper.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BlobDataContainer] .AddDataAndFlush");
using (MemoryStream ms = new MemoryStream())
{
// dump all data into a memory stream
data.Dump(ms);
// create timer that updates "CommonDataLastUpdateTime" metadata peoriodically
Timer updateMetadataTimer = new Timer(
this.MarkBlobAsBeingUploaded,
null,
TimeSpan.FromMilliseconds(Constant.LastUpdateTimeUpdateIntervalInMilliseconds),
TimeSpan.FromMilliseconds(Constant.LastUpdateTimeUpdateIntervalInMilliseconds));
// write data
Exception transferException = null;
try
{
BlobTransferOptions transferOptions = new BlobTransferOptions
{
Concurrency = Environment.ProcessorCount * 8,
};
using (BlobTransferManager transferManager = new BlobTransferManager(transferOptions))
{
transferManager.QueueUpload(
this.dataBlob,
ms,
null,
delegate(object userData, double speed, double progress)
{
TraceHelper.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BlobDataContainer] .AddDataAndFlush: progress={0}%", progress);
},
delegate(object userData, Exception ex)
{
if (ex != null)
{
transferException = ex;
}
},
null);
transferManager.WaitForCompletion();
}
}
finally
{
updateMetadataTimer.Dispose();
}
TraceHelper.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BlobDataContainer] .AddDataAndFlush: data transfer done");
DataException dataException = null;
if (transferException != null)
{
dataException = TranslateTransferExceptionToDataException(transferException);
}
try
{
int errorCode = DataErrorCode.Success;
string errorMessage = string.Empty;
if (dataException != null)
{
errorCode = dataException.ErrorCode;
errorMessage = dataException.Message;
}
AzureBlobHelper.MarkBlobAsCompleted(this.dataBlob, errorCode.ToString(), errorMessage);
}
catch (StorageException ex)
{
TraceHelper.TraceSource.TraceEvent(
TraceEventType.Error,
0,
"[BlobDataContainer] .AddDataAndFlush: failed to mark blob as completed. blob url={0}. error code={1}, exception={2}",
this.dataBlob.Uri.AbsoluteUri,
BurstUtility.GetStorageErrorCode(ex),
ex);
}
catch (Exception ex)
{
TraceHelper.TraceSource.TraceEvent(
TraceEventType.Error,
0,
"[BlobDataContainer] .AddDataAndFlush: failed to mark blob as completed. blob url={0}. Exception={1}",
this.dataBlob.Uri.AbsoluteUri,
ex);
}
if (dataException != null)
{
throw dataException;
}
}
}
//*********************************************************************
///
/// <summary>
///
/// </summary>
/// <param name="config"></param>
/// <param name="progressCallback"></param>
/// <param name="speedChangeCallback"></param>
/// <returns></returns>
///
//*********************************************************************
//*** NOTE * Network
public AzureAdminClientLib.Response UploadFileToCloud(BlobTransferInfo config,
ProgressFileCallbackDelegate progressCallback, SpeedChangeDelegate speedChangeCallback)
{
var resp = new AzureAdminClientLib.Response();
_PFCBD = progressCallback;
_SCD = speedChangeCallback;
_OverwriteDestination = config.OverwriteDestination;
_SourceFileLength = config.LocalFile.Length;
var pivot = config.LocalFile.FullName.LastIndexOf('\\');
var sourceLocation = config.LocalFile.FullName.Substring(0, pivot);
var theFilePatterns = new List <string>(1)
{
config.LocalFile.FullName.Substring(pivot + 1)
};
var destinationLocation = config.Url.AbsoluteUri;
var destinationStorageKey = config.Key;
var rem = config.LocalFile.Length % PAGEBLOBCHUNCKSIZE;
Microsoft.WindowsAzure.Storage.Blob.BlobType theUploadBlobType;
if (0 == rem)
{
theUploadBlobType = Microsoft.WindowsAzure.Storage.Blob.BlobType.PageBlob;
}
else
{
theUploadBlobType = Microsoft.WindowsAzure.Storage.Blob.BlobType.BlockBlob;
}
var ShouldTransferSnapshots = false;
var ShouldExcludeNewer = false;
var ShouldExcludeOlder = false;
var FileIncludedAttributes = new System.IO.FileAttributes();
var FileExcludedAttributes = new System.IO.FileAttributes();
var ShouldOnlyFilesWithArchiveBits = false;
var ShouldRecursive = false;
var ShouldFakeTransfer = false;
object TagData = resp;
if (0 == config.TransferThreadCount)
{
config.TransferThreadCount = Environment.ProcessorCount * 8;
}
ConfigureServicePointManager(config.TransferThreadCount);
var transferOptions = new BlobTransferOptions
{
Concurrency = GetTransferThreadCount(config.TransferThreadCount),
OverwritePromptCallback =
new BlobTransferOverwritePromptCallback(this.OverwritePromptCallback)
};
transferOptions.AppendToClientRequestId(GetAzCopyClientRequestID());
BlobTransferRecursiveTransferOptions recursiveTransferOptions = null;
using (var transferManager = new BlobTransferManager(transferOptions))
{
try
{
recursiveTransferOptions = new BlobTransferRecursiveTransferOptions
{
DestinationKey = destinationStorageKey,
TransferSnapshots = ShouldTransferSnapshots,
ExcludeNewer = ShouldExcludeNewer,
ExcludeOlder = ShouldExcludeOlder,
IncludedAttributes = FileIncludedAttributes,
ExcludedAttributes = FileExcludedAttributes,
OnlyFilesWithArchiveBit = ShouldOnlyFilesWithArchiveBits,
UploadBlobType = theUploadBlobType,
FilePatterns = theFilePatterns,
Recursive = ShouldRecursive,
FakeTransfer = ShouldFakeTransfer,
MoveFile = config.DeleteSource,
FileTransferStatus = null
};
transferManager.GlobalCopySpeedUpdated +=
new EventHandler <BlobTransferManagerEventArgs>(this.SpeedChangeCallback);
transferManager.GlobalDownloadSpeedUpdated +=
new EventHandler <BlobTransferManagerEventArgs>(this.SpeedChangeCallback);
transferManager.GlobalUploadSpeedUpdated +=
new EventHandler <BlobTransferManagerEventArgs>(this.SpeedChangeCallback);
}
catch (Exception ex)
{
resp.ResultStatus = Response.ResultStatusEnum.EXCEPTION;
resp.ExceptionMessage = "Exception in UploadFileToCloud() during setup : " +
CmpInterfaceModel.Utilities.UnwindExceptionMessages(ex);
}
try
{
transferManager.QueueRecursiveTransfer(sourceLocation, destinationLocation,
recursiveTransferOptions, StartCallback, ExceptionCallback, StartFileCallback,
ProgressFileCallback, FinishFileCallback, TagData);
}
catch (Exception ex)
{
resp.ResultStatus = Response.ResultStatusEnum.EXCEPTION;
resp.ExceptionMessage = "Exception in UploadFileToCloud() during queue up : " +
CmpInterfaceModel.Utilities.UnwindExceptionMessages(ex);
}
transferManager.WaitForCompletion();
return(resp);
}
}