private void SetRequestOptionsInDataMovement(BlobTransferOptions opts) { BlobRequestOptions cmdletOptions = RequestOptions; if (cmdletOptions == null) { return; } foreach (BlobRequestOperation operation in Enum.GetValues(typeof(BlobRequestOperation))) { BlobRequestOptions requestOptions = opts.GetBlobRequestOptions(operation); if (cmdletOptions.MaximumExecutionTime != null) { requestOptions.MaximumExecutionTime = cmdletOptions.MaximumExecutionTime; } if (cmdletOptions.ServerTimeout != null) { requestOptions.ServerTimeout = cmdletOptions.ServerTimeout; } opts.SetBlobRequestOptions(operation, requestOptions); } }
/// <summary> /// Cmdlet begin processing /// </summary> protected override void BeginProcessing() { base.BeginProcessing(); BlobTransferOptions opts = new BlobTransferOptions(); opts.Concurrency = GetCmdletConcurrency(); opts.AppendToClientRequestId(CmdletOperationContext.ClientRequestId); opts.OverwritePromptCallback = ConfirmOverwrite; SetRequestOptionsInDataMovement(opts); transferManager = new BlobTransferManager(opts); CmdletCancellationToken.Register(() => transferManager.CancelWork()); }
/// <summary> /// Cmdlet begin processing /// </summary> protected override void BeginProcessing() { if (concurrentTaskCount == 0) { concurrentTaskCount = Environment.ProcessorCount * asyncTasksPerCoreMultiplier; } ConfigureServicePointManager(); BlobTransferOptions opts = new BlobTransferOptions(); opts.Concurrency = concurrentTaskCount; transferManager = new BlobTransferManager(opts); base.BeginProcessing(); }
/// <summary> /// Gets data content from the data container /// </summary> /// <returns>data content in the data container</returns> public byte[] GetData() { TraceHelper.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BlobDataContainer] .GetData: blob url={0}", this.dataBlob.Uri.AbsoluteUri); WaitUntilTransferComplete(this.dataBlob); Exception transferException = null; byte[] data = new byte[this.dataBlob.Properties.Length]; using (MemoryStream ms = new MemoryStream(data)) { TraceHelper.TraceSource.TraceEvent( TraceEventType.Verbose, 0, "[BlobDataContainer] .GetData: download blob. threadCount={0} minBackoffInSeconds={1} maxBackoffInSeconds={2} retryCount={3} timeoutInSeconds={4}", downloadBlobThreadCount, downloadBlobMinBackoffInSeconds, downloadBlobMaxBackoffInSeconds, downloadBlobRetryCount, downloadBlobTimeoutInSeconds); ExponentialRetry downloadRetryPolicy = new ExponentialRetry( TimeSpan.FromSeconds(downloadBlobMinBackoffInSeconds), downloadBlobRetryCount); BlobRequestOptions blobRequestOptions = new BlobRequestOptions() { RetryPolicy = downloadRetryPolicy, MaximumExecutionTime = TimeSpan.FromSeconds(downloadBlobTimeoutInSeconds), }; BlobTransferOptions options = new BlobTransferOptions(); options.Concurrency = downloadBlobThreadCount; options.SetBlobRequestOptions(BlobRequestOperation.OpenRead, blobRequestOptions); using (BlobTransferManager transferManager = new BlobTransferManager(options)) { transferManager.QueueDownload( this.dataBlob, ms, true, null, delegate(object userData, double speed, double progress) { TraceHelper.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BlobDataContainer] .GetData: progress={0}%", progress); }, delegate(object userData, Exception ex) { if (ex != null) { transferException = ex; } }, null); transferManager.WaitForCompletion(); } } TraceHelper.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BlobDataContainer] .GetData: data transfer done"); if (transferException == null) { return(data); } TraceHelper.TraceSource.TraceEvent(TraceEventType.Error, 0, "[BlobDataContainer] .GetData: received exception={0}", transferException); throw TranslateTransferExceptionToDataException(transferException); }
/// <summary> /// Write a data item into data container and flush /// </summary> /// <param name="data">data content to be written</param> public void AddDataAndFlush(DataContent data) { TraceHelper.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BlobDataContainer] .AddDataAndFlush"); using (MemoryStream ms = new MemoryStream()) { // dump all data into a memory stream data.Dump(ms); // create timer that updates "CommonDataLastUpdateTime" metadata peoriodically Timer updateMetadataTimer = new Timer( this.MarkBlobAsBeingUploaded, null, TimeSpan.FromMilliseconds(Constant.LastUpdateTimeUpdateIntervalInMilliseconds), TimeSpan.FromMilliseconds(Constant.LastUpdateTimeUpdateIntervalInMilliseconds)); // write data Exception transferException = null; try { BlobTransferOptions transferOptions = new BlobTransferOptions { Concurrency = Environment.ProcessorCount * 8, }; using (BlobTransferManager transferManager = new BlobTransferManager(transferOptions)) { transferManager.QueueUpload( this.dataBlob, ms, null, delegate(object userData, double speed, double progress) { TraceHelper.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BlobDataContainer] .AddDataAndFlush: progress={0}%", progress); }, delegate(object userData, Exception ex) { if (ex != null) { transferException = ex; } }, null); transferManager.WaitForCompletion(); } } finally { updateMetadataTimer.Dispose(); } TraceHelper.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BlobDataContainer] .AddDataAndFlush: data transfer done"); DataException dataException = null; if (transferException != null) { dataException = TranslateTransferExceptionToDataException(transferException); } try { int errorCode = DataErrorCode.Success; string errorMessage = string.Empty; if (dataException != null) { errorCode = dataException.ErrorCode; errorMessage = dataException.Message; } AzureBlobHelper.MarkBlobAsCompleted(this.dataBlob, errorCode.ToString(), errorMessage); } catch (StorageException ex) { TraceHelper.TraceSource.TraceEvent( TraceEventType.Error, 0, "[BlobDataContainer] .AddDataAndFlush: failed to mark blob as completed. blob url={0}. error code={1}, exception={2}", this.dataBlob.Uri.AbsoluteUri, BurstUtility.GetStorageErrorCode(ex), ex); } catch (Exception ex) { TraceHelper.TraceSource.TraceEvent( TraceEventType.Error, 0, "[BlobDataContainer] .AddDataAndFlush: failed to mark blob as completed. blob url={0}. Exception={1}", this.dataBlob.Uri.AbsoluteUri, ex); } if (dataException != null) { throw dataException; } } }
//********************************************************************* /// /// <summary> /// /// </summary> /// <param name="config"></param> /// <param name="progressCallback"></param> /// <param name="speedChangeCallback"></param> /// <returns></returns> /// //********************************************************************* //*** NOTE * Network public AzureAdminClientLib.Response UploadFileToCloud(BlobTransferInfo config, ProgressFileCallbackDelegate progressCallback, SpeedChangeDelegate speedChangeCallback) { var resp = new AzureAdminClientLib.Response(); _PFCBD = progressCallback; _SCD = speedChangeCallback; _OverwriteDestination = config.OverwriteDestination; _SourceFileLength = config.LocalFile.Length; var pivot = config.LocalFile.FullName.LastIndexOf('\\'); var sourceLocation = config.LocalFile.FullName.Substring(0, pivot); var theFilePatterns = new List <string>(1) { config.LocalFile.FullName.Substring(pivot + 1) }; var destinationLocation = config.Url.AbsoluteUri; var destinationStorageKey = config.Key; var rem = config.LocalFile.Length % PAGEBLOBCHUNCKSIZE; Microsoft.WindowsAzure.Storage.Blob.BlobType theUploadBlobType; if (0 == rem) { theUploadBlobType = Microsoft.WindowsAzure.Storage.Blob.BlobType.PageBlob; } else { theUploadBlobType = Microsoft.WindowsAzure.Storage.Blob.BlobType.BlockBlob; } var ShouldTransferSnapshots = false; var ShouldExcludeNewer = false; var ShouldExcludeOlder = false; var FileIncludedAttributes = new System.IO.FileAttributes(); var FileExcludedAttributes = new System.IO.FileAttributes(); var ShouldOnlyFilesWithArchiveBits = false; var ShouldRecursive = false; var ShouldFakeTransfer = false; object TagData = resp; if (0 == config.TransferThreadCount) { config.TransferThreadCount = Environment.ProcessorCount * 8; } ConfigureServicePointManager(config.TransferThreadCount); var transferOptions = new BlobTransferOptions { Concurrency = GetTransferThreadCount(config.TransferThreadCount), OverwritePromptCallback = new BlobTransferOverwritePromptCallback(this.OverwritePromptCallback) }; transferOptions.AppendToClientRequestId(GetAzCopyClientRequestID()); BlobTransferRecursiveTransferOptions recursiveTransferOptions = null; using (var transferManager = new BlobTransferManager(transferOptions)) { try { recursiveTransferOptions = new BlobTransferRecursiveTransferOptions { DestinationKey = destinationStorageKey, TransferSnapshots = ShouldTransferSnapshots, ExcludeNewer = ShouldExcludeNewer, ExcludeOlder = ShouldExcludeOlder, IncludedAttributes = FileIncludedAttributes, ExcludedAttributes = FileExcludedAttributes, OnlyFilesWithArchiveBit = ShouldOnlyFilesWithArchiveBits, UploadBlobType = theUploadBlobType, FilePatterns = theFilePatterns, Recursive = ShouldRecursive, FakeTransfer = ShouldFakeTransfer, MoveFile = config.DeleteSource, FileTransferStatus = null }; transferManager.GlobalCopySpeedUpdated += new EventHandler <BlobTransferManagerEventArgs>(this.SpeedChangeCallback); transferManager.GlobalDownloadSpeedUpdated += new EventHandler <BlobTransferManagerEventArgs>(this.SpeedChangeCallback); transferManager.GlobalUploadSpeedUpdated += new EventHandler <BlobTransferManagerEventArgs>(this.SpeedChangeCallback); } catch (Exception ex) { resp.ResultStatus = Response.ResultStatusEnum.EXCEPTION; resp.ExceptionMessage = "Exception in UploadFileToCloud() during setup : " + CmpInterfaceModel.Utilities.UnwindExceptionMessages(ex); } try { transferManager.QueueRecursiveTransfer(sourceLocation, destinationLocation, recursiveTransferOptions, StartCallback, ExceptionCallback, StartFileCallback, ProgressFileCallback, FinishFileCallback, TagData); } catch (Exception ex) { resp.ResultStatus = Response.ResultStatusEnum.EXCEPTION; resp.ExceptionMessage = "Exception in UploadFileToCloud() during queue up : " + CmpInterfaceModel.Utilities.UnwindExceptionMessages(ex); } transferManager.WaitForCompletion(); return(resp); } }