public async Task CopyBlobCopiesSourceContainerToDestinationPath()
        {
            VerifyKeys();

            // Arrange
            var account = new CloudStorageAccount(new StorageCredentials(AccountName, AccountKey), useHttps: true);
            var transferManager = new BlobTransferManager();
            var client = account.CreateCloudBlobClient();
            var srcContainer = client.GetContainerReference("feed-s25-049");
            var destLocation = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());

            try
            {
                // Act
                await transferManager.CopyContainer(srcContainer, destLocation);

                // Assert
                var srcItems = srcContainer.ListBlobs(prefix: null, useFlatBlobListing: true)
                                           .Select(s => s.Uri.LocalPath.TrimStart('/').Substring(srcContainer.Name.Length));
                destLocation = Path.Combine(destLocation, srcContainer.Name);
                var destItems = Directory.EnumerateFiles(destLocation, "*", SearchOption.AllDirectories)
                                         .Select(s => s.Substring(destLocation.Length).Replace('\\', '/'));
                Assert.True(new HashSet<string>(destItems, StringComparer.OrdinalIgnoreCase).SetEquals(srcItems));
            }
            finally
            {
                Directory.Delete(destLocation, recursive: true);
            }
        }
        public async Task CopyBlobCopiesSourceContainerToDestContainer()
        {
            VerifyKeys();

            // Arrange
            var account = new CloudStorageAccount(new StorageCredentials(AccountName, AccountKey), useHttps: true);
            var transferManager = new BlobTransferManager();
            var client = account.CreateCloudBlobClient();
            var srcContainer = client.GetContainerReference("feed-s25-049");
            var destContainer = client.GetContainerReference(Guid.NewGuid().ToString("N").ToLower());
            await destContainer.CreateAsync();

            try
            {
                // Act
                await transferManager.CopyContainer(srcContainer, destContainer);

                // Assert
                var srcItems = srcContainer.ListBlobs(prefix: null, useFlatBlobListing: true)
                                           .Select(s => s.Uri.LocalPath.TrimStart('/').Substring(srcContainer.Name.Length));
                var destItems = destContainer.ListBlobs(prefix: null, useFlatBlobListing: true)
                                             .Select(s => s.Uri.LocalPath.TrimStart('/').Substring(destContainer.Name.Length));
                Assert.True(new HashSet<string>(destItems, StringComparer.OrdinalIgnoreCase).SetEquals(srcItems));
            }
            finally
            {
                destContainer.DeleteIfExists();
            }
        }
Exemplo n.º 3
0
 public void Dispose()
 {
     if (transferManager != null)
     {
         transferManager.WaitForCompletion();
         transferManager.Dispose();
         transferManager = null;
     }
 }
Exemplo n.º 4
0
 /// <summary>
 /// Dispose DataMovement cmdlet
 /// </summary>
 /// <param name="disposing">User disposing</param>
 protected virtual void Dispose(bool disposing)
 {
     if (disposing)
     {
         if (transferManager != null)
         {
             transferManager.WaitForCompletion();
             transferManager.Dispose();
             transferManager = null;
         }
     }
 }
Exemplo n.º 5
0
        /// <summary>
        /// Cmdlet begin processing
        /// </summary>
        protected override void BeginProcessing()
        {
            base.BeginProcessing();

            BlobTransferOptions opts = new BlobTransferOptions();

            opts.Concurrency = GetCmdletConcurrency();
            opts.AppendToClientRequestId(CmdletOperationContext.ClientRequestId);
            opts.OverwritePromptCallback = ConfirmOverwrite;
            SetRequestOptionsInDataMovement(opts);
            transferManager = new BlobTransferManager(opts);
            CmdletCancellationToken.Register(() => transferManager.CancelWork());
        }
        /// <summary>
        /// Cmdlet begin processing
        /// </summary>
        protected override void BeginProcessing()
        {
            if (concurrentTaskCount == 0)
            {
                concurrentTaskCount = Environment.ProcessorCount * asyncTasksPerCoreMultiplier;
            }

            ConfigureServicePointManager();

            BlobTransferOptions opts = new BlobTransferOptions();

            opts.Concurrency = concurrentTaskCount;
            transferManager  = new BlobTransferManager(opts);

            base.BeginProcessing();
        }
 /// <summary>
 /// Dispose DataMovement cmdlet
 /// </summary>
 /// <param name="disposing">User disposing</param>
 protected virtual void Dispose(bool disposing)
 {
     if (disposing)
     {
         if (transferManager != null)
         {
             transferManager.WaitForCompletion();
             transferManager.Dispose();
             transferManager = null;
         }
     }
 }
        /// <summary>
        /// Cmdlet begin processing
        /// </summary>
        protected override void BeginProcessing()
        {
            if (concurrentTaskCount == 0)
            {
                concurrentTaskCount = Environment.ProcessorCount * asyncTasksPerCoreMultiplier;
            }

            ConfigureServicePointManager();

            BlobTransferOptions opts = new BlobTransferOptions();
            opts.Concurrency = concurrentTaskCount;
            transferManager = new BlobTransferManager(opts);

            base.BeginProcessing();
        }
        /// <summary>
        /// Cmdlet begin processing
        /// </summary>
        protected override void BeginProcessing()
        {
            base.BeginProcessing();

            BlobTransferOptions opts = new BlobTransferOptions();
            opts.Concurrency = GetCmdletConcurrency();
            opts.AppendToClientRequestId(CmdletOperationContext.ClientRequestId);
            opts.OverwritePromptCallback = ConfirmOverwrite;
            SetRequestOptionsInDataMovement(opts);
            transferManager = new BlobTransferManager(opts);
            CmdletCancellationToken.Register(() => transferManager.CancelWork());
        }
 public void Dispose()
 {
     if (transferManager != null)
     {
         transferManager.WaitForCompletion();
         transferManager.Dispose();
         transferManager = null;
     }
 }
Exemplo n.º 11
0
        /// <summary>
        /// Gets data content from the data container
        /// </summary>
        /// <returns>data content in the data container</returns>
        public byte[] GetData()
        {
            TraceHelper.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BlobDataContainer] .GetData: blob url={0}", this.dataBlob.Uri.AbsoluteUri);

            WaitUntilTransferComplete(this.dataBlob);

            Exception transferException = null;

            byte[] data = new byte[this.dataBlob.Properties.Length];
            using (MemoryStream ms = new MemoryStream(data))
            {
                TraceHelper.TraceSource.TraceEvent(
                    TraceEventType.Verbose,
                    0,
                    "[BlobDataContainer] .GetData: download blob. threadCount={0} minBackoffInSeconds={1} maxBackoffInSeconds={2} retryCount={3} timeoutInSeconds={4}",
                    downloadBlobThreadCount,
                    downloadBlobMinBackoffInSeconds,
                    downloadBlobMaxBackoffInSeconds,
                    downloadBlobRetryCount,
                    downloadBlobTimeoutInSeconds);

                ExponentialRetry downloadRetryPolicy =
                    new ExponentialRetry(
                        TimeSpan.FromSeconds(downloadBlobMinBackoffInSeconds),
                        downloadBlobRetryCount);

                BlobRequestOptions blobRequestOptions = new BlobRequestOptions()
                {
                    RetryPolicy          = downloadRetryPolicy,
                    MaximumExecutionTime = TimeSpan.FromSeconds(downloadBlobTimeoutInSeconds),
                };

                BlobTransferOptions options = new BlobTransferOptions();
                options.Concurrency = downloadBlobThreadCount;
                options.SetBlobRequestOptions(BlobRequestOperation.OpenRead, blobRequestOptions);

                using (BlobTransferManager transferManager = new BlobTransferManager(options))
                {
                    transferManager.QueueDownload(
                        this.dataBlob,
                        ms,
                        true,
                        null,
                        delegate(object userData, double speed, double progress)
                    {
                        TraceHelper.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BlobDataContainer] .GetData: progress={0}%", progress);
                    },
                        delegate(object userData, Exception ex)
                    {
                        if (ex != null)
                        {
                            transferException = ex;
                        }
                    },
                        null);

                    transferManager.WaitForCompletion();
                }
            }

            TraceHelper.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BlobDataContainer] .GetData: data transfer done");

            if (transferException == null)
            {
                return(data);
            }

            TraceHelper.TraceSource.TraceEvent(TraceEventType.Error, 0, "[BlobDataContainer] .GetData: received exception={0}", transferException);
            throw TranslateTransferExceptionToDataException(transferException);
        }
Exemplo n.º 12
0
        /// <summary>
        /// Write a data item into data container and flush
        /// </summary>
        /// <param name="data">data content to be written</param>
        public void AddDataAndFlush(DataContent data)
        {
            TraceHelper.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BlobDataContainer] .AddDataAndFlush");

            using (MemoryStream ms = new MemoryStream())
            {
                // dump all data into a memory stream
                data.Dump(ms);

                // create timer that updates "CommonDataLastUpdateTime" metadata peoriodically
                Timer updateMetadataTimer = new Timer(
                    this.MarkBlobAsBeingUploaded,
                    null,
                    TimeSpan.FromMilliseconds(Constant.LastUpdateTimeUpdateIntervalInMilliseconds),
                    TimeSpan.FromMilliseconds(Constant.LastUpdateTimeUpdateIntervalInMilliseconds));

                // write data
                Exception transferException = null;
                try
                {
                    BlobTransferOptions transferOptions = new BlobTransferOptions
                    {
                        Concurrency = Environment.ProcessorCount * 8,
                    };

                    using (BlobTransferManager transferManager = new BlobTransferManager(transferOptions))
                    {
                        transferManager.QueueUpload(
                            this.dataBlob,
                            ms,
                            null,
                            delegate(object userData, double speed, double progress)
                        {
                            TraceHelper.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BlobDataContainer] .AddDataAndFlush: progress={0}%", progress);
                        },
                            delegate(object userData, Exception ex)
                        {
                            if (ex != null)
                            {
                                transferException = ex;
                            }
                        },
                            null);

                        transferManager.WaitForCompletion();
                    }
                }
                finally
                {
                    updateMetadataTimer.Dispose();
                }

                TraceHelper.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BlobDataContainer] .AddDataAndFlush: data transfer done");

                DataException dataException = null;
                if (transferException != null)
                {
                    dataException = TranslateTransferExceptionToDataException(transferException);
                }

                try
                {
                    int    errorCode    = DataErrorCode.Success;
                    string errorMessage = string.Empty;
                    if (dataException != null)
                    {
                        errorCode    = dataException.ErrorCode;
                        errorMessage = dataException.Message;
                    }

                    AzureBlobHelper.MarkBlobAsCompleted(this.dataBlob, errorCode.ToString(), errorMessage);
                }
                catch (StorageException ex)
                {
                    TraceHelper.TraceSource.TraceEvent(
                        TraceEventType.Error,
                        0,
                        "[BlobDataContainer] .AddDataAndFlush: failed to mark blob as completed. blob url={0}. error code={1}, exception={2}",
                        this.dataBlob.Uri.AbsoluteUri,
                        BurstUtility.GetStorageErrorCode(ex),
                        ex);
                }
                catch (Exception ex)
                {
                    TraceHelper.TraceSource.TraceEvent(
                        TraceEventType.Error,
                        0,
                        "[BlobDataContainer] .AddDataAndFlush: failed to mark blob as completed. blob url={0}. Exception={1}",
                        this.dataBlob.Uri.AbsoluteUri,
                        ex);
                }

                if (dataException != null)
                {
                    throw dataException;
                }
            }
        }
Exemplo n.º 13
0
        //*********************************************************************
        ///
        /// <summary>
        ///
        /// </summary>
        /// <param name="config"></param>
        /// <param name="progressCallback"></param>
        /// <param name="speedChangeCallback"></param>
        /// <returns></returns>
        ///
        //*********************************************************************

        //*** NOTE * Network

        public AzureAdminClientLib.Response UploadFileToCloud(BlobTransferInfo config,
                                                              ProgressFileCallbackDelegate progressCallback, SpeedChangeDelegate speedChangeCallback)
        {
            var resp = new AzureAdminClientLib.Response();

            _PFCBD = progressCallback;
            _SCD   = speedChangeCallback;
            _OverwriteDestination = config.OverwriteDestination;
            _SourceFileLength     = config.LocalFile.Length;

            var pivot = config.LocalFile.FullName.LastIndexOf('\\');

            var sourceLocation  = config.LocalFile.FullName.Substring(0, pivot);
            var theFilePatterns = new List <string>(1)
            {
                config.LocalFile.FullName.Substring(pivot + 1)
            };

            var destinationLocation   = config.Url.AbsoluteUri;
            var destinationStorageKey = config.Key;

            var rem = config.LocalFile.Length % PAGEBLOBCHUNCKSIZE;

            Microsoft.WindowsAzure.Storage.Blob.BlobType theUploadBlobType;

            if (0 == rem)
            {
                theUploadBlobType = Microsoft.WindowsAzure.Storage.Blob.BlobType.PageBlob;
            }
            else
            {
                theUploadBlobType = Microsoft.WindowsAzure.Storage.Blob.BlobType.BlockBlob;
            }

            var    ShouldTransferSnapshots        = false;
            var    ShouldExcludeNewer             = false;
            var    ShouldExcludeOlder             = false;
            var    FileIncludedAttributes         = new System.IO.FileAttributes();
            var    FileExcludedAttributes         = new System.IO.FileAttributes();
            var    ShouldOnlyFilesWithArchiveBits = false;
            var    ShouldRecursive    = false;
            var    ShouldFakeTransfer = false;
            object TagData            = resp;

            if (0 == config.TransferThreadCount)
            {
                config.TransferThreadCount = Environment.ProcessorCount * 8;
            }

            ConfigureServicePointManager(config.TransferThreadCount);

            var transferOptions = new BlobTransferOptions
            {
                Concurrency             = GetTransferThreadCount(config.TransferThreadCount),
                OverwritePromptCallback =
                    new BlobTransferOverwritePromptCallback(this.OverwritePromptCallback)
            };

            transferOptions.AppendToClientRequestId(GetAzCopyClientRequestID());
            BlobTransferRecursiveTransferOptions recursiveTransferOptions = null;

            using (var transferManager = new BlobTransferManager(transferOptions))
            {
                try
                {
                    recursiveTransferOptions = new BlobTransferRecursiveTransferOptions
                    {
                        DestinationKey          = destinationStorageKey,
                        TransferSnapshots       = ShouldTransferSnapshots,
                        ExcludeNewer            = ShouldExcludeNewer,
                        ExcludeOlder            = ShouldExcludeOlder,
                        IncludedAttributes      = FileIncludedAttributes,
                        ExcludedAttributes      = FileExcludedAttributes,
                        OnlyFilesWithArchiveBit = ShouldOnlyFilesWithArchiveBits,
                        UploadBlobType          = theUploadBlobType,
                        FilePatterns            = theFilePatterns,
                        Recursive          = ShouldRecursive,
                        FakeTransfer       = ShouldFakeTransfer,
                        MoveFile           = config.DeleteSource,
                        FileTransferStatus = null
                    };

                    transferManager.GlobalCopySpeedUpdated +=
                        new EventHandler <BlobTransferManagerEventArgs>(this.SpeedChangeCallback);
                    transferManager.GlobalDownloadSpeedUpdated +=
                        new EventHandler <BlobTransferManagerEventArgs>(this.SpeedChangeCallback);
                    transferManager.GlobalUploadSpeedUpdated +=
                        new EventHandler <BlobTransferManagerEventArgs>(this.SpeedChangeCallback);
                }
                catch (Exception ex)
                {
                    resp.ResultStatus     = Response.ResultStatusEnum.EXCEPTION;
                    resp.ExceptionMessage = "Exception in UploadFileToCloud() during setup : " +
                                            CmpInterfaceModel.Utilities.UnwindExceptionMessages(ex);
                }

                try
                {
                    transferManager.QueueRecursiveTransfer(sourceLocation, destinationLocation,
                                                           recursiveTransferOptions, StartCallback, ExceptionCallback, StartFileCallback,
                                                           ProgressFileCallback, FinishFileCallback, TagData);
                }
                catch (Exception ex)
                {
                    resp.ResultStatus     = Response.ResultStatusEnum.EXCEPTION;
                    resp.ExceptionMessage = "Exception in UploadFileToCloud() during queue up : " +
                                            CmpInterfaceModel.Utilities.UnwindExceptionMessages(ex);
                }

                transferManager.WaitForCompletion();
                return(resp);
            }
        }