コード例 #1
0
        /// <summary>
        /// This operation will create a new
        /// block blob of arbitrary size by uploading it as indiviually staged
        /// blocks if it's larger than the
        /// <paramref name="transferOptions"/> MaximumTransferLength.
        /// </summary>
        /// <param name="content">
        /// A <see cref="Stream"/> containing the content to upload.
        /// </param>
        /// <param name="blobHttpHeaders">
        /// Optional standard HTTP header properties that can be set for the
        /// block blob.
        /// </param>
        /// <param name="metadata">
        /// Optional custom metadata to set for this block blob.
        /// </param>
        /// <param name="conditions">
        /// Optional <see cref="BlobRequestConditions"/> to add conditions on
        /// the creation of this new block blob.
        /// </param>
        /// <param name="progressHandler">
        /// Optional <see cref="IProgress{Long}"/> to provide
        /// progress updates about data transfers.
        /// </param>
        /// <param name="accessTier">
        /// Optional <see cref="AccessTier"/>
        /// Indicates the tier to be set on the blob.
        /// </param>
        /// <param name="transferOptions">
        /// Optional <see cref="StorageTransferOptions"/> to configure
        /// parallel transfer behavior.
        /// </param>
        /// <param name="async">
        /// </param>
        /// <param name="cancellationToken">
        /// Optional <see cref="CancellationToken"/> to propagate
        /// notifications that the operation should be cancelled.
        /// </param>
        /// <returns>
        /// A <see cref="Response{BlobContentInfo}"/> describing the
        /// state of the updated block blob.
        /// </returns>
        /// <remarks>
        /// A <see cref="RequestFailedException"/> will be thrown if
        /// a failure occurs.
        /// </remarks>
        internal async Task <Response <BlobContentInfo> > StagedUploadAsync(
            Stream content,
            BlobHttpHeaders blobHttpHeaders,
            Metadata metadata,
            BlobRequestConditions conditions,
            IProgress <long> progressHandler,
            AccessTier?accessTier = default,
            StorageTransferOptions transferOptions = default,
            bool async = true,
            CancellationToken cancellationToken = default)
        {
            if (UsingClientSideEncryption)
            {
                // content is now unseekable, so PartitionedUploader will be forced to do a buffered multipart upload
                (content, metadata) = await new BlobClientSideEncryptor(new ClientSideEncryptor(ClientSideEncryption))
                                      .ClientSideEncryptInternal(content, metadata, async, cancellationToken).ConfigureAwait(false);
            }

            var client = new BlockBlobClient(Uri, Pipeline, Version, ClientDiagnostics, CustomerProvidedKey, EncryptionScope);

            PartitionedUploader uploader = new PartitionedUploader(
                client,
                transferOptions,
                operationName: $"{nameof(BlobClient)}.{nameof(Upload)}");

            if (async)
            {
                return(await uploader.UploadAsync(content, blobHttpHeaders, metadata, conditions, progressHandler, accessTier, cancellationToken).ConfigureAwait(false));
            }
            else
            {
                return(uploader.Upload(content, blobHttpHeaders, metadata, conditions, progressHandler, accessTier, cancellationToken));
            }
        }
コード例 #2
0
        /// <summary>
        /// This operation will create a new
        /// block blob of arbitrary size by uploading it as indiviually staged
        /// blocks if it's larger than the
        /// <paramref name="singleUploadThreshold"/>.
        /// </summary>
        /// <param name="content">
        /// A <see cref="Stream"/> containing the content to upload.
        /// </param>
        /// <param name="blobHttpHeaders">
        /// Optional standard HTTP header properties that can be set for the
        /// block blob.
        /// </param>
        /// <param name="metadata">
        /// Optional custom metadata to set for this block blob.
        /// </param>
        /// <param name="conditions">
        /// Optional <see cref="BlobRequestConditions"/> to add conditions on
        /// the creation of this new block blob.
        /// </param>
        /// <param name="progressHandler">
        /// Optional <see cref="IProgress{Long}"/> to provide
        /// progress updates about data transfers.
        /// </param>
        /// <param name="accessTier">
        /// Optional <see cref="AccessTier"/>
        /// Indicates the tier to be set on the blob.
        /// </param>
        /// <param name="singleUploadThreshold">
        /// The maximum size stream that we'll upload as a single block.  The
        /// default value is 256MB.
        /// </param>
        /// <param name="transferOptions">
        /// Optional <see cref="StorageTransferOptions"/> to configure
        /// parallel transfer behavior.
        /// </param>
        /// <param name="async">
        /// </param>
        /// <param name="cancellationToken">
        /// Optional <see cref="CancellationToken"/> to propagate
        /// notifications that the operation should be cancelled.
        /// </param>
        /// <returns>
        /// A <see cref="Response{BlobContentInfo}"/> describing the
        /// state of the updated block blob.
        /// </returns>
        /// <remarks>
        /// A <see cref="RequestFailedException"/> will be thrown if
        /// a failure occurs.
        /// </remarks>
        internal async Task <Response <BlobContentInfo> > StagedUploadAsync(
            Stream content,
            BlobHttpHeaders blobHttpHeaders,
            Metadata metadata,
            BlobRequestConditions conditions,
            IProgress <long> progressHandler,
            AccessTier?accessTier                  = default,
            long?singleUploadThreshold             = default,
            StorageTransferOptions transferOptions = default,
            bool async = true,
            CancellationToken cancellationToken = default)
        {
            var client = new BlockBlobClient(Uri, Pipeline, Version, ClientDiagnostics, CustomerProvidedKey, EncryptionScope);

            singleUploadThreshold ??= client.BlockBlobMaxUploadBlobBytes;
            Debug.Assert(singleUploadThreshold <= client.BlockBlobMaxUploadBlobBytes);

            PartitionedUploader uploader = new PartitionedUploader(
                client,
                transferOptions,
                singleUploadThreshold,
                operationName: $"{nameof(BlobClient)}.{nameof(Upload)}");

            if (async)
            {
                return(await uploader.UploadAsync(content, blobHttpHeaders, metadata, conditions, progressHandler, accessTier, cancellationToken).ConfigureAwait(false));
            }
            else
            {
                return(uploader.Upload(content, blobHttpHeaders, metadata, conditions, progressHandler, accessTier, cancellationToken));
            }
        }
コード例 #3
0
 public BlockBlobWriteStream(
     BlockBlobClient blockBlobClient,
     long bufferSize,
     long position,
     BlobRequestConditions conditions,
     IProgress <long> progressHandler,
     BlobHttpHeaders blobHttpHeaders,
     IDictionary <string, string> metadata,
     IDictionary <string, string> tags
     // TODO #27253
     //UploadTransactionalHashingOptions hashingOptions
     ) : base(
         position,
         bufferSize,
         progressHandler
         //hashingOptions
         )
 {
     ValidateBufferSize(bufferSize);
     _blockBlobClient = blockBlobClient;
     _conditions      = conditions ?? new BlobRequestConditions();
     _blockIds        = new List <string>();
     _blobHttpHeaders = blobHttpHeaders;
     _metadata        = metadata;
     _tags            = tags;
 }
コード例 #4
0
 public BlockBlobWriteStream(
     BlockBlobClient blockBlobClient,
     long bufferSize,
     long position,
     BlobRequestConditions conditions,
     IProgress <long> progressHandler) : base(
         position,
         bufferSize,
         progressHandler)
 {
     ValidateBufferSize(bufferSize);
     _blockBlobClient = blockBlobClient;
     _conditions      = conditions ?? new BlobRequestConditions();
     _blockIds        = new List <string>();
 }
コード例 #5
0
        public PartitionedUploader(
            BlockBlobClient client,
            StorageTransferOptions transferOptions,
            ArrayPool <byte> arrayPool = null,
            string operationName       = null)
        {
            _client    = client;
            _arrayPool = arrayPool ?? ArrayPool <byte> .Shared;

            // Set _maxWorkerCount
            if (transferOptions.MaximumConcurrency.HasValue &&
                transferOptions.MaximumConcurrency > 0)
            {
                _maxWorkerCount = transferOptions.MaximumConcurrency.Value;
            }
            else
            {
                _maxWorkerCount = Constants.Blob.Block.DefaultConcurrentTransfersCount;
            }

            // Set _singleUploadThreshold
            if (transferOptions.InitialTransferLength.HasValue &&
                transferOptions.InitialTransferLength.Value > 0)
            {
                _singleUploadThreshold = Math.Min(transferOptions.InitialTransferLength.Value, Constants.Blob.Block.MaxUploadBytes);
            }
            else
            {
                _singleUploadThreshold = Constants.Blob.Block.MaxUploadBytes;
            }

            // Set _blockSize
            if (transferOptions.MaximumTransferLength.HasValue &&
                transferOptions.MaximumTransferLength > 0)
            {
                _blockSize = Math.Min(
                    Constants.Blob.Block.MaxStageBytes,
                    transferOptions.MaximumTransferLength.Value);
            }

            _operationName = operationName;
        }
コード例 #6
0
 public PartitionedUploader(
     BlockBlobClient client,
     StorageTransferOptions transferOptions,
     long?singleUploadThreshold = null,
     ArrayPool <byte> arrayPool = null)
 {
     _client         = client;
     _arrayPool      = arrayPool ?? ArrayPool <byte> .Shared;
     _maxWorkerCount =
         transferOptions.MaximumConcurrency ??
         Constants.Blob.Block.DefaultConcurrentTransfersCount;
     _singleUploadThreshold = singleUploadThreshold ?? Constants.Blob.Block.MaxUploadBytes;
     _blockSize             = null;
     if (transferOptions.MaximumTransferLength != null)
     {
         _blockSize = Math.Min(
             Constants.Blob.Block.MaxStageBytes,
             transferOptions.MaximumTransferLength.Value);
     }
 }
コード例 #7
0
        /// <summary>
        /// Creates a crypto transform stream to write blob contents to.
        /// </summary>
        /// <param name="blobClient">
        /// BlobClient to open write with.
        /// </param>
        /// <param name="overwrite">
        /// Overwrite parameter to open write.
        /// </param>
        /// <param name="options">
        /// Options parameter to open write.
        /// </param>
        /// <param name="async">
        /// Whether to perform this operation asynchronously.
        /// </param>
        /// <param name="cancellationToken">
        /// Cancellation token.
        /// </param>
        /// <returns>
        /// Content transform write stream and metadata.
        /// </returns>
        public async Task <Stream> ClientSideEncryptionOpenWriteInternal(
            BlockBlobClient blobClient,
            bool overwrite,
            BlockBlobOpenWriteOptions options,
            bool async,
            CancellationToken cancellationToken)
        {
            Stream encryptionWriteStream = await _encryptor.EncryptedOpenWriteInternal(
                async (encryptiondata, funcAsync, funcCancellationToken) =>
            {
                options.Metadata = TransformMetadata(options.Metadata, encryptiondata);

                return(await blobClient.OpenWriteInternal(
                           overwrite,
                           options,
                           funcAsync,
                           funcCancellationToken).ConfigureAwait(false));
            },
                async,
                cancellationToken).ConfigureAwait(false);

            return(encryptionWriteStream);
        }
コード例 #8
0
        /// <summary>
        /// The <see cref="StagedUploadAsync"/> operation will create a new
        /// block blob of arbitrary size by uploading it as indiviually staged
        /// blocks if it's larger than the
        /// <paramref name="singleBlockThreshold"/>.
        /// </summary>
        /// <param name="content">
        /// A <see cref="Stream"/> containing the content to upload.
        /// </param>
        /// <param name="blobHttpHeaders">
        /// Optional standard HTTP header properties that can be set for the
        /// block blob.
        /// </param>
        /// <param name="metadata">
        /// Optional custom metadata to set for this block blob.
        /// </param>
        /// <param name="blobAccessConditions">
        /// Optional <see cref="BlobAccessConditions"/> to add conditions on
        /// the creation of this new block blob.
        /// </param>
        /// <param name="progressHandler">
        /// Optional <see cref="IProgress{StorageProgress}"/> to provide
        /// progress updates about data transfers.
        /// </param>
        /// <param name="singleBlockThreshold">
        /// The maximum size stream that we'll upload as a single block.  The
        /// default value is 256MB.
        /// </param>
        /// <param name="blockSize">
        /// The size of individually staged blocks.  The default value is 4MB.
        /// </param>
        /// <param name="async">
        /// </param>
        /// <param name="cancellationToken">
        /// Optional <see cref="CancellationToken"/> to propagate
        /// notifications that the operation should be cancelled.
        /// </param>
        /// <returns>
        /// A <see cref="Response{BlobContentInfo}"/> describing the
        /// state of the updated block blob.
        /// </returns>
        /// <remarks>
        /// A <see cref="StorageRequestFailedException"/> will be thrown if
        /// a failure occurs.
        /// </remarks>
        internal async Task <Response <BlobContentInfo> > StagedUploadAsync(
            Stream content,
            BlobHttpHeaders?blobHttpHeaders,
            Metadata metadata,
            BlobAccessConditions?blobAccessConditions,
            IProgress <StorageProgress> progressHandler,
            long singleBlockThreshold = BlockBlobClient.BlockBlobMaxUploadBlobBytes,
            int blockSize             = Constants.DefaultBufferSize,
            bool async = true,
            CancellationToken cancellationToken = default)
        {
            Debug.Assert(singleBlockThreshold <= BlockBlobClient.BlockBlobMaxUploadBlobBytes);

            var client     = new BlockBlobClient(this.Uri, this.Pipeline);
            var blockList  = new List <string>();
            var uploadTask = ChunkedUploader.UploadAsync(
                UploadStreamAsync,
                StageBlockAsync,
                CommitBlockListAsync,
                content,
                singleBlockThreshold,
                blockSize,
                async,
                cancellationToken);

            return(async ?
                   await uploadTask.ConfigureAwait(false) :
                   uploadTask.EnsureCompleted());

            // Upload the entire stream
            Task <Response <BlobContentInfo> > UploadStreamAsync(
                Stream content,
                bool async,
                CancellationToken cancellation) =>
            client.UploadInternal(
                content,
                blobHttpHeaders,
                metadata,
                blobAccessConditions,
                progressHandler,
                async,
                cancellationToken);

            // Upload a single chunk of the stream
            Task <Response <BlockInfo> > StageBlockAsync(
                Stream chunk,
                int blockNumber,
                bool async,
                CancellationToken cancellation)
            {
                // Create a new block ID
                var blockId = Constants.BlockNameFormat;

                blockId = String.Format(CultureInfo.InvariantCulture, blockId, blockNumber);
                blockId = Convert.ToBase64String(Encoding.UTF8.GetBytes(blockId));
                blockList.Add(blockId);

                // Upload the block
                return(client.StageBlockInternal(
                           blockId,
                           chunk,
                           null,
                           blobAccessConditions?.LeaseAccessConditions,
                           progressHandler,
                           async,
                           cancellationToken));
            }

            // Commit a series of chunks
            Task <Response <BlobContentInfo> > CommitBlockListAsync(
                bool async,
                CancellationToken cancellation) =>
            client.CommitBlockListInternal(
                blockList,
                blobHttpHeaders,
                metadata,
                blobAccessConditions,
                async,
                cancellationToken);
        }
コード例 #9
0
        /// <summary>
        /// This operation will create a new
        /// block blob of arbitrary size by uploading it as indiviually staged
        /// blocks if it's larger than the
        /// <paramref name="singleBlockThreshold"/>.
        /// </summary>
        /// <param name="file">
        /// A <see cref="FileInfo"/> of the file to upload.
        /// </param>
        /// <param name="blobHttpHeaders">
        /// Optional standard HTTP header properties that can be set for the
        /// block blob.
        /// </param>
        /// <param name="metadata">
        /// Optional custom metadata to set for this block blob.
        /// </param>
        /// <param name="blobAccessConditions">
        /// Optional <see cref="BlobAccessConditions"/> to add conditions on
        /// the creation of this new block blob.
        /// </param>
        /// <param name="progressHandler">
        /// Optional <see cref="IProgress{StorageProgress}"/> to provide
        /// progress updates about data transfers.
        /// </param>
        /// <param name="singleBlockThreshold">
        /// The maximum size stream that we'll upload as a single block.  The
        /// default value is 256MB.
        /// </param>
        /// <param name="parallelTransferOptions">
        /// Optional <see cref="ParallelTransferOptions"/> to configure
        /// parallel transfer behavior.
        /// </param>
        /// <param name="async">
        /// </param>
        /// <param name="cancellationToken">
        /// Optional <see cref="CancellationToken"/> to propagate
        /// notifications that the operation should be cancelled.
        /// </param>
        /// <returns>
        /// A <see cref="Response{BlobContentInfo}"/> describing the
        /// state of the updated block blob.
        /// </returns>
        /// <remarks>
        /// A <see cref="StorageRequestFailedException"/> will be thrown if
        /// a failure occurs.
        /// </remarks>
        internal async Task <Response <BlobContentInfo> > StagedUploadAsync(
            FileInfo file,
            BlobHttpHeaders?blobHttpHeaders,
            Metadata metadata,
            BlobAccessConditions?blobAccessConditions,
            IProgress <StorageProgress> progressHandler,
            long singleBlockThreshold = BlockBlobClient.BlockBlobMaxUploadBlobBytes,
            ParallelTransferOptions parallelTransferOptions = default,
            bool async = true,
            CancellationToken cancellationToken = default)
        {
            Debug.Assert(singleBlockThreshold <= BlockBlobClient.BlockBlobMaxUploadBlobBytes);

            var client     = new BlockBlobClient(this.Uri, this.Pipeline);
            var blockMap   = new ConcurrentDictionary <long, string>();
            var blockName  = 0;
            var uploadTask = PartitionedUploader.UploadAsync(
                UploadStreamAsync,
                StageBlockAsync,
                CommitBlockListAsync,
                threshold => file.Length < threshold,
                memoryPool => new StreamPartitioner(file, memoryPool),
                singleBlockThreshold,
                parallelTransferOptions,
                async,
                cancellationToken);

            return(async ?
                   await uploadTask.ConfigureAwait(false) :
                   uploadTask.EnsureCompleted());

            string GetNewBase64BlockId(long blockOrdinal)
            {
                // Create and record a new block ID, storing the order information
                // (nominally the block's start position in the original stream)

                var newBlockName = Interlocked.Increment(ref blockName);
                var blockId      = Constants.BlockNameFormat;

                blockId = String.Format(CultureInfo.InvariantCulture, blockId, newBlockName);
                blockId = Convert.ToBase64String(Encoding.UTF8.GetBytes(blockId));
                var success = blockMap.TryAdd(blockOrdinal, blockId);

                Debug.Assert(success);

                return(blockId);
            }

            // Upload the entire stream
            async Task <Response <BlobContentInfo> > UploadStreamAsync()
            {
                using (var stream = file.OpenRead())
                {
                    return
                        (await client.UploadInternal(
                             stream,
                             blobHttpHeaders,
                             metadata,
                             blobAccessConditions,
                             progressHandler,
                             async,
                             cancellationToken)
                         .ConfigureAwait(false));
                }
            }

            // Upload a single partition of the stream
            Task <Response <BlockInfo> > StageBlockAsync(
                Stream partition,
                long blockOrdinal,
                bool async,
                CancellationToken cancellation)
            {
                var base64BlockId = GetNewBase64BlockId(blockOrdinal);

                //var bytes = new byte[10];
                //partition.Read(bytes, 0, 10);
                partition.Position = 0;
                //Console.WriteLine($"Commiting partition {blockOrdinal} => {base64BlockId}, {String.Join(" ", bytes)}");

                // Upload the block
                return(client.StageBlockInternal(
                           base64BlockId,
                           partition,
                           null,
                           blobAccessConditions?.LeaseAccessConditions,
                           progressHandler,
                           async,
                           cancellationToken));
            }

            // Commit a series of partitions
            Task <Response <BlobContentInfo> > CommitBlockListAsync(
                bool async,
                CancellationToken cancellation)
            {
                var base64BlockIds = blockMap.OrderBy(kvp => kvp.Key).Select(kvp => kvp.Value).ToArray();

                //Console.WriteLine($"Commiting block list:\n{String.Join("\n", base64BlockIds)}");

                return
                    (client.CommitBlockListInternal(
                         base64BlockIds,
                         blobHttpHeaders,
                         metadata,
                         blobAccessConditions,
                         async,
                         cancellationToken));
            }
        }