/// <summary> /// Dispatches a write operation. /// </summary> /// <param name="asyncResult">The reference to the pending asynchronous request to finish.</param> private void DispatchWrite(StorageAsyncResult <NullType> asyncResult) { if (this.internalBuffer.Length == 0) { if (asyncResult != null) { asyncResult.OnComplete(this.lastException); } return; } MultiBufferMemoryStream bufferToUpload = this.internalBuffer; this.internalBuffer = new MultiBufferMemoryStream(this.Blob.ServiceClient.BufferManager); bufferToUpload.Seek(0, SeekOrigin.Begin); string bufferMD5 = null; if (this.blockMD5 != null) { bufferMD5 = this.blockMD5.ComputeHash(); this.blockMD5.Dispose(); this.blockMD5 = new MD5Wrapper(); } if (this.blockBlob != null) { string blockId = this.GetCurrentBlockId(); this.blockList.Add(blockId); this.WriteBlock(bufferToUpload, blockId, bufferMD5, asyncResult); } else if (this.pageBlob != null) { if ((bufferToUpload.Length % Constants.PageSize) != 0) { this.lastException = new IOException(SR.InvalidPageSize); throw this.lastException; } long offset = this.currentBlobOffset; this.currentBlobOffset += bufferToUpload.Length; this.WritePages(bufferToUpload, offset, bufferMD5, asyncResult); } else { long offset = this.currentBlobOffset; this.currentBlobOffset += bufferToUpload.Length; // We cannot differentiate between max size condition failing only in the retry versus failing in the first attempt and retry. // So we will eliminate the latter and handle the former in the append operation callback. if (this.accessCondition.IfMaxSizeLessThanOrEqual.HasValue && this.currentBlobOffset > this.accessCondition.IfMaxSizeLessThanOrEqual.Value) { this.lastException = new IOException(SR.InvalidBlockSize); throw this.lastException; } this.WriteAppendBlock(bufferToUpload, offset, bufferMD5, asyncResult); } }
private RESTCommand <NullType> SetServicePropertiesImpl(ServiceProperties properties, TableRequestOptions requestOptions) { MultiBufferMemoryStream memoryStream = new MultiBufferMemoryStream(null /* bufferManager */, (int)(1 * Constants.KB)); try { properties.WriteServiceProperties(memoryStream); } catch (InvalidOperationException invalidOpException) { throw new ArgumentException(invalidOpException.Message, "properties"); } RESTCommand <NullType> retCmd = new RESTCommand <NullType>(this.Credentials, this.StorageUri); requestOptions.ApplyToStorageCommand(retCmd); retCmd.BuildRequest = (cmd, uri, builder, cnt, serverTimeout, ctx) => TableHttpRequestMessageFactory.SetServiceProperties(uri, serverTimeout, cnt, ctx, this.GetCanonicalizer(), this.Credentials); retCmd.BuildContent = (cmd, ctx) => HttpContentFactory.BuildContentFromStream(memoryStream, 0, memoryStream.Length, null /* md5 */, cmd, ctx); retCmd.StreamToDispose = memoryStream; retCmd.PreProcessResponse = (cmd, resp, ex, ctx) => HttpResponseParsers.ProcessExpectedStatusCodeNoException(HttpStatusCode.Accepted, resp, null /* retVal */, cmd, ex); requestOptions.ApplyToStorageCommand(retCmd); return(retCmd); }
/// <summary> /// Initializes a new instance of the FileWriteStreamBase class for a file. /// </summary> /// <param name="file">File reference to write to.</param> /// <param name="fileSize">Size of the file.</param> /// <param name="createNew">Use <c>true</c> if the file is newly created, <c>false</c> otherwise.</param> /// <param name="accessCondition">An <see cref="AccessCondition"/> object that represents the access conditions for the file. If <c>null</c>, no condition is used.</param> /// <param name="options">An <see cref="FileRequestOptions"/> object that specifies additional options for the request.</param> /// <param name="operationContext">An <see cref="OperationContext"/> object for tracking the current operation.</param> protected FileWriteStreamBase(CloudFile file, long fileSize, bool createNew, AccessCondition accessCondition, FileRequestOptions options, OperationContext operationContext) : base() { this.internalBuffer = new MultiBufferMemoryStream(file.ServiceClient.BufferManager); this.currentOffset = 0; this.accessCondition = accessCondition; this.options = options; this.operationContext = operationContext; this.noPendingWritesEvent = new CounterEvent(); this.fileMD5 = this.options.StoreFileContentMD5.Value ? new MD5Wrapper() : null; this.rangeMD5 = this.options.UseTransactionalMD5.Value ? new MD5Wrapper() : null; #if !(NETCORE || WINDOWS_RT) this.parallelOperationSemaphoreAsync = new AsyncSemaphoreAsync(options.ParallelOperationThreadCount.Value); #else this.parallelOperationSemaphore = new AsyncSemaphore(options.ParallelOperationThreadCount.Value); #endif this.lastException = null; this.committed = false; this.disposed = false; this.currentFileOffset = 0; this.file = file; this.fileSize = fileSize; this.streamWriteSizeInBytes = file.StreamWriteSizeInBytes; this.newFile = createNew; }
private static void EndGetRequestStream <T>(IAsyncResult getRequestStreamResult) { ExecutionState <T> executionState = (ExecutionState <T>)getRequestStreamResult.AsyncState; executionState.CurrentOperation = ExecutorOperation.EndGetRequestStream; try { executionState.UpdateCompletedSynchronously(getRequestStreamResult.CompletedSynchronously); executionState.ReqStream = executionState.Req.EndGetRequestStream(getRequestStreamResult); executionState.CurrentOperation = ExecutorOperation.BeginUploadRequest; Logger.LogInformational(executionState.OperationContext, SR.TraceUpload); MultiBufferMemoryStream multiBufferMemoryStream = executionState.RestCMD.SendStream as MultiBufferMemoryStream; if (multiBufferMemoryStream != null && !executionState.RestCMD.SendStreamLength.HasValue) { multiBufferMemoryStream.BeginFastCopyTo(executionState.ReqStream, executionState.OperationExpiryTime, EndFastCopyTo <T>, executionState); } else { // don't calculate md5 here as we should have already set this for auth purposes executionState.RestCMD.SendStream.WriteToAsync(executionState.ReqStream, executionState.RestCMD.SendStreamLength, null /* maxLength */, false, executionState, null /* streamCopyState */, EndSendStreamCopy); } } catch (Exception ex) { Logger.LogWarning(executionState.OperationContext, SR.TraceUploadError, ex.Message); executionState.ExceptionRef = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError); Executor.EndOperation(executionState); } }
/// <summary> /// Initializes a new instance of the <see cref="FileReadStreamBase"/> class. /// </summary> /// <param name="file">File reference to read from</param> /// <param name="accessCondition">An <see cref="AccessCondition"/> object that represents the access conditions for the file. If <c>null</c>, no condition is used.</param> /// <param name="options">An <see cref="FileRequestOptions"/> object that specifies additional options for the request.</param> /// <param name="operationContext">An <see cref="OperationContext"/> object for tracking the current operation.</param> protected FileReadStreamBase(CloudFile file, AccessCondition accessCondition, FileRequestOptions options, OperationContext operationContext) { if (options.ChecksumOptions.UseTransactionalMD5.Value) { CommonUtility.AssertInBounds("StreamMinimumReadSizeInBytes", file.StreamMinimumReadSizeInBytes, 1, Constants.MaxRangeGetContentMD5Size); } if (options.ChecksumOptions.UseTransactionalCRC64.Value) { CommonUtility.AssertInBounds("StreamMinimumReadSizeInBytes", file.StreamMinimumReadSizeInBytes, 1, Constants.MaxRangeGetContentCRC64Size); } this.file = file; this.fileProperties = new FileProperties(file.Properties); this.currentOffset = 0; this.streamMinimumReadSizeInBytes = this.file.StreamMinimumReadSizeInBytes; this.internalBuffer = new MultiBufferMemoryStream(file.ServiceClient.BufferManager); this.accessCondition = accessCondition; this.options = options; this.operationContext = operationContext; this.fileChecksum = new ChecksumWrapper( calcMd5: !(this.options.ChecksumOptions.DisableContentMD5Validation.Value || string.IsNullOrEmpty(this.fileProperties.ContentChecksum.MD5)), calcCrc64: !(this.options.ChecksumOptions.DisableContentCRC64Validation.Value || string.IsNullOrEmpty(this.fileProperties.ContentChecksum.CRC64)) ); this.lastException = null; }
/// <summary> /// Releases the file resources used by the Stream. /// </summary> /// <param name="disposing">true to release both managed and unmanaged resources; false to release only unmanaged resources.</param> protected override void Dispose(bool disposing) { if (disposing) { if (this.fileChecksum != null) { this.fileChecksum.Dispose(); this.fileChecksum = null; } if (this.rangeChecksum != null) { this.rangeChecksum.Dispose(); this.rangeChecksum = null; } if (this.internalBuffer != null) { this.internalBuffer.Dispose(); this.internalBuffer = null; } } base.Dispose(disposing); }
/// <summary> /// Releases the file resources used by the Stream. /// </summary> /// <param name="disposing">true to release both managed and unmanaged resources; false to release only unmanaged resources.</param> protected override void Dispose(bool disposing) { if (disposing) { if (this.fileMD5 != null) { this.fileMD5.Dispose(); this.fileMD5 = null; } if (this.rangeMD5 != null) { this.rangeMD5.Dispose(); this.rangeMD5 = null; } if (this.internalBuffer != null) { this.internalBuffer.Dispose(); this.internalBuffer = null; } if (this.noPendingWritesEvent != null) { this.noPendingWritesEvent.Dispose(); this.noPendingWritesEvent = null; } } base.Dispose(disposing); }
/// <summary> /// Asynchronously dispatches a write operation. /// </summary> /// <returns>A task that represents the asynchronous write operation.</returns> private async Task DispatchWriteAsync() { if (this.internalBuffer.Length == 0) { return; } MultiBufferMemoryStream bufferToUpload = this.internalBuffer; this.internalBuffer = new MultiBufferMemoryStream(this.file.ServiceClient.BufferManager); bufferToUpload.Seek(0, SeekOrigin.Begin); string bufferMD5 = null; if (this.rangeMD5 != null) { bufferMD5 = this.rangeMD5.ComputeHash(); this.rangeMD5.Dispose(); this.rangeMD5 = new MD5Wrapper(); } long offset = this.currentFileOffset; this.currentFileOffset += bufferToUpload.Length; await this.WriteRangeAsync(bufferToUpload, offset, bufferMD5).ConfigureAwait(false); }
/// <summary> /// Asynchronously dispatches a write operation. /// </summary> /// <returns>A task that represents the asynchronous write operation.</returns> private async Task DispatchWriteAsync(TaskCompletionSource <bool> continuetcs, CancellationToken token) { if (this.internalBuffer.Length == 0) { if (continuetcs != null) { Task.Run(() => continuetcs.TrySetResult(true)); } return; } MultiBufferMemoryStream bufferToUpload = this.internalBuffer; this.internalBuffer = new MultiBufferMemoryStream(this.file.ServiceClient.BufferManager); bufferToUpload.Seek(0, SeekOrigin.Begin); string bufferMD5 = null; if (this.rangeMD5 != null) { bufferMD5 = this.rangeMD5.ComputeHash(); this.rangeMD5.Dispose(); this.rangeMD5 = new MD5Wrapper(); } long offset = this.currentFileOffset; this.currentFileOffset += bufferToUpload.Length; await this.WriteRangeAsync(continuetcs, bufferToUpload, offset, bufferMD5, token).ConfigureAwait(false); }
private RESTCommand <NullType> SetServicePropertiesImpl(ServiceProperties properties, TableRequestOptions requestOptions) { MultiBufferMemoryStream str = new MultiBufferMemoryStream(null /* bufferManager */, (int)(1 * Constants.KB)); try { properties.WriteServiceProperties(str); } catch (InvalidOperationException invalidOpException) { throw new ArgumentException(invalidOpException.Message, "properties"); } str.Seek(0, SeekOrigin.Begin); RESTCommand <NullType> retCmd = new RESTCommand <NullType>(this.Credentials, this.StorageUri); retCmd.SendStream = str; retCmd.BuildRequestDelegate = TableHttpWebRequestFactory.SetServiceProperties; retCmd.RecoveryAction = RecoveryActions.RewindStream; retCmd.SignRequest = this.AuthenticationHandler.SignRequest; retCmd.ParseError = StorageExtendedErrorInformation.ReadFromStreamUsingODataLib; retCmd.PreProcessResponse = (cmd, resp, ex, ctx) => HttpResponseParsers.ProcessExpectedStatusCodeNoException(HttpStatusCode.Accepted, resp, NullType.Value, cmd, ex); requestOptions.ApplyToStorageCommand(retCmd); return(retCmd); }
/// <summary> /// Dispatches a write operation. /// </summary> /// <param name="asyncResult">The reference to the pending asynchronous request to finish.</param> private void DispatchWrite(StorageAsyncResult <NullType> asyncResult) { if (this.internalBuffer.Length == 0) { if (asyncResult != null) { asyncResult.OnComplete(this.lastException); } return; } MultiBufferMemoryStream bufferToUpload = this.internalBuffer; this.internalBuffer = new MultiBufferMemoryStream(this.file.ServiceClient.BufferManager); bufferToUpload.Seek(0, SeekOrigin.Begin); string bufferMD5 = null; if (this.rangeMD5 != null) { bufferMD5 = this.rangeMD5.ComputeHash(); this.rangeMD5.Dispose(); this.rangeMD5 = new MD5Wrapper(); } long offset = this.currentFileOffset; this.currentFileOffset += bufferToUpload.Length; this.WriteRange(bufferToUpload, offset, bufferMD5, asyncResult); }
private RESTCommand <NullType> SetServicePropertiesImpl(ServiceProperties properties, QueueRequestOptions requestOptions) { MultiBufferMemoryStream memoryStream = new MultiBufferMemoryStream(null /* bufferManager */, (int)(1 * Constants.KB)); try { properties.WriteServiceProperties(memoryStream); } catch (InvalidOperationException invalidOpException) { throw new ArgumentException(invalidOpException.Message, "properties"); } RESTCommand <NullType> retCmd = new RESTCommand <NullType>(this.Credentials, this.BaseUri); retCmd.ApplyRequestOptions(requestOptions); retCmd.BuildRequest = (cmd, cnt, ctx) => QueueHttpRequestMessageFactory.SetServiceProperties(cmd.Uri, cmd.ServerTimeoutInSeconds, cnt, ctx); retCmd.BuildContent = (cmd, ctx) => HttpContentFactory.BuildContentFromStream(memoryStream, 0, memoryStream.Length, null /* md5 */, cmd, ctx); retCmd.Handler = this.AuthenticationHandler; retCmd.BuildClient = HttpClientFactory.BuildHttpClient; retCmd.PreProcessResponse = (cmd, resp, ex, ctx) => HttpResponseParsers.ProcessExpectedStatusCodeNoException(System.Net.HttpStatusCode.Accepted, resp, null /* retVal */, cmd, ex); retCmd.ApplyRequestOptions(requestOptions); return(retCmd); }
/// <summary> /// Releases the blob resources used by the Stream. /// </summary> /// <param name="disposing">true to release both managed and unmanaged resources; false to release only unmanaged resources.</param> protected override void Dispose(bool disposing) { if (disposing) { if (this.blobMD5 != null) { this.blobMD5.Dispose(); this.blobMD5 = null; } if (this.blockMD5 != null) { this.blockMD5.Dispose(); this.blockMD5 = null; } if (this.internalBuffer != null) { this.internalBuffer.Dispose(); this.internalBuffer = null; } } base.Dispose(disposing); }
private async Task DispatchWriteAsync(TaskCompletionSource <bool> continuetcs, CancellationToken token) { if (this.internalBuffer.Length == 0) { if (continuetcs != null) { Task.Run(() => continuetcs.TrySetResult(true)); } return; } MultiBufferMemoryStream bufferToUpload = this.internalBuffer; this.internalBuffer = new MultiBufferMemoryStream(this.Blob.ServiceClient.BufferManager); bufferToUpload.Seek(0, SeekOrigin.Begin); string bufferMD5 = null; if (this.blockMD5 != null) { bufferMD5 = this.blockMD5.ComputeHash(); this.blockMD5.Dispose(); this.blockMD5 = new MD5Wrapper(); } if (this.blockBlob != null) { string blockId = this.GetCurrentBlockId(); this.blockList.Add(blockId); await this.WriteBlockAsync(continuetcs, bufferToUpload, blockId, bufferMD5, token).ConfigureAwait(false); } else if (this.pageBlob != null) { if ((bufferToUpload.Length % Constants.PageSize) != 0) { this.lastException = new IOException(SR.InvalidPageSize); throw this.lastException; } long offset = this.currentBlobOffset; this.currentBlobOffset += bufferToUpload.Length; await this.WritePagesAsync(continuetcs, bufferToUpload, offset, bufferMD5, token).ConfigureAwait(false); } else { long offset = this.currentBlobOffset; this.currentBlobOffset += bufferToUpload.Length; // We cannot differentiate between max size condition failing only in the retry versus failing in the first attempt and retry. // So we will eliminate the latter and handle the former in the append operation callback. if (this.accessCondition.IfMaxSizeLessThanOrEqual.HasValue && this.currentBlobOffset > this.accessCondition.IfMaxSizeLessThanOrEqual.Value) { this.lastException = new IOException(SR.InvalidBlockSize); throw this.lastException; } await this.WriteAppendBlockAsync(continuetcs, bufferToUpload, offset, bufferMD5, token).ConfigureAwait(false); } }
/// <summary> /// Stores a new entity in the configured blob container. /// </summary> /// <param name="changes">The changes to write to storage.</param> /// <param name="cancellationToken">A cancellation token that can be used by other objects /// or threads to receive notice of cancellation.</param> /// <returns>A task that represents the work queued to execute.</returns> public async Task WriteAsync(IDictionary <string, object> changes, CancellationToken cancellationToken = default(CancellationToken)) { if (changes == null) { throw new ArgumentNullException(nameof(changes)); } var blobClient = _storageAccount.CreateCloudBlobClient(); var blobContainer = blobClient.GetContainerReference(_containerName); // this should only happen once - assuming this is a singleton if (Interlocked.CompareExchange(ref _checkforContainerExistance, 0, 1) == 1) { await blobContainer.CreateIfNotExistsAsync(cancellationToken).ConfigureAwait(false); } var blobRequestOptions = new BlobRequestOptions(); var operationContext = new OperationContext(); foreach (var keyValuePair in changes) { var newValue = keyValuePair.Value; var storeItem = newValue as IStoreItem; // "*" eTag in IStoreItem converts to null condition for AccessCondition var accessCondition = storeItem?.ETag != "*" ? AccessCondition.GenerateIfMatchCondition(storeItem?.ETag) : AccessCondition.GenerateEmptyCondition(); var blobName = GetBlobName(keyValuePair.Key); var blobReference = blobContainer.GetBlockBlobReference(blobName); try { using (var memoryStream = new MultiBufferMemoryStream(blobReference.ServiceClient.BufferManager)) using (var streamWriter = new StreamWriter(memoryStream)) { _jsonSerializer.Serialize(streamWriter, newValue); await streamWriter.FlushAsync().ConfigureAwait(false); memoryStream.Seek(0, SeekOrigin.Begin); await blobReference.UploadFromStreamAsync(memoryStream, accessCondition, blobRequestOptions, operationContext, cancellationToken).ConfigureAwait(false); } } catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.BadRequest && ex.RequestInformation.ErrorCode == BlobErrorCodeStrings.InvalidBlockList) { throw new InvalidOperationException( $"An error ocurred while trying to write an object. The underlying '{BlobErrorCodeStrings.InvalidBlockList}' error is commonly caused due to concurrently uploading an object larger than 128MB in size.", ex); } } }
internal static StorageRequestMessage BuildRequestForTableOperation <T>(RESTCommand <T> cmd, Uri uri, UriQueryBuilder builder, int?timeout, TableOperation operation, CloudTableClient client, HttpContent content, OperationContext ctx, TablePayloadFormat payloadFormat, ICanonicalizer canonicalizer, StorageCredentials credentials) { StorageRequestMessage msg = BuildRequestCore(uri, builder, operation.HttpMethod, timeout, content, ctx, canonicalizer, credentials); // Set Accept and Content-Type based on the payload format. SetAcceptHeaderForHttpWebRequest(msg, payloadFormat); Logger.LogInformational(ctx, SR.PayloadFormat, payloadFormat); msg.Headers.Add(Constants.HeaderConstants.DataServiceVersion, Constants.HeaderConstants.DataServiceVersionValue); if (operation.OperationType == TableOperationType.InsertOrMerge || operation.OperationType == TableOperationType.Merge) { // post tunnelling msg.Headers.Add(Constants.HeaderConstants.PostTunnelling, "MERGE"); } // etag if (operation.OperationType == TableOperationType.Delete || operation.OperationType == TableOperationType.Replace || operation.OperationType == TableOperationType.Merge) { msg.Headers.Add(Constants.HeaderConstants.IfMatch, operation.ETag); } // Prefer header if (operation.OperationType == TableOperationType.Insert) { msg.Headers.Add(Constants.HeaderConstants.Prefer, operation.EchoContent ? Constants.HeaderConstants.PreferReturnContent : Constants.HeaderConstants.PreferReturnNoContent); } if (operation.OperationType == TableOperationType.Insert || operation.OperationType == TableOperationType.Merge || operation.OperationType == TableOperationType.InsertOrMerge || operation.OperationType == TableOperationType.InsertOrReplace || operation.OperationType == TableOperationType.Replace) { MultiBufferMemoryStream ms = new MultiBufferMemoryStream(client.BufferManager); using (JsonTextWriter jsonWriter = new JsonTextWriter(new StreamWriter(new NonCloseableStream(ms)))) { WriteEntityContent(operation, ctx, jsonWriter); } ms.Seek(0, SeekOrigin.Begin); msg.Content = new StreamContent(ms); msg.Content.Headers.ContentLength = ms.Length; if (!operation.HttpMethod.Equals("HEAD") && !operation.HttpMethod.Equals("GET")) { SetContentTypeForHttpWebRequest(msg); } return(msg); } return(msg); }
/// <summary> /// Initializes a new instance of the BlobWriteStreamBase class. /// </summary> /// <param name="serviceClient">The service client.</param> /// <param name="accessCondition">An <see cref="AccessCondition"/> object that represents the condition that must be met in order for the request to proceed. If <c>null</c>, no condition is used.</param> /// <param name="options">A <see cref="BlobRequestOptions"/> object that specifies additional options for the request.</param> /// <param name="operationContext">An <see cref="OperationContext"/> object that represents the context for the current operation.</param> private BlobWriteStreamBase(CloudBlobClient serviceClient, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext) : base() { this.internalBuffer = new MultiBufferMemoryStream(serviceClient.BufferManager); this.accessCondition = accessCondition; this.currentOffset = 0; this.options = options; this.operationContext = operationContext; this.noPendingWritesEvent = new CounterEvent(); this.blobMD5 = this.options.StoreBlobContentMD5.Value ? new MD5Wrapper() : null; this.blockMD5 = this.options.UseTransactionalMD5.Value ? new MD5Wrapper() : null; this.parallelOperationSemaphore = new AsyncSemaphore(options.ParallelOperationThreadCount.Value); this.lastException = null; this.committed = false; this.disposed = false; }
/// <summary> /// Stores a new entity in the configured blob container. /// </summary> /// <param name="changes">The Dictionary of changes that are to be made.</param> /// <param name="cancellationToken">A cancellation token that can be used by other objects /// or threads to receive notice of cancellation.</param> /// <returns>A <see cref="Task"/>A task that represents the work queued to execute.</returns> public async Task WriteAsync(IDictionary <string, object> changes, CancellationToken cancellationToken = default(CancellationToken)) { if (changes == null) { throw new ArgumentNullException(nameof(changes)); } var blobContainer = await GetBlobContainer().ConfigureAwait(false); var blobRequestOptions = new BlobRequestOptions(); var operationContext = new OperationContext(); await Task.WhenAll( changes.Select(async(keyValuePair) => { var newValue = keyValuePair.Value; var storeItem = newValue as IStoreItem; // "*" eTag in IStoreItem converts to null condition for AccessCondition var accessCondition = storeItem?.ETag == "*" ? AccessCondition.GenerateEmptyCondition() : AccessCondition.GenerateIfMatchCondition(storeItem?.ETag); var blobName = GetBlobName(keyValuePair.Key); var blobReference = blobContainer.GetBlockBlobReference(blobName); try { using (var memoryStream = new MultiBufferMemoryStream(blobReference.ServiceClient.BufferManager)) using (var streamWriter = new StreamWriter(memoryStream)) { JsonSerializer.Serialize(streamWriter, newValue); streamWriter.Flush(); memoryStream.Seek(0, SeekOrigin.Begin); await blobReference.UploadFromStreamAsync(memoryStream, accessCondition, blobRequestOptions, operationContext).ConfigureAwait(false); } } catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.BadRequest && ex.RequestInformation.ErrorCode == BlobErrorCodeStrings.InvalidBlockList) { throw new Exception( $"An error ocurred while trying to write an object. The underlying '{BlobErrorCodeStrings.InvalidBlockList}' error is commonly caused due to concurrently uploading an object larger than 128MB in size.", ex); } })).ConfigureAwait(false); }
/// <summary> /// Asynchronously dispatches a write operation. /// </summary> /// <returns>A task that represents the asynchronous write operation.</returns> private async Task DispatchWriteAsync(TaskCompletionSource <bool> continuetcs, CancellationToken token) { if (this.internalBuffer.Length == 0) { if (continuetcs != null) { Task.Run(() => continuetcs.TrySetResult(true)); } return; } // bufferToUpload needs to be disposed, or we will leak memory // // Unfortunately, because of the async nature of the work, we cannot safely // put this in a using block, so the Write*Async methods must handle disposal. MultiBufferMemoryStream bufferToUpload = this.internalBuffer; this.internalBuffer = new MultiBufferMemoryStream(this.file.ServiceClient.BufferManager); bufferToUpload.Seek(0, SeekOrigin.Begin); Checksum bufferChecksum = Checksum.None; if (this.rangeChecksum != null) { bool computeCRC64 = false; bool computeMD5 = false; if (this.rangeChecksum.MD5 != null) { bufferChecksum.MD5 = this.rangeChecksum.MD5.ComputeHash(); computeMD5 = true; } if (this.rangeChecksum.CRC64 != null) { bufferChecksum.CRC64 = this.rangeChecksum.CRC64.ComputeHash(); computeCRC64 = true; } this.rangeChecksum.Dispose(); this.rangeChecksum = new ChecksumWrapper(computeMD5, computeCRC64); } long offset = this.currentFileOffset; this.currentFileOffset += bufferToUpload.Length; await this.WriteRangeAsync(continuetcs, bufferToUpload, offset, bufferChecksum, token).ConfigureAwait(false); }
/// <summary> /// Dispatches a write operation. /// </summary> /// <param name="asyncResult">The reference to the pending asynchronous request to finish.</param> private void DispatchWrite(StorageAsyncResult <NullType> asyncResult) { if (this.internalBuffer.Length == 0) { if (asyncResult != null) { asyncResult.OnComplete(this.lastException); } return; } MultiBufferMemoryStream bufferToUpload = this.internalBuffer; this.internalBuffer = new MultiBufferMemoryStream(this.Blob.ServiceClient.BufferManager); bufferToUpload.Seek(0, SeekOrigin.Begin); string bufferMD5 = null; if (this.blockMD5 != null) { bufferMD5 = this.blockMD5.ComputeHash(); this.blockMD5.Dispose(); this.blockMD5 = new MD5Wrapper(); } if (this.blockBlob != null) { string blockId = this.GetCurrentBlockId(); this.blockList.Add(blockId); this.WriteBlock(bufferToUpload, blockId, bufferMD5, asyncResult); } else { if ((bufferToUpload.Length % Constants.PageSize) != 0) { this.lastException = new IOException(SR.InvalidPageSize); throw this.lastException; } long offset = this.currentPageOffset; this.currentPageOffset += bufferToUpload.Length; this.WritePages(bufferToUpload, offset, bufferMD5, asyncResult); } }
/// <summary> /// Initializes a new instance of the BlobReadStreamBase class. /// </summary> /// <param name="blob">Blob reference to read from</param> /// <param name="accessCondition">An <see cref="AccessCondition"/> object that represents the condition that must be met in order for the request to proceed. If <c>null</c>, no condition is used.</param> /// <param name="options">A <see cref="BlobRequestOptions"/> object that specifies additional options for the request.</param> /// <param name="operationContext">An <see cref="OperationContext"/> object that represents the context for the current operation.</param> protected BlobReadStreamBase(CloudBlob blob, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext) { if (options.UseTransactionalMD5.Value) { CommonUtility.AssertInBounds("StreamMinimumReadSizeInBytes", blob.StreamMinimumReadSizeInBytes, 1, Constants.MaxRangeGetContentMD5Size); } this.blob = blob; this.blobProperties = new BlobProperties(blob.Properties); this.currentOffset = 0; this.streamMinimumReadSizeInBytes = this.blob.StreamMinimumReadSizeInBytes; this.internalBuffer = new MultiBufferMemoryStream(blob.ServiceClient.BufferManager); this.accessCondition = accessCondition; this.options = options; this.operationContext = operationContext; this.blobMD5 = (this.options.DisableContentMD5Validation.Value || string.IsNullOrEmpty(this.blobProperties.ContentMD5)) ? null : new MD5Wrapper(); this.lastException = null; }
/// <summary> /// Initializes a new instance of the BlobWriteStreamBase class. /// </summary> /// <param name="serviceClient">The service client.</param> /// <param name="accessCondition">An <see cref="AccessCondition"/> object that represents the condition that must be met in order for the request to proceed. If <c>null</c>, no condition is used.</param> /// <param name="options">A <see cref="BlobRequestOptions"/> object that specifies additional options for the request.</param> /// <param name="operationContext">An <see cref="OperationContext"/> object that represents the context for the current operation.</param> private BlobWriteStreamBase(CloudBlobClient serviceClient, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext) : base() { this.internalBuffer = new MultiBufferMemoryStream(serviceClient.BufferManager); this.accessCondition = accessCondition; this.currentOffset = 0; this.options = options; this.operationContext = operationContext; this.noPendingWritesEvent = new CounterEventAsync(); this.blobChecksum = new ChecksumWrapper(this.options.ChecksumOptions.StoreContentMD5.Value, this.options.ChecksumOptions.StoreContentCRC64.Value); this.blockChecksum = new ChecksumWrapper(this.options.ChecksumOptions.UseTransactionalMD5.Value, this.options.ChecksumOptions.UseTransactionalCRC64.Value); #if WINDOWS_DESKTOP this.parallelOperationSemaphoreAsync = new AsyncSemaphoreAsync(options.ParallelOperationThreadCount.Value); #else this.parallelOperationSemaphore = new AsyncSemaphore(options.ParallelOperationThreadCount.Value); #endif this.lastException = null; this.committed = false; this.disposed = false; }
private static void EndFastCopyTo <T>(IAsyncResult fastCopyToResult) { ExecutionState <T> executionState = (ExecutionState <T>)fastCopyToResult.AsyncState; try { executionState.UpdateCompletedSynchronously(fastCopyToResult.CompletedSynchronously); MultiBufferMemoryStream multiBufferMemoryStream = (MultiBufferMemoryStream)executionState.RestCMD.SendStream; multiBufferMemoryStream.EndFastCopyTo(fastCopyToResult); Executor.EndSendStreamCopy(executionState); } catch (Exception ex) { Logger.LogWarning(executionState.OperationContext, SR.TraceUploadError, ex.Message); executionState.ExceptionRef = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError); Executor.EndOperation(executionState); } }
/// <summary> /// Implementation for the SetPermissions method. /// </summary> /// <param name="acl">The permissions to set.</param> /// <param name="accessCondition">An <see cref="AccessCondition"/> object that represents the access conditions for the share. If <c>null</c>, no condition is used.</param> /// <param name="options">A <see cref="FileRequestOptions"/> object that specifies additional options for the request.</param> /// <returns>A <see cref="RESTCommand"/> that sets the permissions.</returns> private RESTCommand <NullType> SetPermissionsImpl(FileSharePermissions acl, AccessCondition accessCondition, FileRequestOptions options) { MultiBufferMemoryStream memoryStream = new MultiBufferMemoryStream(null /* bufferManager */, (int)(1 * Constants.KB)); FileRequest.WriteSharedAccessIdentifiers(acl.SharedAccessPolicies, memoryStream); RESTCommand <NullType> putCmd = new RESTCommand <NullType>(this.ServiceClient.Credentials, this.StorageUri); options.ApplyToStorageCommand(putCmd); putCmd.BuildRequest = (cmd, uri, builder, cnt, serverTimeout, ctx) => ShareHttpRequestMessageFactory.SetAcl(uri, serverTimeout, FileSharePublicAccessType.Off, accessCondition, cnt, ctx, this.ServiceClient.GetCanonicalizer(), this.ServiceClient.Credentials); putCmd.BuildContent = (cmd, ctx) => HttpContentFactory.BuildContentFromStream(memoryStream, 0, memoryStream.Length, null /* md5 */, cmd, ctx); putCmd.StreamToDispose = memoryStream; putCmd.PreProcessResponse = (cmd, resp, ex, ctx) => { HttpResponseParsers.ProcessExpectedStatusCodeNoException(HttpStatusCode.OK, resp, NullType.Value, cmd, ex); this.UpdateETagAndLastModified(resp); return(NullType.Value); }; return(putCmd); }
/// <summary> /// Implementation for the SetPermissions method. /// </summary> /// <param name="acl">The permissions to set.</param> /// <param name="requestOptions">A <see cref="TableRequestOptions"/> object that specifies execution options, such as retry policy and timeout settings, for the operation.</param> /// <returns>A <see cref="RESTCommand"/> that sets the permissions.</returns> private RESTCommand <NullType> SetPermissionsImpl(TablePermissions acl, TableRequestOptions requestOptions) { MultiBufferMemoryStream memoryStream = new MultiBufferMemoryStream(null /* bufferManager */, (int)(1 * Constants.KB)); TableRequest.WriteSharedAccessIdentifiers(acl.SharedAccessPolicies, memoryStream); RESTCommand <NullType> putCmd = new RESTCommand <NullType>(this.ServiceClient.Credentials, this.Uri); putCmd.ApplyRequestOptions(requestOptions); putCmd.Handler = this.ServiceClient.AuthenticationHandler; putCmd.BuildClient = HttpClientFactory.BuildHttpClient; putCmd.BuildRequest = (cmd, cnt, ctx) => TableHttpRequestMessageFactory.SetAcl(cmd.Uri, cmd.ServerTimeoutInSeconds, cnt, ctx); putCmd.BuildContent = (cmd, ctx) => HttpContentFactory.BuildContentFromStream(memoryStream, 0, memoryStream.Length, null /* md5 */, cmd, ctx); putCmd.PreProcessResponse = (cmd, resp, ex, ctx) => { HttpResponseParsers.ProcessExpectedStatusCodeNoException(HttpStatusCode.NoContent, resp, NullType.Value, cmd, ex); return(NullType.Value); }; return(putCmd); }
private async Task DispatchWriteAsync(TaskCompletionSource <bool> continuetcs, CancellationToken token) { if (this.internalBuffer.Length == 0) { if (continuetcs != null) { Task.Run(() => continuetcs.TrySetResult(true)); } return; } // bufferToUpload needs to be disposed, or we will leak memory // // Unfortunately, because of the async nature of the work, we cannot safely // put this in a using block, so the Write*Async methods must handle disposal. MultiBufferMemoryStream bufferToUpload = this.internalBuffer; this.internalBuffer = new MultiBufferMemoryStream(this.Blob.ServiceClient.BufferManager); bufferToUpload.Seek(0, SeekOrigin.Begin); Checksum bufferChecksum = Checksum.None; if (this.blockChecksum != null) { bool computeCRC64 = false; bool computeMD5 = false; if (this.blockChecksum.MD5 != null) { bufferChecksum.MD5 = this.blockChecksum.MD5.ComputeHash(); computeMD5 = true; } if (this.blockChecksum.CRC64 != null) { bufferChecksum.CRC64 = this.blockChecksum.CRC64.ComputeHash(); computeCRC64 = true; } this.blockChecksum.Dispose(); this.blockChecksum = new ChecksumWrapper(computeMD5, computeCRC64); } if (this.blockBlob != null) { string blockId = this.GetCurrentBlockId(); this.blockList.Add(blockId); await this.WriteBlockAsync(continuetcs, bufferToUpload, blockId, bufferChecksum, token).ConfigureAwait(false); } else if (this.pageBlob != null) { if ((bufferToUpload.Length % Constants.PageSize) != 0) { this.lastException = new IOException(SR.InvalidPageSize); throw this.lastException; } long offset = this.currentBlobOffset; this.currentBlobOffset += bufferToUpload.Length; await this.WritePagesAsync(continuetcs, bufferToUpload, offset, bufferChecksum, token).ConfigureAwait(false); } else { long offset = this.currentBlobOffset; this.currentBlobOffset += bufferToUpload.Length; // We cannot differentiate between max size condition failing only in the retry versus failing in the first attempt and retry. // So we will eliminate the latter and handle the former in the append operation callback. if (this.accessCondition.IfMaxSizeLessThanOrEqual.HasValue && this.currentBlobOffset > this.accessCondition.IfMaxSizeLessThanOrEqual.Value) { this.lastException = new IOException(SR.InvalidBlockSize); throw this.lastException; } await this.WriteAppendBlockAsync(continuetcs, bufferToUpload, offset, bufferChecksum, token).ConfigureAwait(false); } }
public static T ExecuteSync <T>(RESTCommand <T> cmd, IRetryPolicy policy, OperationContext operationContext) { // Note all code below will reference state, not params directly, this will allow common code with async executor using (ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext)) { bool shouldRetry = false; TimeSpan delay = TimeSpan.Zero; do { try { executionState.Init(); // 0. Begin Request Executor.StartRequestAttempt(executionState); // Steps 1-4 Executor.ProcessStartOfRequest(executionState, SR.TraceStartRequestSync); Executor.CheckTimeout <T>(executionState, true); } catch (Exception ex) { Logger.LogError(executionState.OperationContext, SR.TraceInitRequestError, ex.Message); // Store exception and throw here. All operations in this try would be non-retryable by default. At this point, the request is not even made. // Therefore, we will not get extended error info. Hence ParseError doesn't matter here. StorageException storageEx = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError); storageEx.IsRetryable = false; executionState.ExceptionRef = storageEx; throw executionState.ExceptionRef; } // Enter Retryable Section of execution try { // 5. potentially upload data if (cmd.SendStream != null) { executionState.CurrentOperation = ExecutorOperation.BeginGetRequestStream; Logger.LogInformational(executionState.OperationContext, SR.TracePrepareUpload); executionState.Req.Timeout = (int)executionState.RemainingTimeout.TotalMilliseconds; executionState.ReqStream = executionState.Req.GetRequestStream(); executionState.CurrentOperation = ExecutorOperation.BeginUploadRequest; Logger.LogInformational(executionState.OperationContext, SR.TraceUpload); MultiBufferMemoryStream multiBufferMemoryStream = cmd.SendStream as MultiBufferMemoryStream; try { if (multiBufferMemoryStream != null && !cmd.SendStreamLength.HasValue) { multiBufferMemoryStream.FastCopyTo(executionState.ReqStream, executionState.OperationExpiryTime); } else { // don't calculate md5 here as we should have already set this for auth purposes executionState.RestCMD.SendStream.WriteToSync(executionState.ReqStream, cmd.SendStreamLength, null /* maxLength */, false, true, executionState, null /* streamCopyState */); } executionState.ReqStream.Flush(); executionState.ReqStream.Dispose(); executionState.ReqStream = null; } catch (Exception) { executionState.Req.Abort(); throw; } } // 6. Get response try { executionState.CurrentOperation = ExecutorOperation.BeginGetResponse; Logger.LogInformational(executionState.OperationContext, SR.TraceGetResponse); executionState.Req.Timeout = (int)executionState.RemainingTimeout.TotalMilliseconds; executionState.Resp = (HttpWebResponse)executionState.Req.GetResponse(); executionState.CurrentOperation = ExecutorOperation.EndGetResponse; } catch (WebException ex) { Logger.LogWarning(executionState.OperationContext, SR.TraceGetResponseError, ex.Message); executionState.Resp = (HttpWebResponse)ex.Response; if (ex.Status == WebExceptionStatus.Timeout || executionState.ReqTimedOut) { throw new TimeoutException(); } if (executionState.Resp == null) { throw; } else { executionState.ExceptionRef = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError); } } // Response Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag); Executor.FireResponseReceived(executionState); // 7. Do Response parsing (headers etc, no stream available here) if (cmd.PreProcessResponse != null) { executionState.CurrentOperation = ExecutorOperation.PreProcess; executionState.Result = cmd.PreProcessResponse(cmd, executionState.Resp, executionState.ExceptionRef, executionState.OperationContext); // clear exception executionState.ExceptionRef = null; Logger.LogInformational(executionState.OperationContext, SR.TracePreProcessDone); } // 8. (Potentially reads stream from server) executionState.CurrentOperation = ExecutorOperation.GetResponseStream; cmd.ResponseStream = executionState.Resp.GetResponseStream(); if (!cmd.RetrieveResponseStream) { cmd.DestinationStream = Stream.Null; } if (cmd.DestinationStream != null) { if (cmd.StreamCopyState == null) { cmd.StreamCopyState = new StreamDescriptor(); } try { executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse; Logger.LogInformational(executionState.OperationContext, SR.TraceDownload); cmd.ResponseStream.WriteToSync(cmd.DestinationStream, null /* copyLength */, null /* maxLength */, cmd.CalculateMd5ForResponseStream, false, executionState, cmd.StreamCopyState); } finally { cmd.ResponseStream.Dispose(); cmd.ResponseStream = null; } } // Step 9 - This will not be called if an exception is raised during stream copying Executor.ProcessEndOfRequest(executionState); Executor.FinishRequestAttempt(executionState); return(executionState.Result); } catch (Exception e) { Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, e.Message); Executor.FinishRequestAttempt(executionState); StorageException translatedException = ExecutorBase.TranslateExceptionBasedOnParseError(e, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError); executionState.ExceptionRef = translatedException; Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message); shouldRetry = false; if (translatedException.IsRetryable && (executionState.RetryPolicy != null)) { executionState.CurrentLocation = Executor.GetNextLocation(executionState.CurrentLocation, cmd.LocationMode); Logger.LogInformational(executionState.OperationContext, SR.TraceNextLocation, executionState.CurrentLocation); IExtendedRetryPolicy extendedRetryPolicy = executionState.RetryPolicy as IExtendedRetryPolicy; if (extendedRetryPolicy != null) { RetryContext retryContext = new RetryContext( executionState.RetryCount++, cmd.CurrentResult, executionState.CurrentLocation, cmd.LocationMode); RetryInfo retryInfo = extendedRetryPolicy.Evaluate(retryContext, executionState.OperationContext); if (retryInfo != null) { Logger.LogInformational(executionState.OperationContext, SR.TraceRetryInfo, retryInfo.TargetLocation, retryInfo.UpdatedLocationMode); shouldRetry = true; executionState.CurrentLocation = retryInfo.TargetLocation; cmd.LocationMode = retryInfo.UpdatedLocationMode; delay = retryInfo.RetryInterval; } } else { shouldRetry = executionState.RetryPolicy.ShouldRetry( executionState.RetryCount++, cmd.CurrentResult.HttpStatusCode, executionState.ExceptionRef, out delay, executionState.OperationContext); } if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff)) { delay = Constants.MaximumRetryBackoff; } } } finally { if (executionState.Resp != null) { executionState.Resp.Close(); executionState.Resp = null; } } if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0)) { Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message); throw executionState.ExceptionRef; } else { if (executionState.Cmd.RecoveryAction != null) { // I.E. Rewind stream etc. executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext); } if (delay > TimeSpan.Zero) { Logger.LogInformational(executionState.OperationContext, SR.TraceRetryDelay, (int)delay.TotalMilliseconds); Thread.Sleep(delay); } Logger.LogInformational(executionState.OperationContext, SR.TraceRetry); } Executor.FireRetrying(executionState); }while (shouldRetry); } // should never get here, either return, or throw; throw new NotImplementedException(SR.InternalStorageError); }
internal static Tuple <HttpWebRequest, Stream> BuildRequestForTableOperation(Uri uri, UriQueryBuilder builder, IBufferManager bufferManager, int?timeout, TableOperation operation, bool useVersionHeader, OperationContext ctx, TableRequestOptions options) { HttpWebRequest msg = BuildRequestCore(uri, builder, operation.HttpMethod, timeout, useVersionHeader, ctx); TablePayloadFormat payloadFormat = options.PayloadFormat.Value; // Set Accept and Content-Type based on the payload format. SetAcceptHeaderForHttpWebRequest(msg, payloadFormat); Logger.LogInformational(ctx, SR.PayloadFormat, payloadFormat); msg.Headers.Add(Constants.HeaderConstants.DataServiceVersion, Constants.HeaderConstants.DataServiceVersionValue); if (operation.HttpMethod != "HEAD" && operation.HttpMethod != "GET") { msg.ContentType = Constants.JsonContentTypeHeaderValue; } if (operation.OperationType == TableOperationType.InsertOrMerge || operation.OperationType == TableOperationType.Merge) { // Client-side encryption is not supported on merge requests. // This is because we maintain the list of encrypted properties as a property on the entity, and we can't update this // properly for merge operations. options.AssertNoEncryptionPolicyOrStrictMode(); // post tunnelling msg.Headers.Add(Constants.HeaderConstants.PostTunnelling, "MERGE"); } if (operation.OperationType == TableOperationType.RotateEncryptionKey) { // post tunnelling msg.Headers.Add(Constants.HeaderConstants.PostTunnelling, "MERGE"); } // etag if (operation.OperationType == TableOperationType.Delete || operation.OperationType == TableOperationType.Replace || operation.OperationType == TableOperationType.Merge || operation.OperationType == TableOperationType.RotateEncryptionKey) { if (operation.ETag != null) { msg.Headers.Add(Constants.HeaderConstants.IfMatch, operation.ETag); } } // Prefer header if (operation.OperationType == TableOperationType.Insert) { msg.Headers.Add(Constants.HeaderConstants.Prefer, operation.EchoContent ? Constants.HeaderConstants.PreferReturnContent : Constants.HeaderConstants.PreferReturnNoContent); } if (operation.OperationType == TableOperationType.Insert || operation.OperationType == TableOperationType.Merge || operation.OperationType == TableOperationType.InsertOrMerge || operation.OperationType == TableOperationType.InsertOrReplace || operation.OperationType == TableOperationType.Replace || operation.OperationType == TableOperationType.RotateEncryptionKey) { MultiBufferMemoryStream ms = new MultiBufferMemoryStream(bufferManager); using (JsonTextWriter jsonWriter = new JsonTextWriter(new StreamWriter(new NonCloseableStream(ms)))) { WriteEntityContent(operation, ctx, options, jsonWriter); } ms.Seek(0, SeekOrigin.Begin); msg.ContentLength = ms.Length; return(new Tuple <HttpWebRequest, Stream>(msg, ms)); } return(new Tuple <HttpWebRequest, Stream>(msg, null)); }
internal static StorageRequestMessage BuildRequestForTableBatchOperation <T>(RESTCommand <T> cmd, Uri uri, UriQueryBuilder builder, int?timeout, string tableName, TableBatchOperation batch, CloudTableClient client, HttpContent content, OperationContext ctx, TablePayloadFormat payloadFormat, ICanonicalizer canonicalizer, StorageCredentials credentials) { StorageRequestMessage msg = BuildRequestCore(NavigationHelper.AppendPathToSingleUri(uri, "$batch"), builder, HttpMethod.Post, timeout, content, ctx, canonicalizer, credentials); Logger.LogInformational(ctx, SR.PayloadFormat, payloadFormat); MultiBufferMemoryStream batchContentStream = new MultiBufferMemoryStream(client.BufferManager); string batchID = Guid.NewGuid().ToString(); string changesetID = Guid.NewGuid().ToString(); using (StreamWriter contentWriter = new StreamWriter(new NonCloseableStream(batchContentStream))) { msg.Headers.Add(Constants.HeaderConstants.DataServiceVersion, Constants.HeaderConstants.DataServiceVersionValue); string batchSeparator = Constants.BatchSeparator + batchID; string changesetSeparator = Constants.ChangesetSeparator + changesetID; string acceptHeader = "Accept: "; switch (payloadFormat) { case TablePayloadFormat.Json: acceptHeader = acceptHeader + Constants.JsonLightAcceptHeaderValue; break; case TablePayloadFormat.JsonFullMetadata: acceptHeader = acceptHeader + Constants.JsonFullMetadataAcceptHeaderValue; break; case TablePayloadFormat.JsonNoMetadata: acceptHeader = acceptHeader + Constants.JsonNoMetadataAcceptHeaderValue; break; } contentWriter.WriteLine(batchSeparator); bool isQuery = batch.Count == 1 && batch[0].OperationType == TableOperationType.Retrieve; // Query operations should not be inside changeset in payload if (!isQuery) { // Start Operation contentWriter.WriteLine(Constants.ChangesetBoundaryMarker + changesetID); contentWriter.WriteLine(); } foreach (TableOperation operation in batch) { string httpMethod = operation.OperationType == TableOperationType.Merge || operation.OperationType == TableOperationType.InsertOrMerge ? "MERGE" : operation.HttpMethod.Method; if (!isQuery) { contentWriter.WriteLine(changesetSeparator); } contentWriter.WriteLine(Constants.ContentTypeApplicationHttp); contentWriter.WriteLine(Constants.ContentTransferEncodingBinary); contentWriter.WriteLine(); string tableURI = Uri.EscapeUriString(operation.GenerateRequestURI(uri, tableName).ToString()); // "EscapeUriString" is almost exactly what we need, except that it contains special logic for // the percent sign, which results in an off-by-one error in the number of times "%" is encoded. // This corrects for that. tableURI = tableURI.Replace(@"%25", @"%"); contentWriter.WriteLine(httpMethod + " " + tableURI + " " + Constants.HTTP1_1); contentWriter.WriteLine(acceptHeader); contentWriter.WriteLine(Constants.ContentTypeApplicationJson); if (operation.OperationType == TableOperationType.Insert) { contentWriter.WriteLine(Constants.HeaderConstants.Prefer + @": " + (operation.EchoContent ? Constants.HeaderConstants.PreferReturnContent : Constants.HeaderConstants.PreferReturnNoContent)); } contentWriter.WriteLine(Constants.HeaderConstants.DataServiceVersion + ": " + Constants.HeaderConstants.DataServiceVersionValue); // etag if (operation.OperationType == TableOperationType.Delete || operation.OperationType == TableOperationType.Replace || operation.OperationType == TableOperationType.Merge) { contentWriter.WriteLine(Constants.HeaderConstants.IfMatch + @": " + operation.ETag); } contentWriter.WriteLine(); if (operation.OperationType != TableOperationType.Delete && operation.OperationType != TableOperationType.Retrieve) { using (JsonTextWriter jsonWriter = new JsonTextWriter(contentWriter)) { jsonWriter.CloseOutput = false; WriteEntityContent(operation, ctx, jsonWriter); } contentWriter.WriteLine(); } } if (!isQuery) { contentWriter.WriteLine(changesetSeparator + "--"); } contentWriter.WriteLine(batchSeparator + "--"); } batchContentStream.Seek(0, SeekOrigin.Begin); msg.Content = new StreamContent(batchContentStream); msg.Content.Headers.ContentLength = batchContentStream.Length; msg.Content.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse(Constants.BatchBoundaryMarker + batchID);//new System.Net.Http.Headers.MediaTypeHeaderValue(Constants.BatchBoundaryMarker + batchID); return(msg); }
public void Dispose() { this.msg = null; this.outStr = null; this.content = null; }