/// <summary>
/// Dispatches a write operation.
/// </summary>
/// <param name="asyncResult">The reference to the pending asynchronous request to finish.</param>
private void DispatchWrite(StorageAsyncResult <NullType> asyncResult)
{
if (this.internalBuffer.Length == 0)
{
if (asyncResult != null)
{
asyncResult.OnComplete(this.lastException);
}
return;
}
MultiBufferMemoryStream bufferToUpload = this.internalBuffer;
this.internalBuffer = new MultiBufferMemoryStream(this.Blob.ServiceClient.BufferManager);
bufferToUpload.Seek(0, SeekOrigin.Begin);
string bufferMD5 = null;
if (this.blockMD5 != null)
{
bufferMD5 = this.blockMD5.ComputeHash();
this.blockMD5.Dispose();
this.blockMD5 = new MD5Wrapper();
}
if (this.blockBlob != null)
{
string blockId = this.GetCurrentBlockId();
this.blockList.Add(blockId);
this.WriteBlock(bufferToUpload, blockId, bufferMD5, asyncResult);
}
else if (this.pageBlob != null)
{
if ((bufferToUpload.Length % Constants.PageSize) != 0)
{
this.lastException = new IOException(SR.InvalidPageSize);
throw this.lastException;
}
long offset = this.currentBlobOffset;
this.currentBlobOffset += bufferToUpload.Length;
this.WritePages(bufferToUpload, offset, bufferMD5, asyncResult);
}
else
{
long offset = this.currentBlobOffset;
this.currentBlobOffset += bufferToUpload.Length;
// We cannot differentiate between max size condition failing only in the retry versus failing in the first attempt and retry.
// So we will eliminate the latter and handle the former in the append operation callback.
if (this.accessCondition.IfMaxSizeLessThanOrEqual.HasValue && this.currentBlobOffset > this.accessCondition.IfMaxSizeLessThanOrEqual.Value)
{
this.lastException = new IOException(SR.InvalidBlockSize);
throw this.lastException;
}
this.WriteAppendBlock(bufferToUpload, offset, bufferMD5, asyncResult);
}
}
private RESTCommand <NullType> SetServicePropertiesImpl(ServiceProperties properties, TableRequestOptions requestOptions)
{
MultiBufferMemoryStream memoryStream = new MultiBufferMemoryStream(null /* bufferManager */, (int)(1 * Constants.KB));
try
{
properties.WriteServiceProperties(memoryStream);
}
catch (InvalidOperationException invalidOpException)
{
throw new ArgumentException(invalidOpException.Message, "properties");
}
RESTCommand <NullType> retCmd = new RESTCommand <NullType>(this.Credentials, this.StorageUri);
requestOptions.ApplyToStorageCommand(retCmd);
retCmd.BuildRequest = (cmd, uri, builder, cnt, serverTimeout, ctx) => TableHttpRequestMessageFactory.SetServiceProperties(uri, serverTimeout, cnt, ctx, this.GetCanonicalizer(), this.Credentials);
retCmd.BuildContent = (cmd, ctx) => HttpContentFactory.BuildContentFromStream(memoryStream, 0, memoryStream.Length, null /* md5 */, cmd, ctx);
retCmd.StreamToDispose = memoryStream;
retCmd.PreProcessResponse =
(cmd, resp, ex, ctx) =>
HttpResponseParsers.ProcessExpectedStatusCodeNoException(HttpStatusCode.Accepted, resp, null /* retVal */, cmd, ex);
requestOptions.ApplyToStorageCommand(retCmd);
return(retCmd);
}
/// <summary>
/// Initializes a new instance of the FileWriteStreamBase class for a file.
/// </summary>
/// <param name="file">File reference to write to.</param>
/// <param name="fileSize">Size of the file.</param>
/// <param name="createNew">Use <c>true</c> if the file is newly created, <c>false</c> otherwise.</param>
/// <param name="accessCondition">An <see cref="AccessCondition"/> object that represents the access conditions for the file. If <c>null</c>, no condition is used.</param>
/// <param name="options">An <see cref="FileRequestOptions"/> object that specifies additional options for the request.</param>
/// <param name="operationContext">An <see cref="OperationContext"/> object for tracking the current operation.</param>
protected FileWriteStreamBase(CloudFile file, long fileSize, bool createNew, AccessCondition accessCondition, FileRequestOptions options, OperationContext operationContext)
: base()
{
this.internalBuffer = new MultiBufferMemoryStream(file.ServiceClient.BufferManager);
this.currentOffset = 0;
this.accessCondition = accessCondition;
this.options = options;
this.operationContext = operationContext;
this.noPendingWritesEvent = new CounterEvent();
this.fileMD5 = this.options.StoreFileContentMD5.Value ? new MD5Wrapper() : null;
this.rangeMD5 = this.options.UseTransactionalMD5.Value ? new MD5Wrapper() : null;
#if !(NETCORE || WINDOWS_RT)
this.parallelOperationSemaphoreAsync = new AsyncSemaphoreAsync(options.ParallelOperationThreadCount.Value);
#else
this.parallelOperationSemaphore = new AsyncSemaphore(options.ParallelOperationThreadCount.Value);
#endif
this.lastException = null;
this.committed = false;
this.disposed = false;
this.currentFileOffset = 0;
this.file = file;
this.fileSize = fileSize;
this.streamWriteSizeInBytes = file.StreamWriteSizeInBytes;
this.newFile = createNew;
}
private static void EndGetRequestStream <T>(IAsyncResult getRequestStreamResult)
{
ExecutionState <T> executionState = (ExecutionState <T>)getRequestStreamResult.AsyncState;
executionState.CurrentOperation = ExecutorOperation.EndGetRequestStream;
try
{
executionState.UpdateCompletedSynchronously(getRequestStreamResult.CompletedSynchronously);
executionState.ReqStream = executionState.Req.EndGetRequestStream(getRequestStreamResult);
executionState.CurrentOperation = ExecutorOperation.BeginUploadRequest;
Logger.LogInformational(executionState.OperationContext, SR.TraceUpload);
MultiBufferMemoryStream multiBufferMemoryStream = executionState.RestCMD.SendStream as MultiBufferMemoryStream;
if (multiBufferMemoryStream != null && !executionState.RestCMD.SendStreamLength.HasValue)
{
multiBufferMemoryStream.BeginFastCopyTo(executionState.ReqStream, executionState.OperationExpiryTime, EndFastCopyTo <T>, executionState);
}
else
{
// don't calculate md5 here as we should have already set this for auth purposes
executionState.RestCMD.SendStream.WriteToAsync(executionState.ReqStream, executionState.RestCMD.SendStreamLength, null /* maxLength */, false, executionState, null /* streamCopyState */, EndSendStreamCopy);
}
}
catch (Exception ex)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceUploadError, ex.Message);
executionState.ExceptionRef = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
Executor.EndOperation(executionState);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="FileReadStreamBase"/> class.
/// </summary>
/// <param name="file">File reference to read from</param>
/// <param name="accessCondition">An <see cref="AccessCondition"/> object that represents the access conditions for the file. If <c>null</c>, no condition is used.</param>
/// <param name="options">An <see cref="FileRequestOptions"/> object that specifies additional options for the request.</param>
/// <param name="operationContext">An <see cref="OperationContext"/> object for tracking the current operation.</param>
protected FileReadStreamBase(CloudFile file, AccessCondition accessCondition, FileRequestOptions options, OperationContext operationContext)
{
if (options.ChecksumOptions.UseTransactionalMD5.Value)
{
CommonUtility.AssertInBounds("StreamMinimumReadSizeInBytes", file.StreamMinimumReadSizeInBytes, 1, Constants.MaxRangeGetContentMD5Size);
}
if (options.ChecksumOptions.UseTransactionalCRC64.Value)
{
CommonUtility.AssertInBounds("StreamMinimumReadSizeInBytes", file.StreamMinimumReadSizeInBytes, 1, Constants.MaxRangeGetContentCRC64Size);
}
this.file = file;
this.fileProperties = new FileProperties(file.Properties);
this.currentOffset = 0;
this.streamMinimumReadSizeInBytes = this.file.StreamMinimumReadSizeInBytes;
this.internalBuffer = new MultiBufferMemoryStream(file.ServiceClient.BufferManager);
this.accessCondition = accessCondition;
this.options = options;
this.operationContext = operationContext;
this.fileChecksum =
new ChecksumWrapper(
calcMd5: !(this.options.ChecksumOptions.DisableContentMD5Validation.Value || string.IsNullOrEmpty(this.fileProperties.ContentChecksum.MD5)),
calcCrc64: !(this.options.ChecksumOptions.DisableContentCRC64Validation.Value || string.IsNullOrEmpty(this.fileProperties.ContentChecksum.CRC64))
);
this.lastException = null;
}
/// <summary>
/// Releases the file resources used by the Stream.
/// </summary>
/// <param name="disposing">true to release both managed and unmanaged resources; false to release only unmanaged resources.</param>
protected override void Dispose(bool disposing)
{
if (disposing)
{
if (this.fileChecksum != null)
{
this.fileChecksum.Dispose();
this.fileChecksum = null;
}
if (this.rangeChecksum != null)
{
this.rangeChecksum.Dispose();
this.rangeChecksum = null;
}
if (this.internalBuffer != null)
{
this.internalBuffer.Dispose();
this.internalBuffer = null;
}
}
base.Dispose(disposing);
}
/// <summary>
/// Releases the file resources used by the Stream.
/// </summary>
/// <param name="disposing">true to release both managed and unmanaged resources; false to release only unmanaged resources.</param>
protected override void Dispose(bool disposing)
{
if (disposing)
{
if (this.fileMD5 != null)
{
this.fileMD5.Dispose();
this.fileMD5 = null;
}
if (this.rangeMD5 != null)
{
this.rangeMD5.Dispose();
this.rangeMD5 = null;
}
if (this.internalBuffer != null)
{
this.internalBuffer.Dispose();
this.internalBuffer = null;
}
if (this.noPendingWritesEvent != null)
{
this.noPendingWritesEvent.Dispose();
this.noPendingWritesEvent = null;
}
}
base.Dispose(disposing);
}
/// <summary>
/// Asynchronously dispatches a write operation.
/// </summary>
/// <returns>A task that represents the asynchronous write operation.</returns>
private async Task DispatchWriteAsync()
{
if (this.internalBuffer.Length == 0)
{
return;
}
MultiBufferMemoryStream bufferToUpload = this.internalBuffer;
this.internalBuffer = new MultiBufferMemoryStream(this.file.ServiceClient.BufferManager);
bufferToUpload.Seek(0, SeekOrigin.Begin);
string bufferMD5 = null;
if (this.rangeMD5 != null)
{
bufferMD5 = this.rangeMD5.ComputeHash();
this.rangeMD5.Dispose();
this.rangeMD5 = new MD5Wrapper();
}
long offset = this.currentFileOffset;
this.currentFileOffset += bufferToUpload.Length;
await this.WriteRangeAsync(bufferToUpload, offset, bufferMD5).ConfigureAwait(false);
}
/// <summary>
/// Asynchronously dispatches a write operation.
/// </summary>
/// <returns>A task that represents the asynchronous write operation.</returns>
private async Task DispatchWriteAsync(TaskCompletionSource <bool> continuetcs, CancellationToken token)
{
if (this.internalBuffer.Length == 0)
{
if (continuetcs != null)
{
Task.Run(() => continuetcs.TrySetResult(true));
}
return;
}
MultiBufferMemoryStream bufferToUpload = this.internalBuffer;
this.internalBuffer = new MultiBufferMemoryStream(this.file.ServiceClient.BufferManager);
bufferToUpload.Seek(0, SeekOrigin.Begin);
string bufferMD5 = null;
if (this.rangeMD5 != null)
{
bufferMD5 = this.rangeMD5.ComputeHash();
this.rangeMD5.Dispose();
this.rangeMD5 = new MD5Wrapper();
}
long offset = this.currentFileOffset;
this.currentFileOffset += bufferToUpload.Length;
await this.WriteRangeAsync(continuetcs, bufferToUpload, offset, bufferMD5, token).ConfigureAwait(false);
}
private RESTCommand <NullType> SetServicePropertiesImpl(ServiceProperties properties, TableRequestOptions requestOptions)
{
MultiBufferMemoryStream str = new MultiBufferMemoryStream(null /* bufferManager */, (int)(1 * Constants.KB));
try
{
properties.WriteServiceProperties(str);
}
catch (InvalidOperationException invalidOpException)
{
throw new ArgumentException(invalidOpException.Message, "properties");
}
str.Seek(0, SeekOrigin.Begin);
RESTCommand <NullType> retCmd = new RESTCommand <NullType>(this.Credentials, this.StorageUri);
retCmd.SendStream = str;
retCmd.BuildRequestDelegate = TableHttpWebRequestFactory.SetServiceProperties;
retCmd.RecoveryAction = RecoveryActions.RewindStream;
retCmd.SignRequest = this.AuthenticationHandler.SignRequest;
retCmd.ParseError = StorageExtendedErrorInformation.ReadFromStreamUsingODataLib;
retCmd.PreProcessResponse =
(cmd, resp, ex, ctx) => HttpResponseParsers.ProcessExpectedStatusCodeNoException(HttpStatusCode.Accepted, resp, NullType.Value, cmd, ex);
requestOptions.ApplyToStorageCommand(retCmd);
return(retCmd);
}
/// <summary>
/// Dispatches a write operation.
/// </summary>
/// <param name="asyncResult">The reference to the pending asynchronous request to finish.</param>
private void DispatchWrite(StorageAsyncResult <NullType> asyncResult)
{
if (this.internalBuffer.Length == 0)
{
if (asyncResult != null)
{
asyncResult.OnComplete(this.lastException);
}
return;
}
MultiBufferMemoryStream bufferToUpload = this.internalBuffer;
this.internalBuffer = new MultiBufferMemoryStream(this.file.ServiceClient.BufferManager);
bufferToUpload.Seek(0, SeekOrigin.Begin);
string bufferMD5 = null;
if (this.rangeMD5 != null)
{
bufferMD5 = this.rangeMD5.ComputeHash();
this.rangeMD5.Dispose();
this.rangeMD5 = new MD5Wrapper();
}
long offset = this.currentFileOffset;
this.currentFileOffset += bufferToUpload.Length;
this.WriteRange(bufferToUpload, offset, bufferMD5, asyncResult);
}
private RESTCommand <NullType> SetServicePropertiesImpl(ServiceProperties properties, QueueRequestOptions requestOptions)
{
MultiBufferMemoryStream memoryStream = new MultiBufferMemoryStream(null /* bufferManager */, (int)(1 * Constants.KB));
try
{
properties.WriteServiceProperties(memoryStream);
}
catch (InvalidOperationException invalidOpException)
{
throw new ArgumentException(invalidOpException.Message, "properties");
}
RESTCommand <NullType> retCmd = new RESTCommand <NullType>(this.Credentials, this.BaseUri);
retCmd.ApplyRequestOptions(requestOptions);
retCmd.BuildRequest = (cmd, cnt, ctx) => QueueHttpRequestMessageFactory.SetServiceProperties(cmd.Uri, cmd.ServerTimeoutInSeconds, cnt, ctx);
retCmd.BuildContent = (cmd, ctx) => HttpContentFactory.BuildContentFromStream(memoryStream, 0, memoryStream.Length, null /* md5 */, cmd, ctx);
retCmd.Handler = this.AuthenticationHandler;
retCmd.BuildClient = HttpClientFactory.BuildHttpClient;
retCmd.PreProcessResponse =
(cmd, resp, ex, ctx) =>
HttpResponseParsers.ProcessExpectedStatusCodeNoException(System.Net.HttpStatusCode.Accepted, resp, null /* retVal */, cmd, ex);
retCmd.ApplyRequestOptions(requestOptions);
return(retCmd);
}
/// <summary>
/// Releases the blob resources used by the Stream.
/// </summary>
/// <param name="disposing">true to release both managed and unmanaged resources; false to release only unmanaged resources.</param>
protected override void Dispose(bool disposing)
{
if (disposing)
{
if (this.blobMD5 != null)
{
this.blobMD5.Dispose();
this.blobMD5 = null;
}
if (this.blockMD5 != null)
{
this.blockMD5.Dispose();
this.blockMD5 = null;
}
if (this.internalBuffer != null)
{
this.internalBuffer.Dispose();
this.internalBuffer = null;
}
}
base.Dispose(disposing);
}
private async Task DispatchWriteAsync(TaskCompletionSource <bool> continuetcs, CancellationToken token)
{
if (this.internalBuffer.Length == 0)
{
if (continuetcs != null)
{
Task.Run(() => continuetcs.TrySetResult(true));
}
return;
}
MultiBufferMemoryStream bufferToUpload = this.internalBuffer;
this.internalBuffer = new MultiBufferMemoryStream(this.Blob.ServiceClient.BufferManager);
bufferToUpload.Seek(0, SeekOrigin.Begin);
string bufferMD5 = null;
if (this.blockMD5 != null)
{
bufferMD5 = this.blockMD5.ComputeHash();
this.blockMD5.Dispose();
this.blockMD5 = new MD5Wrapper();
}
if (this.blockBlob != null)
{
string blockId = this.GetCurrentBlockId();
this.blockList.Add(blockId);
await this.WriteBlockAsync(continuetcs, bufferToUpload, blockId, bufferMD5, token).ConfigureAwait(false);
}
else if (this.pageBlob != null)
{
if ((bufferToUpload.Length % Constants.PageSize) != 0)
{
this.lastException = new IOException(SR.InvalidPageSize);
throw this.lastException;
}
long offset = this.currentBlobOffset;
this.currentBlobOffset += bufferToUpload.Length;
await this.WritePagesAsync(continuetcs, bufferToUpload, offset, bufferMD5, token).ConfigureAwait(false);
}
else
{
long offset = this.currentBlobOffset;
this.currentBlobOffset += bufferToUpload.Length;
// We cannot differentiate between max size condition failing only in the retry versus failing in the first attempt and retry.
// So we will eliminate the latter and handle the former in the append operation callback.
if (this.accessCondition.IfMaxSizeLessThanOrEqual.HasValue && this.currentBlobOffset > this.accessCondition.IfMaxSizeLessThanOrEqual.Value)
{
this.lastException = new IOException(SR.InvalidBlockSize);
throw this.lastException;
}
await this.WriteAppendBlockAsync(continuetcs, bufferToUpload, offset, bufferMD5, token).ConfigureAwait(false);
}
}
/// <summary>
/// Stores a new entity in the configured blob container.
/// </summary>
/// <param name="changes">The changes to write to storage.</param>
/// <param name="cancellationToken">A cancellation token that can be used by other objects
/// or threads to receive notice of cancellation.</param>
/// <returns>A task that represents the work queued to execute.</returns>
public async Task WriteAsync(IDictionary <string, object> changes, CancellationToken cancellationToken = default(CancellationToken))
{
if (changes == null)
{
throw new ArgumentNullException(nameof(changes));
}
var blobClient = _storageAccount.CreateCloudBlobClient();
var blobContainer = blobClient.GetContainerReference(_containerName);
// this should only happen once - assuming this is a singleton
if (Interlocked.CompareExchange(ref _checkforContainerExistance, 0, 1) == 1)
{
await blobContainer.CreateIfNotExistsAsync(cancellationToken).ConfigureAwait(false);
}
var blobRequestOptions = new BlobRequestOptions();
var operationContext = new OperationContext();
foreach (var keyValuePair in changes)
{
var newValue = keyValuePair.Value;
var storeItem = newValue as IStoreItem;
// "*" eTag in IStoreItem converts to null condition for AccessCondition
var accessCondition = storeItem?.ETag != "*"
? AccessCondition.GenerateIfMatchCondition(storeItem?.ETag)
: AccessCondition.GenerateEmptyCondition();
var blobName = GetBlobName(keyValuePair.Key);
var blobReference = blobContainer.GetBlockBlobReference(blobName);
try
{
using (var memoryStream = new MultiBufferMemoryStream(blobReference.ServiceClient.BufferManager))
using (var streamWriter = new StreamWriter(memoryStream))
{
_jsonSerializer.Serialize(streamWriter, newValue);
await streamWriter.FlushAsync().ConfigureAwait(false);
memoryStream.Seek(0, SeekOrigin.Begin);
await blobReference.UploadFromStreamAsync(memoryStream, accessCondition, blobRequestOptions, operationContext, cancellationToken).ConfigureAwait(false);
}
}
catch (StorageException ex)
when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.BadRequest &&
ex.RequestInformation.ErrorCode == BlobErrorCodeStrings.InvalidBlockList)
{
throw new InvalidOperationException(
$"An error ocurred while trying to write an object. The underlying '{BlobErrorCodeStrings.InvalidBlockList}' error is commonly caused due to concurrently uploading an object larger than 128MB in size.",
ex);
}
}
}
internal static StorageRequestMessage BuildRequestForTableOperation <T>(RESTCommand <T> cmd, Uri uri, UriQueryBuilder builder, int?timeout, TableOperation operation, CloudTableClient client, HttpContent content, OperationContext ctx, TablePayloadFormat payloadFormat, ICanonicalizer canonicalizer, StorageCredentials credentials)
{
StorageRequestMessage msg = BuildRequestCore(uri, builder, operation.HttpMethod, timeout, content, ctx, canonicalizer, credentials);
// Set Accept and Content-Type based on the payload format.
SetAcceptHeaderForHttpWebRequest(msg, payloadFormat);
Logger.LogInformational(ctx, SR.PayloadFormat, payloadFormat);
msg.Headers.Add(Constants.HeaderConstants.DataServiceVersion, Constants.HeaderConstants.DataServiceVersionValue);
if (operation.OperationType == TableOperationType.InsertOrMerge || operation.OperationType == TableOperationType.Merge)
{
// post tunnelling
msg.Headers.Add(Constants.HeaderConstants.PostTunnelling, "MERGE");
}
// etag
if (operation.OperationType == TableOperationType.Delete ||
operation.OperationType == TableOperationType.Replace ||
operation.OperationType == TableOperationType.Merge)
{
msg.Headers.Add(Constants.HeaderConstants.IfMatch, operation.ETag);
}
// Prefer header
if (operation.OperationType == TableOperationType.Insert)
{
msg.Headers.Add(Constants.HeaderConstants.Prefer, operation.EchoContent ? Constants.HeaderConstants.PreferReturnContent : Constants.HeaderConstants.PreferReturnNoContent);
}
if (operation.OperationType == TableOperationType.Insert ||
operation.OperationType == TableOperationType.Merge ||
operation.OperationType == TableOperationType.InsertOrMerge ||
operation.OperationType == TableOperationType.InsertOrReplace ||
operation.OperationType == TableOperationType.Replace)
{
MultiBufferMemoryStream ms = new MultiBufferMemoryStream(client.BufferManager);
using (JsonTextWriter jsonWriter = new JsonTextWriter(new StreamWriter(new NonCloseableStream(ms))))
{
WriteEntityContent(operation, ctx, jsonWriter);
}
ms.Seek(0, SeekOrigin.Begin);
msg.Content = new StreamContent(ms);
msg.Content.Headers.ContentLength = ms.Length;
if (!operation.HttpMethod.Equals("HEAD") && !operation.HttpMethod.Equals("GET"))
{
SetContentTypeForHttpWebRequest(msg);
}
return(msg);
}
return(msg);
}
/// <summary>
/// Initializes a new instance of the BlobWriteStreamBase class.
/// </summary>
/// <param name="serviceClient">The service client.</param>
/// <param name="accessCondition">An <see cref="AccessCondition"/> object that represents the condition that must be met in order for the request to proceed. If <c>null</c>, no condition is used.</param>
/// <param name="options">A <see cref="BlobRequestOptions"/> object that specifies additional options for the request.</param>
/// <param name="operationContext">An <see cref="OperationContext"/> object that represents the context for the current operation.</param>
private BlobWriteStreamBase(CloudBlobClient serviceClient, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext)
: base()
{
this.internalBuffer = new MultiBufferMemoryStream(serviceClient.BufferManager);
this.accessCondition = accessCondition;
this.currentOffset = 0;
this.options = options;
this.operationContext = operationContext;
this.noPendingWritesEvent = new CounterEvent();
this.blobMD5 = this.options.StoreBlobContentMD5.Value ? new MD5Wrapper() : null;
this.blockMD5 = this.options.UseTransactionalMD5.Value ? new MD5Wrapper() : null;
this.parallelOperationSemaphore = new AsyncSemaphore(options.ParallelOperationThreadCount.Value);
this.lastException = null;
this.committed = false;
this.disposed = false;
}
/// <summary>
/// Stores a new entity in the configured blob container.
/// </summary>
/// <param name="changes">The Dictionary of changes that are to be made.</param>
/// <param name="cancellationToken">A cancellation token that can be used by other objects
/// or threads to receive notice of cancellation.</param>
/// <returns>A <see cref="Task"/>A task that represents the work queued to execute.</returns>
public async Task WriteAsync(IDictionary <string, object> changes, CancellationToken cancellationToken = default(CancellationToken))
{
if (changes == null)
{
throw new ArgumentNullException(nameof(changes));
}
var blobContainer = await GetBlobContainer().ConfigureAwait(false);
var blobRequestOptions = new BlobRequestOptions();
var operationContext = new OperationContext();
await Task.WhenAll(
changes.Select(async(keyValuePair) =>
{
var newValue = keyValuePair.Value;
var storeItem = newValue as IStoreItem;
// "*" eTag in IStoreItem converts to null condition for AccessCondition
var accessCondition = storeItem?.ETag == "*"
? AccessCondition.GenerateEmptyCondition()
: AccessCondition.GenerateIfMatchCondition(storeItem?.ETag);
var blobName = GetBlobName(keyValuePair.Key);
var blobReference = blobContainer.GetBlockBlobReference(blobName);
try
{
using (var memoryStream = new MultiBufferMemoryStream(blobReference.ServiceClient.BufferManager))
using (var streamWriter = new StreamWriter(memoryStream))
{
JsonSerializer.Serialize(streamWriter, newValue);
streamWriter.Flush();
memoryStream.Seek(0, SeekOrigin.Begin);
await blobReference.UploadFromStreamAsync(memoryStream, accessCondition, blobRequestOptions, operationContext).ConfigureAwait(false);
}
}
catch (StorageException ex)
when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.BadRequest &&
ex.RequestInformation.ErrorCode == BlobErrorCodeStrings.InvalidBlockList)
{
throw new Exception(
$"An error ocurred while trying to write an object. The underlying '{BlobErrorCodeStrings.InvalidBlockList}' error is commonly caused due to concurrently uploading an object larger than 128MB in size.",
ex);
}
})).ConfigureAwait(false);
}
/// <summary>
/// Asynchronously dispatches a write operation.
/// </summary>
/// <returns>A task that represents the asynchronous write operation.</returns>
private async Task DispatchWriteAsync(TaskCompletionSource <bool> continuetcs, CancellationToken token)
{
if (this.internalBuffer.Length == 0)
{
if (continuetcs != null)
{
Task.Run(() => continuetcs.TrySetResult(true));
}
return;
}
// bufferToUpload needs to be disposed, or we will leak memory
//
// Unfortunately, because of the async nature of the work, we cannot safely
// put this in a using block, so the Write*Async methods must handle disposal.
MultiBufferMemoryStream bufferToUpload = this.internalBuffer;
this.internalBuffer = new MultiBufferMemoryStream(this.file.ServiceClient.BufferManager);
bufferToUpload.Seek(0, SeekOrigin.Begin);
Checksum bufferChecksum = Checksum.None;
if (this.rangeChecksum != null)
{
bool computeCRC64 = false;
bool computeMD5 = false;
if (this.rangeChecksum.MD5 != null)
{
bufferChecksum.MD5 = this.rangeChecksum.MD5.ComputeHash();
computeMD5 = true;
}
if (this.rangeChecksum.CRC64 != null)
{
bufferChecksum.CRC64 = this.rangeChecksum.CRC64.ComputeHash();
computeCRC64 = true;
}
this.rangeChecksum.Dispose();
this.rangeChecksum = new ChecksumWrapper(computeMD5, computeCRC64);
}
long offset = this.currentFileOffset;
this.currentFileOffset += bufferToUpload.Length;
await this.WriteRangeAsync(continuetcs, bufferToUpload, offset, bufferChecksum, token).ConfigureAwait(false);
}
/// <summary>
/// Dispatches a write operation.
/// </summary>
/// <param name="asyncResult">The reference to the pending asynchronous request to finish.</param>
private void DispatchWrite(StorageAsyncResult <NullType> asyncResult)
{
if (this.internalBuffer.Length == 0)
{
if (asyncResult != null)
{
asyncResult.OnComplete(this.lastException);
}
return;
}
MultiBufferMemoryStream bufferToUpload = this.internalBuffer;
this.internalBuffer = new MultiBufferMemoryStream(this.Blob.ServiceClient.BufferManager);
bufferToUpload.Seek(0, SeekOrigin.Begin);
string bufferMD5 = null;
if (this.blockMD5 != null)
{
bufferMD5 = this.blockMD5.ComputeHash();
this.blockMD5.Dispose();
this.blockMD5 = new MD5Wrapper();
}
if (this.blockBlob != null)
{
string blockId = this.GetCurrentBlockId();
this.blockList.Add(blockId);
this.WriteBlock(bufferToUpload, blockId, bufferMD5, asyncResult);
}
else
{
if ((bufferToUpload.Length % Constants.PageSize) != 0)
{
this.lastException = new IOException(SR.InvalidPageSize);
throw this.lastException;
}
long offset = this.currentPageOffset;
this.currentPageOffset += bufferToUpload.Length;
this.WritePages(bufferToUpload, offset, bufferMD5, asyncResult);
}
}
/// <summary>
/// Initializes a new instance of the BlobReadStreamBase class.
/// </summary>
/// <param name="blob">Blob reference to read from</param>
/// <param name="accessCondition">An <see cref="AccessCondition"/> object that represents the condition that must be met in order for the request to proceed. If <c>null</c>, no condition is used.</param>
/// <param name="options">A <see cref="BlobRequestOptions"/> object that specifies additional options for the request.</param>
/// <param name="operationContext">An <see cref="OperationContext"/> object that represents the context for the current operation.</param>
protected BlobReadStreamBase(CloudBlob blob, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext)
{
if (options.UseTransactionalMD5.Value)
{
CommonUtility.AssertInBounds("StreamMinimumReadSizeInBytes", blob.StreamMinimumReadSizeInBytes, 1, Constants.MaxRangeGetContentMD5Size);
}
this.blob = blob;
this.blobProperties = new BlobProperties(blob.Properties);
this.currentOffset = 0;
this.streamMinimumReadSizeInBytes = this.blob.StreamMinimumReadSizeInBytes;
this.internalBuffer = new MultiBufferMemoryStream(blob.ServiceClient.BufferManager);
this.accessCondition = accessCondition;
this.options = options;
this.operationContext = operationContext;
this.blobMD5 = (this.options.DisableContentMD5Validation.Value || string.IsNullOrEmpty(this.blobProperties.ContentMD5)) ? null : new MD5Wrapper();
this.lastException = null;
}
/// <summary>
/// Initializes a new instance of the BlobWriteStreamBase class.
/// </summary>
/// <param name="serviceClient">The service client.</param>
/// <param name="accessCondition">An <see cref="AccessCondition"/> object that represents the condition that must be met in order for the request to proceed. If <c>null</c>, no condition is used.</param>
/// <param name="options">A <see cref="BlobRequestOptions"/> object that specifies additional options for the request.</param>
/// <param name="operationContext">An <see cref="OperationContext"/> object that represents the context for the current operation.</param>
private BlobWriteStreamBase(CloudBlobClient serviceClient, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext)
: base()
{
this.internalBuffer = new MultiBufferMemoryStream(serviceClient.BufferManager);
this.accessCondition = accessCondition;
this.currentOffset = 0;
this.options = options;
this.operationContext = operationContext;
this.noPendingWritesEvent = new CounterEventAsync();
this.blobChecksum = new ChecksumWrapper(this.options.ChecksumOptions.StoreContentMD5.Value, this.options.ChecksumOptions.StoreContentCRC64.Value);
this.blockChecksum = new ChecksumWrapper(this.options.ChecksumOptions.UseTransactionalMD5.Value, this.options.ChecksumOptions.UseTransactionalCRC64.Value);
#if WINDOWS_DESKTOP
this.parallelOperationSemaphoreAsync = new AsyncSemaphoreAsync(options.ParallelOperationThreadCount.Value);
#else
this.parallelOperationSemaphore = new AsyncSemaphore(options.ParallelOperationThreadCount.Value);
#endif
this.lastException = null;
this.committed = false;
this.disposed = false;
}
private static void EndFastCopyTo <T>(IAsyncResult fastCopyToResult)
{
ExecutionState <T> executionState = (ExecutionState <T>)fastCopyToResult.AsyncState;
try
{
executionState.UpdateCompletedSynchronously(fastCopyToResult.CompletedSynchronously);
MultiBufferMemoryStream multiBufferMemoryStream = (MultiBufferMemoryStream)executionState.RestCMD.SendStream;
multiBufferMemoryStream.EndFastCopyTo(fastCopyToResult);
Executor.EndSendStreamCopy(executionState);
}
catch (Exception ex)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceUploadError, ex.Message);
executionState.ExceptionRef = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
Executor.EndOperation(executionState);
}
}
/// <summary>
/// Implementation for the SetPermissions method.
/// </summary>
/// <param name="acl">The permissions to set.</param>
/// <param name="accessCondition">An <see cref="AccessCondition"/> object that represents the access conditions for the share. If <c>null</c>, no condition is used.</param>
/// <param name="options">A <see cref="FileRequestOptions"/> object that specifies additional options for the request.</param>
/// <returns>A <see cref="RESTCommand"/> that sets the permissions.</returns>
private RESTCommand <NullType> SetPermissionsImpl(FileSharePermissions acl, AccessCondition accessCondition, FileRequestOptions options)
{
MultiBufferMemoryStream memoryStream = new MultiBufferMemoryStream(null /* bufferManager */, (int)(1 * Constants.KB));
FileRequest.WriteSharedAccessIdentifiers(acl.SharedAccessPolicies, memoryStream);
RESTCommand <NullType> putCmd = new RESTCommand <NullType>(this.ServiceClient.Credentials, this.StorageUri);
options.ApplyToStorageCommand(putCmd);
putCmd.BuildRequest = (cmd, uri, builder, cnt, serverTimeout, ctx) => ShareHttpRequestMessageFactory.SetAcl(uri, serverTimeout, FileSharePublicAccessType.Off, accessCondition, cnt, ctx, this.ServiceClient.GetCanonicalizer(), this.ServiceClient.Credentials);
putCmd.BuildContent = (cmd, ctx) => HttpContentFactory.BuildContentFromStream(memoryStream, 0, memoryStream.Length, null /* md5 */, cmd, ctx);
putCmd.StreamToDispose = memoryStream;
putCmd.PreProcessResponse = (cmd, resp, ex, ctx) =>
{
HttpResponseParsers.ProcessExpectedStatusCodeNoException(HttpStatusCode.OK, resp, NullType.Value, cmd, ex);
this.UpdateETagAndLastModified(resp);
return(NullType.Value);
};
return(putCmd);
}
/// <summary>
/// Implementation for the SetPermissions method.
/// </summary>
/// <param name="acl">The permissions to set.</param>
/// <param name="requestOptions">A <see cref="TableRequestOptions"/> object that specifies execution options, such as retry policy and timeout settings, for the operation.</param>
/// <returns>A <see cref="RESTCommand"/> that sets the permissions.</returns>
private RESTCommand <NullType> SetPermissionsImpl(TablePermissions acl, TableRequestOptions requestOptions)
{
MultiBufferMemoryStream memoryStream = new MultiBufferMemoryStream(null /* bufferManager */, (int)(1 * Constants.KB));
TableRequest.WriteSharedAccessIdentifiers(acl.SharedAccessPolicies, memoryStream);
RESTCommand <NullType> putCmd = new RESTCommand <NullType>(this.ServiceClient.Credentials, this.Uri);
putCmd.ApplyRequestOptions(requestOptions);
putCmd.Handler = this.ServiceClient.AuthenticationHandler;
putCmd.BuildClient = HttpClientFactory.BuildHttpClient;
putCmd.BuildRequest = (cmd, cnt, ctx) => TableHttpRequestMessageFactory.SetAcl(cmd.Uri, cmd.ServerTimeoutInSeconds, cnt, ctx);
putCmd.BuildContent = (cmd, ctx) => HttpContentFactory.BuildContentFromStream(memoryStream, 0, memoryStream.Length, null /* md5 */, cmd, ctx);
putCmd.PreProcessResponse = (cmd, resp, ex, ctx) =>
{
HttpResponseParsers.ProcessExpectedStatusCodeNoException(HttpStatusCode.NoContent, resp, NullType.Value, cmd, ex);
return(NullType.Value);
};
return(putCmd);
}
private async Task DispatchWriteAsync(TaskCompletionSource <bool> continuetcs, CancellationToken token)
{
if (this.internalBuffer.Length == 0)
{
if (continuetcs != null)
{
Task.Run(() => continuetcs.TrySetResult(true));
}
return;
}
// bufferToUpload needs to be disposed, or we will leak memory
//
// Unfortunately, because of the async nature of the work, we cannot safely
// put this in a using block, so the Write*Async methods must handle disposal.
MultiBufferMemoryStream bufferToUpload = this.internalBuffer;
this.internalBuffer = new MultiBufferMemoryStream(this.Blob.ServiceClient.BufferManager);
bufferToUpload.Seek(0, SeekOrigin.Begin);
Checksum bufferChecksum = Checksum.None;
if (this.blockChecksum != null)
{
bool computeCRC64 = false;
bool computeMD5 = false;
if (this.blockChecksum.MD5 != null)
{
bufferChecksum.MD5 = this.blockChecksum.MD5.ComputeHash();
computeMD5 = true;
}
if (this.blockChecksum.CRC64 != null)
{
bufferChecksum.CRC64 = this.blockChecksum.CRC64.ComputeHash();
computeCRC64 = true;
}
this.blockChecksum.Dispose();
this.blockChecksum = new ChecksumWrapper(computeMD5, computeCRC64);
}
if (this.blockBlob != null)
{
string blockId = this.GetCurrentBlockId();
this.blockList.Add(blockId);
await this.WriteBlockAsync(continuetcs, bufferToUpload, blockId, bufferChecksum, token).ConfigureAwait(false);
}
else if (this.pageBlob != null)
{
if ((bufferToUpload.Length % Constants.PageSize) != 0)
{
this.lastException = new IOException(SR.InvalidPageSize);
throw this.lastException;
}
long offset = this.currentBlobOffset;
this.currentBlobOffset += bufferToUpload.Length;
await this.WritePagesAsync(continuetcs, bufferToUpload, offset, bufferChecksum, token).ConfigureAwait(false);
}
else
{
long offset = this.currentBlobOffset;
this.currentBlobOffset += bufferToUpload.Length;
// We cannot differentiate between max size condition failing only in the retry versus failing in the first attempt and retry.
// So we will eliminate the latter and handle the former in the append operation callback.
if (this.accessCondition.IfMaxSizeLessThanOrEqual.HasValue && this.currentBlobOffset > this.accessCondition.IfMaxSizeLessThanOrEqual.Value)
{
this.lastException = new IOException(SR.InvalidBlockSize);
throw this.lastException;
}
await this.WriteAppendBlockAsync(continuetcs, bufferToUpload, offset, bufferChecksum, token).ConfigureAwait(false);
}
}
public static T ExecuteSync <T>(RESTCommand <T> cmd, IRetryPolicy policy, OperationContext operationContext)
{
// Note all code below will reference state, not params directly, this will allow common code with async executor
using (ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext))
{
bool shouldRetry = false;
TimeSpan delay = TimeSpan.Zero;
do
{
try
{
executionState.Init();
// 0. Begin Request
Executor.StartRequestAttempt(executionState);
// Steps 1-4
Executor.ProcessStartOfRequest(executionState, SR.TraceStartRequestSync);
Executor.CheckTimeout <T>(executionState, true);
}
catch (Exception ex)
{
Logger.LogError(executionState.OperationContext, SR.TraceInitRequestError, ex.Message);
// Store exception and throw here. All operations in this try would be non-retryable by default. At this point, the request is not even made.
// Therefore, we will not get extended error info. Hence ParseError doesn't matter here.
StorageException storageEx = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
storageEx.IsRetryable = false;
executionState.ExceptionRef = storageEx;
throw executionState.ExceptionRef;
}
// Enter Retryable Section of execution
try
{
// 5. potentially upload data
if (cmd.SendStream != null)
{
executionState.CurrentOperation = ExecutorOperation.BeginGetRequestStream;
Logger.LogInformational(executionState.OperationContext, SR.TracePrepareUpload);
executionState.Req.Timeout = (int)executionState.RemainingTimeout.TotalMilliseconds;
executionState.ReqStream = executionState.Req.GetRequestStream();
executionState.CurrentOperation = ExecutorOperation.BeginUploadRequest;
Logger.LogInformational(executionState.OperationContext, SR.TraceUpload);
MultiBufferMemoryStream multiBufferMemoryStream = cmd.SendStream as MultiBufferMemoryStream;
try
{
if (multiBufferMemoryStream != null && !cmd.SendStreamLength.HasValue)
{
multiBufferMemoryStream.FastCopyTo(executionState.ReqStream, executionState.OperationExpiryTime);
}
else
{
// don't calculate md5 here as we should have already set this for auth purposes
executionState.RestCMD.SendStream.WriteToSync(executionState.ReqStream, cmd.SendStreamLength, null /* maxLength */, false, true, executionState, null /* streamCopyState */);
}
executionState.ReqStream.Flush();
executionState.ReqStream.Dispose();
executionState.ReqStream = null;
}
catch (Exception)
{
executionState.Req.Abort();
throw;
}
}
// 6. Get response
try
{
executionState.CurrentOperation = ExecutorOperation.BeginGetResponse;
Logger.LogInformational(executionState.OperationContext, SR.TraceGetResponse);
executionState.Req.Timeout = (int)executionState.RemainingTimeout.TotalMilliseconds;
executionState.Resp = (HttpWebResponse)executionState.Req.GetResponse();
executionState.CurrentOperation = ExecutorOperation.EndGetResponse;
}
catch (WebException ex)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGetResponseError, ex.Message);
executionState.Resp = (HttpWebResponse)ex.Response;
if (ex.Status == WebExceptionStatus.Timeout || executionState.ReqTimedOut)
{
throw new TimeoutException();
}
if (executionState.Resp == null)
{
throw;
}
else
{
executionState.ExceptionRef = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
}
}
// Response
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
Executor.FireResponseReceived(executionState);
// 7. Do Response parsing (headers etc, no stream available here)
if (cmd.PreProcessResponse != null)
{
executionState.CurrentOperation = ExecutorOperation.PreProcess;
executionState.Result = cmd.PreProcessResponse(cmd, executionState.Resp, executionState.ExceptionRef, executionState.OperationContext);
// clear exception
executionState.ExceptionRef = null;
Logger.LogInformational(executionState.OperationContext, SR.TracePreProcessDone);
}
// 8. (Potentially reads stream from server)
executionState.CurrentOperation = ExecutorOperation.GetResponseStream;
cmd.ResponseStream = executionState.Resp.GetResponseStream();
if (!cmd.RetrieveResponseStream)
{
cmd.DestinationStream = Stream.Null;
}
if (cmd.DestinationStream != null)
{
if (cmd.StreamCopyState == null)
{
cmd.StreamCopyState = new StreamDescriptor();
}
try
{
executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
Logger.LogInformational(executionState.OperationContext, SR.TraceDownload);
cmd.ResponseStream.WriteToSync(cmd.DestinationStream, null /* copyLength */, null /* maxLength */, cmd.CalculateMd5ForResponseStream, false, executionState, cmd.StreamCopyState);
}
finally
{
cmd.ResponseStream.Dispose();
cmd.ResponseStream = null;
}
}
// Step 9 - This will not be called if an exception is raised during stream copying
Executor.ProcessEndOfRequest(executionState);
Executor.FinishRequestAttempt(executionState);
return(executionState.Result);
}
catch (Exception e)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, e.Message);
Executor.FinishRequestAttempt(executionState);
StorageException translatedException = ExecutorBase.TranslateExceptionBasedOnParseError(e, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
executionState.ExceptionRef = translatedException;
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);
shouldRetry = false;
if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
{
executionState.CurrentLocation = Executor.GetNextLocation(executionState.CurrentLocation, cmd.LocationMode);
Logger.LogInformational(executionState.OperationContext, SR.TraceNextLocation, executionState.CurrentLocation);
IExtendedRetryPolicy extendedRetryPolicy = executionState.RetryPolicy as IExtendedRetryPolicy;
if (extendedRetryPolicy != null)
{
RetryContext retryContext = new RetryContext(
executionState.RetryCount++,
cmd.CurrentResult,
executionState.CurrentLocation,
cmd.LocationMode);
RetryInfo retryInfo = extendedRetryPolicy.Evaluate(retryContext, executionState.OperationContext);
if (retryInfo != null)
{
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryInfo, retryInfo.TargetLocation, retryInfo.UpdatedLocationMode);
shouldRetry = true;
executionState.CurrentLocation = retryInfo.TargetLocation;
cmd.LocationMode = retryInfo.UpdatedLocationMode;
delay = retryInfo.RetryInterval;
}
}
else
{
shouldRetry = executionState.RetryPolicy.ShouldRetry(
executionState.RetryCount++,
cmd.CurrentResult.HttpStatusCode,
executionState.ExceptionRef,
out delay,
executionState.OperationContext);
}
if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
{
delay = Constants.MaximumRetryBackoff;
}
}
}
finally
{
if (executionState.Resp != null)
{
executionState.Resp.Close();
executionState.Resp = null;
}
}
if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);
throw executionState.ExceptionRef;
}
else
{
if (executionState.Cmd.RecoveryAction != null)
{
// I.E. Rewind stream etc.
executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
}
if (delay > TimeSpan.Zero)
{
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryDelay, (int)delay.TotalMilliseconds);
Thread.Sleep(delay);
}
Logger.LogInformational(executionState.OperationContext, SR.TraceRetry);
}
Executor.FireRetrying(executionState);
}while (shouldRetry);
}
// should never get here, either return, or throw;
throw new NotImplementedException(SR.InternalStorageError);
}
internal static Tuple <HttpWebRequest, Stream> BuildRequestForTableOperation(Uri uri, UriQueryBuilder builder, IBufferManager bufferManager, int?timeout, TableOperation operation, bool useVersionHeader, OperationContext ctx, TableRequestOptions options)
{
HttpWebRequest msg = BuildRequestCore(uri, builder, operation.HttpMethod, timeout, useVersionHeader, ctx);
TablePayloadFormat payloadFormat = options.PayloadFormat.Value;
// Set Accept and Content-Type based on the payload format.
SetAcceptHeaderForHttpWebRequest(msg, payloadFormat);
Logger.LogInformational(ctx, SR.PayloadFormat, payloadFormat);
msg.Headers.Add(Constants.HeaderConstants.DataServiceVersion, Constants.HeaderConstants.DataServiceVersionValue);
if (operation.HttpMethod != "HEAD" && operation.HttpMethod != "GET")
{
msg.ContentType = Constants.JsonContentTypeHeaderValue;
}
if (operation.OperationType == TableOperationType.InsertOrMerge || operation.OperationType == TableOperationType.Merge)
{
// Client-side encryption is not supported on merge requests.
// This is because we maintain the list of encrypted properties as a property on the entity, and we can't update this
// properly for merge operations.
options.AssertNoEncryptionPolicyOrStrictMode();
// post tunnelling
msg.Headers.Add(Constants.HeaderConstants.PostTunnelling, "MERGE");
}
if (operation.OperationType == TableOperationType.RotateEncryptionKey)
{
// post tunnelling
msg.Headers.Add(Constants.HeaderConstants.PostTunnelling, "MERGE");
}
// etag
if (operation.OperationType == TableOperationType.Delete ||
operation.OperationType == TableOperationType.Replace ||
operation.OperationType == TableOperationType.Merge ||
operation.OperationType == TableOperationType.RotateEncryptionKey)
{
if (operation.ETag != null)
{
msg.Headers.Add(Constants.HeaderConstants.IfMatch, operation.ETag);
}
}
// Prefer header
if (operation.OperationType == TableOperationType.Insert)
{
msg.Headers.Add(Constants.HeaderConstants.Prefer, operation.EchoContent ? Constants.HeaderConstants.PreferReturnContent : Constants.HeaderConstants.PreferReturnNoContent);
}
if (operation.OperationType == TableOperationType.Insert ||
operation.OperationType == TableOperationType.Merge ||
operation.OperationType == TableOperationType.InsertOrMerge ||
operation.OperationType == TableOperationType.InsertOrReplace ||
operation.OperationType == TableOperationType.Replace ||
operation.OperationType == TableOperationType.RotateEncryptionKey)
{
MultiBufferMemoryStream ms = new MultiBufferMemoryStream(bufferManager);
using (JsonTextWriter jsonWriter = new JsonTextWriter(new StreamWriter(new NonCloseableStream(ms))))
{
WriteEntityContent(operation, ctx, options, jsonWriter);
}
ms.Seek(0, SeekOrigin.Begin);
msg.ContentLength = ms.Length;
return(new Tuple <HttpWebRequest, Stream>(msg, ms));
}
return(new Tuple <HttpWebRequest, Stream>(msg, null));
}
internal static StorageRequestMessage BuildRequestForTableBatchOperation <T>(RESTCommand <T> cmd, Uri uri, UriQueryBuilder builder, int?timeout, string tableName, TableBatchOperation batch, CloudTableClient client, HttpContent content, OperationContext ctx, TablePayloadFormat payloadFormat, ICanonicalizer canonicalizer, StorageCredentials credentials)
{
StorageRequestMessage msg = BuildRequestCore(NavigationHelper.AppendPathToSingleUri(uri, "$batch"), builder, HttpMethod.Post, timeout, content, ctx, canonicalizer, credentials);
Logger.LogInformational(ctx, SR.PayloadFormat, payloadFormat);
MultiBufferMemoryStream batchContentStream = new MultiBufferMemoryStream(client.BufferManager);
string batchID = Guid.NewGuid().ToString();
string changesetID = Guid.NewGuid().ToString();
using (StreamWriter contentWriter = new StreamWriter(new NonCloseableStream(batchContentStream)))
{
msg.Headers.Add(Constants.HeaderConstants.DataServiceVersion, Constants.HeaderConstants.DataServiceVersionValue);
string batchSeparator = Constants.BatchSeparator + batchID;
string changesetSeparator = Constants.ChangesetSeparator + changesetID;
string acceptHeader = "Accept: ";
switch (payloadFormat)
{
case TablePayloadFormat.Json:
acceptHeader = acceptHeader + Constants.JsonLightAcceptHeaderValue;
break;
case TablePayloadFormat.JsonFullMetadata:
acceptHeader = acceptHeader + Constants.JsonFullMetadataAcceptHeaderValue;
break;
case TablePayloadFormat.JsonNoMetadata:
acceptHeader = acceptHeader + Constants.JsonNoMetadataAcceptHeaderValue;
break;
}
contentWriter.WriteLine(batchSeparator);
bool isQuery = batch.Count == 1 && batch[0].OperationType == TableOperationType.Retrieve;
// Query operations should not be inside changeset in payload
if (!isQuery)
{
// Start Operation
contentWriter.WriteLine(Constants.ChangesetBoundaryMarker + changesetID);
contentWriter.WriteLine();
}
foreach (TableOperation operation in batch)
{
string httpMethod = operation.OperationType == TableOperationType.Merge || operation.OperationType == TableOperationType.InsertOrMerge ? "MERGE" : operation.HttpMethod.Method;
if (!isQuery)
{
contentWriter.WriteLine(changesetSeparator);
}
contentWriter.WriteLine(Constants.ContentTypeApplicationHttp);
contentWriter.WriteLine(Constants.ContentTransferEncodingBinary);
contentWriter.WriteLine();
string tableURI = Uri.EscapeUriString(operation.GenerateRequestURI(uri, tableName).ToString());
// "EscapeUriString" is almost exactly what we need, except that it contains special logic for
// the percent sign, which results in an off-by-one error in the number of times "%" is encoded.
// This corrects for that.
tableURI = tableURI.Replace(@"%25", @"%");
contentWriter.WriteLine(httpMethod + " " + tableURI + " " + Constants.HTTP1_1);
contentWriter.WriteLine(acceptHeader);
contentWriter.WriteLine(Constants.ContentTypeApplicationJson);
if (operation.OperationType == TableOperationType.Insert)
{
contentWriter.WriteLine(Constants.HeaderConstants.Prefer + @": " + (operation.EchoContent ? Constants.HeaderConstants.PreferReturnContent : Constants.HeaderConstants.PreferReturnNoContent));
}
contentWriter.WriteLine(Constants.HeaderConstants.DataServiceVersion + ": " + Constants.HeaderConstants.DataServiceVersionValue);
// etag
if (operation.OperationType == TableOperationType.Delete ||
operation.OperationType == TableOperationType.Replace ||
operation.OperationType == TableOperationType.Merge)
{
contentWriter.WriteLine(Constants.HeaderConstants.IfMatch + @": " + operation.ETag);
}
contentWriter.WriteLine();
if (operation.OperationType != TableOperationType.Delete && operation.OperationType != TableOperationType.Retrieve)
{
using (JsonTextWriter jsonWriter = new JsonTextWriter(contentWriter))
{
jsonWriter.CloseOutput = false;
WriteEntityContent(operation, ctx, jsonWriter);
}
contentWriter.WriteLine();
}
}
if (!isQuery)
{
contentWriter.WriteLine(changesetSeparator + "--");
}
contentWriter.WriteLine(batchSeparator + "--");
}
batchContentStream.Seek(0, SeekOrigin.Begin);
msg.Content = new StreamContent(batchContentStream);
msg.Content.Headers.ContentLength = batchContentStream.Length;
msg.Content.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse(Constants.BatchBoundaryMarker + batchID);//new System.Net.Http.Headers.MediaTypeHeaderValue(Constants.BatchBoundaryMarker + batchID);
return(msg);
}
public void Dispose()
{
this.msg = null;
this.outStr = null;
this.content = null;
}