/// <summary>
/// Creates and initializes a new asynchronous copy operation.
/// </summary>
/// <param name="src">The source stream.</param>
/// <param name="dest">The destination stream.</param>
/// <param name="state">An ExecutionState used to coordinate copy operation.</param>
/// <param name="buffSize">Size of read and write buffers used to move data.</param>
/// <param name="calculateMd5">Boolean value indicating whether the MD-5 should be calculated.</param>
/// <param name="streamCopyState">An object that represents the state for the current operation.</param>
public AsyncStreamCopier(Stream src, Stream dest, ExecutionState <T> state, int buffSize, bool calculateMd5, StreamDescriptor streamCopyState)
{
this.src = src;
this.dest = dest;
this.state = state;
this.buffSize = buffSize;
this.streamCopyState = streamCopyState;
if (streamCopyState != null && calculateMd5 && streamCopyState.Md5HashRef == null)
{
streamCopyState.Md5HashRef = new MD5Wrapper();
}
}
/// <summary>
/// Creates and initializes a new asynchronous copy operation.
/// </summary>
/// <param name="src">The source stream.</param>
/// <param name="dest">The destination stream.</param>
/// <param name="state">An ExecutionState used to coordinate copy operation.</param>
/// <param name="bufferManager">IBufferManager instance to use. May be null.</param>
/// <param name="buffSize">Size of read and write buffers used to move data. Overrides the default buffer size of bufferManager.</param>
/// <param name="calculateMd5">Boolean value indicating whether the MD-5 should be calculated.</param>
/// <param name="streamCopyState">An object that represents the state for the current operation.</param>
public AsyncStreamCopier(Stream src, Stream dest, ExecutionState <T> state, IBufferManager bufferManager, int?buffSize, bool calculateMd5, StreamDescriptor streamCopyState)
{
this.src = src;
this.dest = dest;
this.state = state;
this.bufferManager = bufferManager;
this.buffSize = buffSize ?? (bufferManager != null ? bufferManager.GetDefaultBufferSize() : Constants.DefaultBufferSize);
this.streamCopyState = streamCopyState;
if (streamCopyState != null && calculateMd5 && streamCopyState.Md5HashRef == null)
{
streamCopyState.Md5HashRef = new MD5Wrapper();
}
}
public AsyncStreamCopier(Stream src, Stream dest, ExecutionState <T> state, int buffSize, bool calculateMd5, OperationContext operationContext, StreamDescriptor streamCopyState)
{
this.src = src;
this.dest = dest;
this.state = state;
this.currentReadBuff = new byte[buffSize];
this.currentWriteBuff = new byte[buffSize];
this.operationContext = operationContext;
this.streamCopyState = streamCopyState;
if (streamCopyState != null && calculateMd5 && streamCopyState.Md5HashRef == null)
{
streamCopyState.Md5HashRef = new MD5Wrapper();
}
}
private static void ValidateCopier(int bufferLength, bool inputSync, int inputDelayInMs, bool outputSync, int outputDelayInMs)
{
byte[] buffer = new byte[bufferLength];
new Random().NextBytes(buffer);
int expectedCallCount = buffer.Length / Constants.DefaultBufferSize;
int totalDelayInMs = (expectedCallCount + 1) * inputDelayInMs + expectedCallCount * outputDelayInMs;
DataValidationStream input = new DataValidationStream(buffer, inputSync, inputDelayInMs);
DataValidationStream output = new DataValidationStream(buffer, outputSync, outputDelayInMs);
OperationContext opContext = new OperationContext();
ExecutionState <NullType> state = new ExecutionState <NullType>(null, new NoRetry(), opContext);
StreamDescriptor copyState = new StreamDescriptor();
using (ManualResetEvent waitHandle = new ManualResetEvent(false))
{
input.WriteToAsync(output, null, null, false, state, opContext, copyState, _ =>
{
waitHandle.Set();
});
Assert.IsTrue(waitHandle.WaitOne(totalDelayInMs + 10 * 1000));
}
Assert.AreEqual(output.LastException, state.ExceptionRef);
if (output.LastException != null)
{
throw output.LastException;
}
Assert.AreEqual(buffer.Length, copyState.Length);
Assert.AreEqual(expectedCallCount + 1, input.ReadCallCount);
Assert.AreEqual(expectedCallCount, output.WriteCallCount);
}
private static void ValidateCopier(int bufferLength, long? copyLength, long? maxLength, bool inputSync, int inputDelayInMs, int inputFailRequest, bool outputSync, int outputDelayInMs, int outputFailRequest, DateTime? copyTimeout, bool seekable)
{
byte[] buffer = GetRandomBuffer(bufferLength);
// Finds ceiling of division operation
int expectedCallCount =
(int)
(copyLength.HasValue && buffer.Length > copyLength
? (-1L + copyLength + Constants.DefaultBufferSize) / Constants.DefaultBufferSize
: (-1L + buffer.Length + Constants.DefaultBufferSize) / Constants.DefaultBufferSize);
int totalDelayInMs = (expectedCallCount + 1) * inputDelayInMs + expectedCallCount * outputDelayInMs;
DataValidationStream input = new DataValidationStream(buffer, inputSync, inputDelayInMs, inputFailRequest, seekable);
DataValidationStream output = new DataValidationStream(buffer, outputSync, outputDelayInMs, outputFailRequest, seekable);
RESTCommand<NullType> cmdWithTimeout = new RESTCommand<NullType>(new StorageCredentials(), null) { OperationExpiryTime = copyTimeout };
ExecutionState<NullType> state = new ExecutionState<NullType>(cmdWithTimeout, null, null);
StreamDescriptor copyState = new StreamDescriptor();
using (ManualResetEvent waitHandle = new ManualResetEvent(false))
{
input.WriteToAsync(output, copyLength, maxLength, false, state, copyState, _ => waitHandle.Set());
Assert.IsTrue(waitHandle.WaitOne(totalDelayInMs + 10 * 1000));
}
if (inputFailRequest >= 0)
{
Assert.AreEqual(input.LastException, state.ExceptionRef);
Assert.AreEqual(inputFailRequest, input.ReadCallCount);
}
if (outputFailRequest >= 0)
{
Assert.AreEqual(output.LastException, state.ExceptionRef);
Assert.AreEqual(outputFailRequest, output.WriteCallCount);
}
if (state.ExceptionRef != null)
{
throw state.ExceptionRef;
}
Assert.AreEqual(copyLength.HasValue ? copyLength : buffer.Length, copyState.Length);
Assert.AreEqual(copyLength.HasValue ? expectedCallCount : expectedCallCount + 1, input.ReadCallCount);
Assert.AreEqual(expectedCallCount, output.WriteCallCount);
}
public IAsyncAction WritePagesAsync(IInputStream pageData, long startOffset, string contentMD5, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext)
{
BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.PageBlob, this.ServiceClient);
bool requiresContentMD5 = (contentMD5 == null) && modifiedOptions.UseTransactionalMD5.Value;
operationContext = operationContext ?? new OperationContext();
return AsyncInfo.Run(async (token) =>
{
Stream pageDataAsStream = pageData.AsStreamForRead();
Stream seekableStream = pageDataAsStream;
if (!pageDataAsStream.CanSeek || requiresContentMD5)
{
DateTime streamCopyStartTime = DateTime.Now;
CancellationToken streamCopyToken = token;
if (modifiedOptions.MaximumExecutionTime.HasValue)
{
// Setup token
CancellationTokenSource cts = new CancellationTokenSource(modifiedOptions.MaximumExecutionTime.Value);
CancellationToken tokenWithTimeout = cts.Token;
// Hookup users token
token.Register(() => cts.Cancel());
streamCopyToken = tokenWithTimeout;
}
Stream writeToStream;
if (pageDataAsStream.CanSeek)
{
writeToStream = Stream.Null;
}
else
{
seekableStream = new MemoryStream();
writeToStream = seekableStream;
}
OperationContext tempOperationContext = new OperationContext();
StreamDescriptor streamCopyState = new StreamDescriptor();
long startPosition = seekableStream.Position;
await pageDataAsStream.WriteToAsync(writeToStream, Constants.MaxBlockSize, requiresContentMD5, tempOperationContext, streamCopyState, streamCopyToken);
seekableStream.Position = startPosition;
if (requiresContentMD5)
{
contentMD5 = streamCopyState.Md5;
}
if (modifiedOptions.MaximumExecutionTime.HasValue)
{
modifiedOptions.MaximumExecutionTime -= DateTime.Now.Subtract(streamCopyStartTime);
}
}
await Executor.ExecuteAsyncNullReturn(
this.PutPageImpl(seekableStream, startOffset, contentMD5, accessCondition, modifiedOptions),
modifiedOptions.RetryPolicy,
operationContext,
token);
});
}
/// <summary>
/// Asynchronously reads the entire content of the stream and writes it to the given output stream.
/// </summary>
/// <param name="stream">The origin stream.</param>
/// <param name="toStream">The destination stream.</param>
/// <param name="copyLength">Number of bytes to copy from source stream to destination stream. Cannot be passed with a value for maxLength.</param>
/// <param name="maxLength">Maximum length of the source stream. Cannot be passed with a value for copyLength.</param>
/// <param name="calculateMd5">Bool value indicating whether the Md5 should be calculated.</param>
/// <param name="executionState">An object that stores state of the operation.</param>
/// <param name="streamCopyState">An object that represents the current state for the copy operation.</param>
/// <param name="token">A CancellationToken to observe while waiting for the copy to complete.</param>
/// <returns>The task object representing the asynchronous operation.</returns>
internal static async Task WriteToAsync <T>(this Stream stream, Stream toStream, long?copyLength, long?maxLength, bool calculateMd5, ExecutionState <T> executionState, StreamDescriptor streamCopyState, CancellationToken token)
{
if (copyLength.HasValue && maxLength.HasValue)
{
throw new ArgumentException(SR.StreamLengthMismatch);
}
if (stream.CanSeek && maxLength.HasValue && stream.Length - stream.Position > maxLength)
{
throw new InvalidOperationException(SR.StreamLengthError);
}
if (stream.CanSeek && copyLength.HasValue && stream.Length - stream.Position < copyLength)
{
throw new ArgumentOutOfRangeException("copyLength", SR.StreamLengthShortError);
}
if (streamCopyState != null && calculateMd5 && streamCopyState.Md5HashRef == null)
{
streamCopyState.Md5HashRef = new MD5Wrapper();
}
CancellationTokenSource cts = null;
try
{
if (executionState.OperationExpiryTime.HasValue)
{
// Setup token for timeout
cts = CancellationTokenSource.CreateLinkedTokenSource(token);
cts.CancelAfter(executionState.RemainingTimeout);
// Switch tokens
token = cts.Token;
}
byte[] buffer = new byte[GetBufferSize(stream)];
long? bytesRemaining = copyLength;
int readCount;
do
{
// Determine how many bytes to read this time so that no more than count bytes are read
int bytesToRead = bytesRemaining.HasValue && bytesRemaining < buffer.Length ? (int)bytesRemaining : buffer.Length;
if (bytesToRead == 0)
{
break;
}
readCount = await stream.ReadAsync(buffer, 0, bytesToRead, token);
if (bytesRemaining.HasValue)
{
bytesRemaining -= readCount;
}
if (readCount > 0)
{
await toStream.WriteAsync(buffer, 0, readCount, token);
// Update the StreamDescriptor after the bytes are successfully committed to the output stream
if (streamCopyState != null)
{
streamCopyState.Length += readCount;
if (maxLength.HasValue && streamCopyState.Length > maxLength.Value)
{
throw new InvalidOperationException(SR.StreamLengthError);
}
if (streamCopyState.Md5HashRef != null)
{
streamCopyState.Md5HashRef.UpdateHash(buffer, 0, readCount);
}
}
}
}while (readCount > 0);
if (bytesRemaining.HasValue && bytesRemaining != 0)
{
throw new ArgumentOutOfRangeException("copyLength", SR.StreamLengthShortError);
}
}
finally
{
if (cts != null)
{
cts.Dispose();
cts = null;
}
}
// Streams opened with AsStreamForWrite extension need to be flushed
// to write all buffered data to the underlying Windows Runtime stream.
await toStream.FlushAsync();
if (streamCopyState != null && streamCopyState.Md5HashRef != null)
{
streamCopyState.Md5 = streamCopyState.Md5HashRef.ComputeHash();
streamCopyState.Md5HashRef = null;
}
}
internal static void WriteToSync <T>(this Stream stream, Stream toStream, long?copyLength, long?maxLength, bool calculateMd5, bool syncRead, ExecutionState <T> executionState, StreamDescriptor streamCopyState)
{
if (copyLength.HasValue && maxLength.HasValue)
{
throw new ArgumentException(SR.StreamLengthMismatch);
}
if (stream.CanSeek && maxLength.HasValue && stream.Length - stream.Position > maxLength)
{
throw new InvalidOperationException(SR.StreamLengthError);
}
if (stream.CanSeek && copyLength.HasValue && stream.Length - stream.Position < copyLength)
{
throw new ArgumentOutOfRangeException("copyLength", SR.StreamLengthShortError);
}
byte[] buffer = new byte[GetBufferSize(stream)];
if (streamCopyState != null && calculateMd5 && streamCopyState.Md5HashRef == null)
{
streamCopyState.Md5HashRef = new MD5Wrapper();
}
RegisteredWaitHandle waitHandle = null;
ManualResetEvent completedEvent = null;
if (!syncRead && executionState.OperationExpiryTime.HasValue)
{
completedEvent = new ManualResetEvent(false);
waitHandle = ThreadPool.RegisterWaitForSingleObject(
completedEvent,
StreamExtensions.MaximumCopyTimeCallback <T>,
executionState,
executionState.RemainingTimeout,
true);
}
try
{
long?bytesRemaining = copyLength;
int readCount;
do
{
if (executionState.OperationExpiryTime.HasValue && DateTime.Now.CompareTo(executionState.OperationExpiryTime.Value) > 0)
{
throw Exceptions.GenerateTimeoutException(executionState.Cmd != null ? executionState.Cmd.CurrentResult : null, null);
}
// Determine how many bytes to read this time so that no more than copyLength bytes are read
int bytesToRead = MinBytesToRead(bytesRemaining, buffer.Length);
if (bytesToRead == 0)
{
break;
}
// Read synchronously or asynchronously
readCount = syncRead
? stream.Read(buffer, 0, bytesToRead)
: stream.EndRead(stream.BeginRead(buffer, 0, bytesToRead, null /* Callback */, null /* State */));
// Decrement bytes to write from bytes read
if (bytesRemaining.HasValue)
{
bytesRemaining -= readCount;
}
// Write
if (readCount > 0)
{
toStream.Write(buffer, 0, readCount);
// Update the StreamDescriptor after the bytes are successfully committed to the output stream
if (streamCopyState != null)
{
streamCopyState.Length += readCount;
if (maxLength.HasValue && streamCopyState.Length > maxLength.Value)
{
throw new InvalidOperationException(SR.StreamLengthError);
}
if (streamCopyState.Md5HashRef != null)
{
streamCopyState.Md5HashRef.UpdateHash(buffer, 0, readCount);
}
}
}
}while (readCount != 0);
if (bytesRemaining.HasValue && bytesRemaining != 0)
{
throw new ArgumentOutOfRangeException("copyLength", SR.StreamLengthShortError);
}
}
catch (Exception)
{
if (executionState.OperationExpiryTime.HasValue && DateTime.Now.CompareTo(executionState.OperationExpiryTime.Value) > 0)
{
throw Exceptions.GenerateTimeoutException(executionState.Cmd != null ? executionState.Cmd.CurrentResult : null, null);
}
else
{
throw;
}
}
finally
{
if (waitHandle != null)
{
waitHandle.Unregister(null);
}
if (completedEvent != null)
{
completedEvent.Close();
}
}
if (streamCopyState != null && streamCopyState.Md5HashRef != null)
{
streamCopyState.Md5 = streamCopyState.Md5HashRef.ComputeHash();
streamCopyState.Md5HashRef = null;
}
}
public ICancellableAsyncResult BeginWritePages(Stream pageData, long startOffset, string contentMD5, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext, AsyncCallback callback, object state)
{
BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.PageBlob, this.ServiceClient);
bool requiresContentMD5 = (contentMD5 == null) && modifiedOptions.UseTransactionalMD5.Value;
operationContext = operationContext ?? new OperationContext();
ChainedAsyncResult<NullType> chainedResult = new ChainedAsyncResult<NullType>(callback, state);
if (pageData.CanSeek && !requiresContentMD5)
{
this.WritePagesHandler(pageData, startOffset, contentMD5, accessCondition, modifiedOptions, operationContext, chainedResult);
}
else
{
DateTime? expiryTime = modifiedOptions.MaximumExecutionTime.HasValue
? DateTime.Now + modifiedOptions.MaximumExecutionTime.Value
: (DateTime?)null;
OperationContext tempOperationContext = new OperationContext();
ExecutionState<NullType> executionState = new ExecutionState<NullType>(null /* cmd */, modifiedOptions.RetryPolicy, tempOperationContext);
chainedResult.CancelDelegate = executionState.Cancel;
Stream seekableStream;
Stream writeToStream;
if (pageData.CanSeek)
{
seekableStream = pageData;
writeToStream = Stream.Null;
}
else
{
seekableStream = new MemoryStream();
writeToStream = seekableStream;
}
long startPosition = seekableStream.Position;
StreamDescriptor streamCopyState = new StreamDescriptor();
pageData.WriteToAsync(
writeToStream,
Constants.MaxBlockSize,
expiryTime,
requiresContentMD5,
executionState,
tempOperationContext,
streamCopyState,
_ =>
{
chainedResult.UpdateCompletedSynchronously(executionState.CompletedSynchronously);
if (executionState.ExceptionRef != null)
{
chainedResult.OnComplete(executionState.ExceptionRef);
}
else
{
try
{
if (requiresContentMD5)
{
contentMD5 = streamCopyState.Md5;
}
seekableStream.Position = startPosition;
this.WritePagesHandler(seekableStream, startOffset, contentMD5, accessCondition, modifiedOptions, operationContext, chainedResult);
}
catch (Exception e)
{
chainedResult.OnComplete(e);
}
}
});
}
return chainedResult;
}
internal static void WriteToAsync <T>(this Stream stream, Stream toStream, long?maxLength, DateTime?expiryTime, bool calculateMd5, ExecutionState <T> executionState, OperationContext operationContext, StreamDescriptor streamCopyState, Action <ExecutionState <T> > completed)
{
AsyncStreamCopier <T> copier = new AsyncStreamCopier <T>(stream, toStream, executionState, GetBufferSize(stream), calculateMd5, operationContext, streamCopyState);
copier.StartCopyStream(completed, maxLength, expiryTime);
}
public ICancellableAsyncResult BeginPutBlock(string blockId, Stream blockData, string contentMD5, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext, AsyncCallback callback, object state)
{
CommonUtility.AssertNotNull("blockData", blockData);
BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient);
bool requiresContentMD5 = (contentMD5 == null) && modifiedOptions.UseTransactionalMD5.Value;
operationContext = operationContext ?? new OperationContext();
StorageAsyncResult<NullType> storageAsyncResult = new StorageAsyncResult<NullType>(callback, state);
if (blockData.CanSeek && !requiresContentMD5)
{
this.PutBlockHandler(blockId, blockData, contentMD5, accessCondition, modifiedOptions, operationContext, storageAsyncResult);
}
else
{
ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions);
storageAsyncResult.CancelDelegate = tempExecutionState.Cancel;
Stream seekableStream;
Stream writeToStream;
if (blockData.CanSeek)
{
seekableStream = blockData;
writeToStream = Stream.Null;
}
else
{
seekableStream = new MultiBufferMemoryStream(this.ServiceClient.BufferManager);
writeToStream = seekableStream;
}
long startPosition = seekableStream.Position;
StreamDescriptor streamCopyState = new StreamDescriptor();
blockData.WriteToAsync(
writeToStream,
null /* copyLength */,
Constants.MaxBlockSize,
requiresContentMD5,
tempExecutionState,
streamCopyState,
completedState =>
{
storageAsyncResult.UpdateCompletedSynchronously(completedState.CompletedSynchronously);
if (completedState.ExceptionRef != null)
{
storageAsyncResult.OnComplete(completedState.ExceptionRef);
}
else
{
try
{
if (requiresContentMD5)
{
contentMD5 = streamCopyState.Md5;
}
seekableStream.Position = startPosition;
this.PutBlockHandler(blockId, seekableStream, contentMD5, accessCondition, modifiedOptions, operationContext, storageAsyncResult);
}
catch (Exception e)
{
storageAsyncResult.OnComplete(e);
}
}
});
}
return storageAsyncResult;
}
internal ICancellableAsyncResult BeginUploadFromStreamHelper(Stream source, long? length, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext, AsyncCallback callback, object state)
{
CommonUtility.AssertNotNull("source", source);
if (length.HasValue)
{
CommonUtility.AssertInBounds("length", length.Value, 1);
if (source.CanSeek && length > source.Length - source.Position)
{
throw new ArgumentOutOfRangeException("length", SR.StreamLengthShortError);
}
}
this.attributes.AssertNoSnapshot();
BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient);
ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions);
StorageAsyncResult<NullType> storageAsyncResult = new StorageAsyncResult<NullType>(callback, state);
bool lessThanSingleBlobThreshold = source.CanSeek &&
(length ?? source.Length - source.Position) <= this.ServiceClient.SingleBlobUploadThresholdInBytes;
if (this.ServiceClient.ParallelOperationThreadCount == 1 && lessThanSingleBlobThreshold)
{
if (modifiedOptions.StoreBlobContentMD5.Value)
{
long startPosition = source.Position;
StreamDescriptor streamCopyState = new StreamDescriptor();
source.WriteToAsync(
Stream.Null,
length,
null /* maxLength */,
true,
tempExecutionState,
streamCopyState,
completedState =>
{
storageAsyncResult.UpdateCompletedSynchronously(completedState.CompletedSynchronously);
try
{
lock (storageAsyncResult.CancellationLockerObject)
{
storageAsyncResult.CancelDelegate = null;
if (completedState.ExceptionRef != null)
{
storageAsyncResult.OnComplete(completedState.ExceptionRef);
}
else
{
source.Position = startPosition;
this.UploadFromStreamHandler(
source,
length,
streamCopyState.Md5,
accessCondition,
operationContext,
modifiedOptions,
storageAsyncResult);
}
}
}
catch (Exception e)
{
storageAsyncResult.OnComplete(e);
}
});
// We do not need to do this inside a lock, as storageAsyncResult is
// not returned to the user yet.
storageAsyncResult.CancelDelegate = tempExecutionState.Cancel;
}
else
{
if (modifiedOptions.UseTransactionalMD5.Value)
{
throw new ArgumentException(SR.PutBlobNeedsStoreBlobContentMD5, "options");
}
this.UploadFromStreamHandler(
source,
length,
null /* contentMD5 */,
accessCondition,
operationContext,
modifiedOptions,
storageAsyncResult);
}
}
else
{
ICancellableAsyncResult result = this.BeginOpenWrite(
accessCondition,
modifiedOptions,
operationContext,
ar =>
{
storageAsyncResult.UpdateCompletedSynchronously(ar.CompletedSynchronously);
lock (storageAsyncResult.CancellationLockerObject)
{
storageAsyncResult.CancelDelegate = null;
try
{
CloudBlobStream blobStream = this.EndOpenWrite(ar);
storageAsyncResult.OperationState = blobStream;
source.WriteToAsync(
blobStream,
length,
null /* maxLength */,
false,
tempExecutionState,
null /* streamCopyState */,
completedState =>
{
storageAsyncResult.UpdateCompletedSynchronously(completedState.CompletedSynchronously);
if (completedState.ExceptionRef != null)
{
storageAsyncResult.OnComplete(completedState.ExceptionRef);
}
else
{
try
{
lock (storageAsyncResult.CancellationLockerObject)
{
storageAsyncResult.CancelDelegate = null;
ICancellableAsyncResult commitResult = blobStream.BeginCommit(
CloudBlobSharedImpl.BlobOutputStreamCommitCallback,
storageAsyncResult);
storageAsyncResult.CancelDelegate = commitResult.Cancel;
if (storageAsyncResult.CancelRequested)
{
storageAsyncResult.Cancel();
}
}
}
catch (Exception e)
{
storageAsyncResult.OnComplete(e);
}
}
});
storageAsyncResult.CancelDelegate = tempExecutionState.Cancel;
if (storageAsyncResult.CancelRequested)
{
storageAsyncResult.Cancel();
}
}
catch (Exception e)
{
storageAsyncResult.OnComplete(e);
}
}
},
null /* state */);
// We do not need to do this inside a lock, as storageAsyncResult is
// not returned to the user yet.
storageAsyncResult.CancelDelegate = result.Cancel;
}
return storageAsyncResult;
}
public virtual Task WritePagesAsync(Stream pageData, long startOffset, string contentMD5, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken)
{
BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.PageBlob, this.ServiceClient);
bool requiresContentMD5 = (contentMD5 == null) && modifiedOptions.UseTransactionalMD5.Value;
operationContext = operationContext ?? new OperationContext();
ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions);
return Task.Run(async () =>
{
Stream pageDataAsStream = pageData;
Stream seekableStream = pageDataAsStream;
bool seekableStreamCreated = false;
try
{
if (!pageDataAsStream.CanSeek || requiresContentMD5)
{
Stream writeToStream;
if (pageDataAsStream.CanSeek)
{
writeToStream = Stream.Null;
}
else
{
seekableStream = new MultiBufferMemoryStream(this.ServiceClient.BufferManager);
seekableStreamCreated = true;
writeToStream = seekableStream;
}
StreamDescriptor streamCopyState = new StreamDescriptor();
long startPosition = seekableStream.Position;
await pageDataAsStream.WriteToAsync(writeToStream, null /* copyLength */, Constants.MaxBlockSize, requiresContentMD5, tempExecutionState, streamCopyState, cancellationToken);
seekableStream.Position = startPosition;
if (requiresContentMD5)
{
contentMD5 = streamCopyState.Md5;
}
}
await Executor.ExecuteAsyncNullReturn(
this.PutPageImpl(seekableStream, startOffset, contentMD5, accessCondition, modifiedOptions),
modifiedOptions.RetryPolicy,
operationContext,
cancellationToken);
}
finally
{
if (seekableStreamCreated)
{
seekableStream.Dispose();
}
}
}, cancellationToken);
}
private static void ValidateCopier(int bufferLength, bool inputSync, int inputDelayInMs, bool outputSync, int outputDelayInMs)
{
byte[] buffer = new byte[bufferLength];
new Random().NextBytes(buffer);
int expectedCallCount = buffer.Length / Constants.DefaultBufferSize;
int totalDelayInMs = (expectedCallCount + 1) * inputDelayInMs + expectedCallCount * outputDelayInMs;
DataValidationStream input = new DataValidationStream(buffer, inputSync, inputDelayInMs);
DataValidationStream output = new DataValidationStream(buffer, outputSync, outputDelayInMs);
OperationContext opContext = new OperationContext();
ExecutionState<NullType> state = new ExecutionState<NullType>(null, new NoRetry(), opContext);
StreamDescriptor copyState = new StreamDescriptor();
using (ManualResetEvent waitHandle = new ManualResetEvent(false))
{
input.WriteToAsync(output, null, null, false, state, opContext, copyState, _ =>
{
waitHandle.Set();
});
Assert.IsTrue(waitHandle.WaitOne(totalDelayInMs + 10 * 1000));
}
Assert.AreEqual(output.LastException, state.ExceptionRef);
if (output.LastException != null)
{
throw output.LastException;
}
Assert.AreEqual(buffer.Length, copyState.Length);
Assert.AreEqual(expectedCallCount + 1, input.ReadCallCount);
Assert.AreEqual(expectedCallCount, output.WriteCallCount);
}
public IAsyncAction PutBlockAsync(string blockId, IInputStream blockData, string contentMD5, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext)
{
BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient);
bool requiresContentMD5 = (contentMD5 == null) && modifiedOptions.UseTransactionalMD5.Value;
operationContext = operationContext ?? new OperationContext();
ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions);
return AsyncInfo.Run(async (token) =>
{
Stream blockDataAsStream = blockData.AsStreamForRead();
Stream seekableStream = blockDataAsStream;
if (!blockDataAsStream.CanSeek || requiresContentMD5)
{
Stream writeToStream;
if (blockDataAsStream.CanSeek)
{
writeToStream = Stream.Null;
}
else
{
seekableStream = new MultiBufferMemoryStream(this.ServiceClient.BufferManager);
writeToStream = seekableStream;
}
StreamDescriptor streamCopyState = new StreamDescriptor();
long startPosition = seekableStream.Position;
await blockDataAsStream.WriteToAsync(writeToStream, null /* copyLength */, Constants.MaxBlockSize, requiresContentMD5, tempExecutionState, streamCopyState, token);
seekableStream.Position = startPosition;
if (requiresContentMD5)
{
contentMD5 = streamCopyState.Md5;
}
}
await Executor.ExecuteAsyncNullReturn(
this.PutBlockImpl(seekableStream, blockId, contentMD5, accessCondition, modifiedOptions),
modifiedOptions.RetryPolicy,
operationContext,
token);
});
}
internal IAsyncAction UploadFromStreamAsyncHelper(IInputStream source, long? length, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext)
{
CommonUtility.AssertNotNull("source", source);
Stream sourceAsStream = source.AsStreamForRead();
if (length.HasValue)
{
CommonUtility.AssertInBounds("length", length.Value, 1);
if (sourceAsStream.CanSeek && length > sourceAsStream.Length - sourceAsStream.Position)
{
throw new ArgumentOutOfRangeException("length", SR.StreamLengthShortError);
}
}
this.attributes.AssertNoSnapshot();
BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient);
operationContext = operationContext ?? new OperationContext();
ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions);
return AsyncInfo.Run(async (token) =>
{
bool lessThanSingleBlobThreshold = sourceAsStream.CanSeek
&& (length ?? sourceAsStream.Length - sourceAsStream.Position)
<= this.ServiceClient.SingleBlobUploadThresholdInBytes;
if (this.ServiceClient.ParallelOperationThreadCount == 1 && lessThanSingleBlobThreshold)
{
string contentMD5 = null;
if (modifiedOptions.StoreBlobContentMD5.Value)
{
StreamDescriptor streamCopyState = new StreamDescriptor();
long startPosition = sourceAsStream.Position;
await sourceAsStream.WriteToAsync(Stream.Null, length, null /* maxLength */, true, tempExecutionState, streamCopyState, token);
sourceAsStream.Position = startPosition;
contentMD5 = streamCopyState.Md5;
}
else
{
if (modifiedOptions.UseTransactionalMD5.Value)
{
throw new ArgumentException(SR.PutBlobNeedsStoreBlobContentMD5, "options");
}
}
await Executor.ExecuteAsyncNullReturn(
this.PutBlobImpl(sourceAsStream, length, contentMD5, accessCondition, modifiedOptions),
modifiedOptions.RetryPolicy,
operationContext,
token);
}
else
{
using (ICloudBlobStream blobStream = await this.OpenWriteAsync(accessCondition, options, operationContext).AsTask(token))
{
// We should always call AsStreamForWrite with bufferSize=0 to prevent buffering. Our
// stream copier only writes 64K buffers at a time anyway, so no buffering is needed.
await sourceAsStream.WriteToAsync(blobStream.AsStreamForWrite(0), length, null /* maxLength */, false, tempExecutionState, null /* streamCopyState */, token);
await blobStream.CommitAsync().AsTask(token);
}
}
});
}
public void WriteToMultiBufferMemoryStreamTestAPM()
{
using (AutoResetEvent waitHandle = new AutoResetEvent(false))
{
byte[] buffer = GetRandomBuffer(1 * 1024 * 1024);
MemoryStream stream1 = new MemoryStream(buffer);
RESTCommand<NullType> cmd = new RESTCommand<NullType>(TestBase.StorageCredentials, null);
ExecutionState<NullType> state = new ExecutionState<NullType>(cmd, new NoRetry(), new OperationContext());
StreamDescriptor copyState = new StreamDescriptor();
MultiBufferMemoryStream stream2 = new MultiBufferMemoryStream(null /* bufferManager */);
stream1.WriteToAsync(stream2, null, null, false, state, copyState, _ => waitHandle.Set());
waitHandle.WaitOne();
if (state.ExceptionRef != null)
{
throw state.ExceptionRef;
}
stream1.Seek(0, SeekOrigin.Begin);
stream2.Seek(0, SeekOrigin.Begin);
TestHelper.AssertStreamsAreEqual(stream1, stream2);
MultiBufferMemoryStream stream3 = new MultiBufferMemoryStream(null /* bufferManager */);
IAsyncResult ar = stream2.BeginFastCopyTo(stream3, DateTime.Now.AddMinutes(-1), _ => waitHandle.Set(), null);
waitHandle.WaitOne();
TestHelper.ExpectedException<TimeoutException>(
() => stream3.EndFastCopyTo(ar),
"Past expiration time should immediately fail");
stream2.Seek(0, SeekOrigin.Begin);
stream3.Seek(0, SeekOrigin.Begin);
ar = stream2.BeginFastCopyTo(stream3, DateTime.Now.AddHours(1), _ => waitHandle.Set(), null);
waitHandle.WaitOne();
stream2.EndFastCopyTo(ar);
stream2.Seek(0, SeekOrigin.Begin);
stream3.Seek(0, SeekOrigin.Begin);
TestHelper.AssertStreamsAreEqual(stream2, stream3);
MultiBufferMemoryStream stream4 = new MultiBufferMemoryStream(null, 12345);
ar = stream3.BeginFastCopyTo(stream4, null, _ => waitHandle.Set(), null);
waitHandle.WaitOne();
stream3.EndFastCopyTo(ar);
stream3.Seek(0, SeekOrigin.Begin);
stream4.Seek(0, SeekOrigin.Begin);
TestHelper.AssertStreamsAreEqual(stream3, stream4);
state = new ExecutionState<NullType>(cmd, new NoRetry(), new OperationContext());
copyState = new StreamDescriptor();
MemoryStream stream5 = new MemoryStream();
stream4.WriteToAsync(stream5, null, null, false, state, copyState, _ => waitHandle.Set());
waitHandle.WaitOne();
if (state.ExceptionRef != null)
{
throw state.ExceptionRef;
}
stream4.Seek(0, SeekOrigin.Begin);
stream5.Seek(0, SeekOrigin.Begin);
TestHelper.AssertStreamsAreEqual(stream4, stream5);
TestHelper.AssertStreamsAreEqual(stream1, stream5);
}
}
public void PutBlock(string blockId, Stream blockData, string contentMD5, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null)
{
CommonUtility.AssertNotNull("blockData", blockData);
BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient);
bool requiresContentMD5 = (contentMD5 == null) && modifiedOptions.UseTransactionalMD5.Value;
operationContext = operationContext ?? new OperationContext();
Stream seekableStream = blockData;
if (!blockData.CanSeek || requiresContentMD5)
{
ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions);
Stream writeToStream;
if (blockData.CanSeek)
{
writeToStream = Stream.Null;
}
else
{
seekableStream = new MultiBufferMemoryStream(this.ServiceClient.BufferManager);
writeToStream = seekableStream;
}
long startPosition = seekableStream.Position;
StreamDescriptor streamCopyState = new StreamDescriptor();
blockData.WriteToSync(writeToStream, null /* copyLength */, Constants.MaxBlockSize, requiresContentMD5, true, tempExecutionState, streamCopyState);
seekableStream.Position = startPosition;
if (requiresContentMD5)
{
contentMD5 = streamCopyState.Md5;
}
}
Executor.ExecuteSync(
this.PutBlockImpl(seekableStream, blockId, contentMD5, accessCondition, modifiedOptions),
modifiedOptions.RetryPolicy,
operationContext);
}
public void UploadFromStream(Stream source, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null)
{
CommonUtils.AssertNotNull("source", source);
this.attributes.AssertNoSnapshot();
BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient);
operationContext = operationContext ?? new OperationContext();
DateTime? expiryTime = modifiedOptions.MaximumExecutionTime.HasValue
? DateTime.Now + modifiedOptions.MaximumExecutionTime.Value
: (DateTime?)null;
if ((this.ServiceClient.ParallelOperationThreadCount == 1) &&
source.CanSeek &&
((source.Length - source.Position) <= this.ServiceClient.SingleBlobUploadThresholdInBytes))
{
string contentMD5 = null;
if (modifiedOptions.StoreBlobContentMD5.Value)
{
OperationContext tempOperationContext = new OperationContext();
StreamDescriptor streamCopyState = new StreamDescriptor();
long startPosition = source.Position;
source.WriteToSync(Stream.Null, null /* maxLength */, expiryTime, true, true, tempOperationContext, streamCopyState);
source.Position = startPosition;
contentMD5 = streamCopyState.Md5;
}
Executor.ExecuteSync(
this.PutBlobImpl(source, contentMD5, accessCondition, modifiedOptions),
modifiedOptions.RetryPolicy,
operationContext);
}
else
{
using (Stream blobStream = this.OpenWrite(accessCondition, modifiedOptions, operationContext))
{
source.WriteToSync(blobStream, null /* maxLength */, expiryTime, false, true, new OperationContext(), null /* streamCopyState */);
}
}
}
internal void UploadFromStreamHelper(Stream source, long? length, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext)
{
CommonUtility.AssertNotNull("source", source);
if (length.HasValue)
{
CommonUtility.AssertInBounds("length", length.Value, 1);
if (source.CanSeek && length > source.Length - source.Position)
{
throw new ArgumentOutOfRangeException("length", SR.StreamLengthShortError);
}
}
this.attributes.AssertNoSnapshot();
BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient);
operationContext = operationContext ?? new OperationContext();
bool lessThanSingleBlobThreshold = source.CanSeek
&& (length ?? source.Length - source.Position)
<= this.ServiceClient.SingleBlobUploadThresholdInBytes;
if (this.ServiceClient.ParallelOperationThreadCount == 1 && lessThanSingleBlobThreshold)
{
string contentMD5 = null;
if (modifiedOptions.StoreBlobContentMD5.Value)
{
using (ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions))
{
StreamDescriptor streamCopyState = new StreamDescriptor();
long startPosition = source.Position;
source.WriteToSync(Stream.Null, length, null /* maxLength */, true, true, tempExecutionState, streamCopyState);
source.Position = startPosition;
contentMD5 = streamCopyState.Md5;
}
}
else
{
// Throw exception if we need to use Transactional MD5 but cannot store it
if (modifiedOptions.UseTransactionalMD5.Value)
{
throw new ArgumentException(SR.PutBlobNeedsStoreBlobContentMD5, "options");
}
}
Executor.ExecuteSync(
this.PutBlobImpl(source, length, contentMD5, accessCondition, modifiedOptions),
modifiedOptions.RetryPolicy,
operationContext);
}
else
{
using (CloudBlobStream blobStream = this.OpenWrite(accessCondition, modifiedOptions, operationContext))
{
using (ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions))
{
source.WriteToSync(blobStream, length, null /* maxLength */, false, true, tempExecutionState, null /* streamCopyState */);
blobStream.Commit();
}
}
}
}
public ICancellableAsyncResult BeginUploadFromStream(Stream source, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext, AsyncCallback callback, object state)
{
CommonUtils.AssertNotNull("source", source);
this.attributes.AssertNoSnapshot();
BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient);
DateTime? expiryTime = modifiedOptions.MaximumExecutionTime.HasValue
? DateTime.Now + modifiedOptions.MaximumExecutionTime.Value
: (DateTime?)null;
OperationContext tempOperationContext = new OperationContext();
ExecutionState<NullType> executionState = new ExecutionState<NullType>(null /* cmd */, modifiedOptions.RetryPolicy, tempOperationContext);
ChainedAsyncResult<NullType> chainedResult = new ChainedAsyncResult<NullType>(callback, state)
{
CancelDelegate = executionState.Cancel,
};
if ((this.ServiceClient.ParallelOperationThreadCount == 1) &&
source.CanSeek &&
((source.Length - source.Position) <= this.ServiceClient.SingleBlobUploadThresholdInBytes))
{
if (modifiedOptions.StoreBlobContentMD5.Value)
{
long startPosition = source.Position;
StreamDescriptor streamCopyState = new StreamDescriptor();
source.WriteToAsync(
Stream.Null,
null /* maxLength */,
expiryTime,
true,
executionState,
tempOperationContext,
streamCopyState,
completedState =>
{
chainedResult.UpdateCompletedSynchronously(executionState.CompletedSynchronously);
try
{
lock (chainedResult.CancellationLockerObject)
{
chainedResult.CancelDelegate = null;
if (completedState.ExceptionRef != null)
{
chainedResult.OnComplete(completedState.ExceptionRef);
}
else
{
source.Position = startPosition;
UploadFromStreamHandler(source, streamCopyState.Md5, accessCondition, operationContext, modifiedOptions, chainedResult);
}
}
}
catch (Exception e)
{
chainedResult.OnComplete(e);
}
});
}
else
{
this.UploadFromStreamHandler(source, null /* contentMD5 */, accessCondition, operationContext, modifiedOptions, chainedResult);
}
}
else
{
Stream blobStream = this.OpenWrite(accessCondition, modifiedOptions, operationContext);
source.WriteToAsync(
blobStream,
null /* maxLength */,
expiryTime,
false,
executionState,
tempOperationContext,
null /* streamCopyState */,
completedState =>
{
chainedResult.UpdateCompletedSynchronously(executionState.CompletedSynchronously);
try
{
blobStream.Close();
chainedResult.OnComplete(executionState.ExceptionRef);
}
catch (Exception e)
{
chainedResult.OnComplete(e);
}
});
}
return chainedResult;
}
public void WritePages(Stream pageData, long startOffset, string contentMD5 = null, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null)
{
BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.PageBlob, this.ServiceClient);
bool requiresContentMD5 = (contentMD5 == null) && modifiedOptions.UseTransactionalMD5.Value;
operationContext = operationContext ?? new OperationContext();
Stream seekableStream = pageData;
if (!pageData.CanSeek || requiresContentMD5)
{
OperationContext tempOperationContext = new OperationContext();
DateTime? expiryTime = modifiedOptions.MaximumExecutionTime.HasValue
? DateTime.Now + modifiedOptions.MaximumExecutionTime.Value
: (DateTime?)null;
Stream writeToStream;
if (pageData.CanSeek)
{
writeToStream = Stream.Null;
}
else
{
seekableStream = new MemoryStream();
writeToStream = seekableStream;
}
long startPosition = seekableStream.Position;
StreamDescriptor streamCopyState = new StreamDescriptor();
pageData.WriteToSync(writeToStream, Constants.MaxBlockSize, expiryTime, requiresContentMD5, true, tempOperationContext, streamCopyState);
seekableStream.Position = startPosition;
if (requiresContentMD5)
{
contentMD5 = streamCopyState.Md5;
}
}
Executor.ExecuteSync(
this.PutPageImpl(seekableStream, startOffset, contentMD5, accessCondition, modifiedOptions),
modifiedOptions.RetryPolicy,
operationContext);
}
public virtual void WriteRange(Stream rangeData, long startOffset, string contentMD5 = null, AccessCondition accessCondition = null, FileRequestOptions options = null, OperationContext operationContext = null)
{
CommonUtility.AssertNotNull("rangeData", rangeData);
FileRequestOptions modifiedOptions = FileRequestOptions.ApplyDefaults(options, this.ServiceClient);
bool requiresContentMD5 = (contentMD5 == null) && modifiedOptions.UseTransactionalMD5.Value;
operationContext = operationContext ?? new OperationContext();
Stream seekableStream = rangeData;
bool seekableStreamCreated = false;
try
{
if (!rangeData.CanSeek || requiresContentMD5)
{
ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions);
Stream writeToStream;
if (rangeData.CanSeek)
{
writeToStream = Stream.Null;
}
else
{
seekableStream = new MultiBufferMemoryStream(this.ServiceClient.BufferManager);
seekableStreamCreated = true;
writeToStream = seekableStream;
}
long startPosition = seekableStream.Position;
StreamDescriptor streamCopyState = new StreamDescriptor();
rangeData.WriteToSync(writeToStream, null /* copyLength */, Constants.MaxBlockSize, requiresContentMD5, true, tempExecutionState, streamCopyState);
seekableStream.Position = startPosition;
if (requiresContentMD5)
{
contentMD5 = streamCopyState.Md5;
}
}
Executor.ExecuteSync(
this.PutRangeImpl(seekableStream, startOffset, contentMD5, accessCondition, modifiedOptions),
modifiedOptions.RetryPolicy,
operationContext);
}
finally
{
if (seekableStreamCreated)
{
seekableStream.Dispose();
}
}
}
internal static async Task WriteToAsync(this Stream stream, Stream toStream, long?maxLength, bool calculateMd5, OperationContext operationContext, StreamDescriptor streamCopyState, CancellationToken token)
{
byte[] buffer = new byte[GetBufferSize(stream)];
if (streamCopyState != null && calculateMd5 && streamCopyState.Md5HashRef == null)
{
streamCopyState.Md5HashRef = new MD5Wrapper();
}
int readCount;
do
{
readCount = await stream.ReadAsync(buffer, 0, buffer.Length, token);
await toStream.WriteAsync(buffer, 0, readCount, token);
// Update the StreamDescriptor after the bytes are successfully committed to the output stream
if (streamCopyState != null)
{
streamCopyState.Length += readCount;
if (maxLength.HasValue && streamCopyState.Length > maxLength.Value)
{
throw new ArgumentOutOfRangeException("stream");
}
if (streamCopyState.Md5HashRef != null)
{
streamCopyState.Md5HashRef.UpdateHash(buffer, 0, readCount);
}
}
}while (readCount != 0);
if (streamCopyState != null && streamCopyState.Md5HashRef != null)
{
streamCopyState.Md5 = streamCopyState.Md5HashRef.ComputeHash();
streamCopyState.Md5HashRef = null;
}
}
public IAsyncAction WriteRangeAsync(IInputStream rangeData, long startOffset, string contentMD5, AccessCondition accessCondition, FileRequestOptions options, OperationContext operationContext)
#endif
{
CommonUtility.AssertNotNull("rangeData", rangeData);
FileRequestOptions modifiedOptions = FileRequestOptions.ApplyDefaults(options, this.ServiceClient);
bool requiresContentMD5 = (contentMD5 == null) && modifiedOptions.UseTransactionalMD5.Value;
operationContext = operationContext ?? new OperationContext();
ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions);
#if ASPNET_K
return Task.Run(async () =>
#else
return AsyncInfo.Run(async (cancellationToken) =>
#endif
{
DateTime streamCopyStartTime = DateTime.Now;
Stream rangeDataAsStream = rangeData.AsStreamForRead();
Stream seekableStream = rangeDataAsStream;
if (!rangeDataAsStream.CanSeek || requiresContentMD5)
{
Stream writeToStream;
if (rangeDataAsStream.CanSeek)
{
writeToStream = Stream.Null;
}
else
{
seekableStream = new MultiBufferMemoryStream(this.ServiceClient.BufferManager);
writeToStream = seekableStream;
}
StreamDescriptor streamCopyState = new StreamDescriptor();
long startPosition = seekableStream.Position;
await rangeDataAsStream.WriteToAsync(writeToStream, null /* copyLength */, Constants.MaxBlockSize, requiresContentMD5, tempExecutionState, streamCopyState, cancellationToken);
seekableStream.Position = startPosition;
if (requiresContentMD5)
{
contentMD5 = streamCopyState.Md5;
}
if (modifiedOptions.MaximumExecutionTime.HasValue)
{
modifiedOptions.MaximumExecutionTime -= DateTime.Now.Subtract(streamCopyStartTime);
}
}
await Executor.ExecuteAsyncNullReturn(
this.PutRangeImpl(seekableStream, startOffset, contentMD5, accessCondition, modifiedOptions),
modifiedOptions.RetryPolicy,
operationContext,
cancellationToken);
#if ASPNET_K
}, cancellationToken);
#else
});
internal static void WriteToSync(this Stream stream, Stream toStream, long?maxLength, DateTime?expiryTime, bool calculateMd5, bool syncRead, OperationContext operationContext, StreamDescriptor streamCopyState)
{
byte[] buffer = new byte[GetBufferSize(stream)];
if (streamCopyState != null && calculateMd5 && streamCopyState.Md5HashRef == null)
{
streamCopyState.Md5HashRef = new MD5Wrapper();
}
int readCount;
do
{
if (expiryTime.HasValue && DateTime.Now.CompareTo(expiryTime.Value) > 0)
{
throw Exceptions.GenerateTimeoutException(operationContext.LastResult, null);
}
if (syncRead)
{
readCount = stream.Read(buffer, 0, buffer.Length);
}
else
{
// Use an async read since Sync read on ConnectStream will not throw for prematurely closed connections
readCount = stream.EndRead(stream.BeginRead(buffer, 0, buffer.Length, null /* Callback */, null /* State */));
}
toStream.Write(buffer, 0, readCount);
// Update the StreamDescriptor after the bytes are successfully committed to the output stream
if (streamCopyState != null)
{
streamCopyState.Length += readCount;
if (maxLength.HasValue && streamCopyState.Length > maxLength.Value)
{
throw new ArgumentOutOfRangeException("stream");
}
if (streamCopyState.Md5HashRef != null)
{
streamCopyState.Md5HashRef.UpdateHash(buffer, 0, readCount);
}
}
}while (readCount != 0);
if (streamCopyState != null && streamCopyState.Md5HashRef != null)
{
streamCopyState.Md5 = streamCopyState.Md5HashRef.ComputeHash();
streamCopyState.Md5HashRef = null;
}
}
public IAsyncAction UploadFromStreamAsync(IInputStream source, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext)
{
CommonUtils.AssertNotNull("source", source);
this.attributes.AssertNoSnapshot();
BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient);
operationContext = operationContext ?? new OperationContext();
Stream sourceAsStream = source.AsStreamForRead();
return AsyncInfo.Run(async (token) =>
{
DateTime streamCopyStartTime = DateTime.Now;
CancellationToken streamCopyToken = token;
if (modifiedOptions.MaximumExecutionTime.HasValue)
{
// Setup token
CancellationTokenSource cts = new CancellationTokenSource(modifiedOptions.MaximumExecutionTime.Value);
CancellationToken tokenWithTimeout = cts.Token;
// Hookup users token
token.Register(() => cts.Cancel());
streamCopyToken = tokenWithTimeout;
}
if ((this.ServiceClient.ParallelOperationThreadCount == 1) &&
sourceAsStream.CanSeek &&
((sourceAsStream.Length - sourceAsStream.Position) <= this.ServiceClient.SingleBlobUploadThresholdInBytes))
{
string contentMD5 = null;
if (modifiedOptions.StoreBlobContentMD5.Value)
{
OperationContext tempOperationContext = new OperationContext();
StreamDescriptor streamCopyState = new StreamDescriptor();
long startPosition = sourceAsStream.Position;
await sourceAsStream.WriteToAsync(Stream.Null, null /* maxLength */, true, tempOperationContext, streamCopyState, streamCopyToken);
sourceAsStream.Position = startPosition;
contentMD5 = streamCopyState.Md5;
}
if (modifiedOptions.MaximumExecutionTime.HasValue)
{
modifiedOptions.MaximumExecutionTime -= DateTime.Now.Subtract(streamCopyStartTime);
}
await Executor.ExecuteAsyncNullReturn(
this.PutBlobImpl(sourceAsStream, contentMD5, accessCondition, modifiedOptions),
modifiedOptions.RetryPolicy,
operationContext,
token);
}
else
{
await Task.Run(async () =>
{
IOutputStream writeStream = await this.OpenWriteAsync(accessCondition, options, operationContext);
// We should always call AsStreamForWrite with bufferSize=0 to prevent buffering. Our
// stream copier only writes 64K buffers at a time anyway, so no buffering is needed.
using (Stream blobStream = writeStream.AsStreamForWrite(0))
{
await sourceAsStream.WriteToAsync(blobStream, null /* maxLength */, false, new OperationContext(), null /* streamCopyState */, streamCopyToken);
}
});
}
});
}
internal static void WriteToAsync <T>(this Stream stream, Stream toStream, long?copyLength, long?maxLength, bool calculateMd5, ExecutionState <T> executionState, StreamDescriptor streamCopyState, Action <ExecutionState <T> > completed)
{
AsyncStreamCopier <T> copier = new AsyncStreamCopier <T>(stream, toStream, executionState, GetBufferSize(stream), calculateMd5, streamCopyState);
copier.StartCopyStream(completed, copyLength, maxLength);
}
internal void UploadFromStreamHelper(Stream source, long? length, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext)
{
CommonUtility.AssertNotNull("source", source);
if (length.HasValue)
{
CommonUtility.AssertInBounds("length", length.Value, 1);
if (source.CanSeek && length > source.Length - source.Position)
{
throw new ArgumentOutOfRangeException("length", SR.StreamLengthShortError);
}
}
this.attributes.AssertNoSnapshot();
BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient);
operationContext = operationContext ?? new OperationContext();
bool lessThanSingleBlobThreshold = CloudBlockBlob.IsLessThanSingleBlobThreshold(source, length, modifiedOptions, false);
modifiedOptions.AssertPolicyIfRequired();
if (modifiedOptions.ParallelOperationThreadCount.Value == 1 && lessThanSingleBlobThreshold)
{
bool usingEncryption = modifiedOptions.EncryptionPolicy != null;
Stream sourceStream = source;
using (MemoryStream tempStream = !usingEncryption ? null : new MemoryStream())
{
// Encrypt if necessary
if (usingEncryption)
{
modifiedOptions.AssertPolicyIfRequired();
if (modifiedOptions.EncryptionPolicy.EncryptionMode != BlobEncryptionMode.FullBlob)
{
throw new InvalidOperationException(SR.InvalidEncryptionMode, null);
}
ICryptoTransform transform = modifiedOptions.EncryptionPolicy.CreateAndSetEncryptionContext(this.Metadata, false /* noPadding */);
CryptoStream cryptoStream = new CryptoStream(tempStream, transform, CryptoStreamMode.Write);
using (ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(options))
{
source.WriteToSync(cryptoStream, length, null, false, true, tempExecutionState, null);
cryptoStream.FlushFinalBlock();
}
// After the tempStream has been written to, we need to seek back to the beginning, so that it can be read from.
tempStream.Seek(0, SeekOrigin.Begin);
length = tempStream.Length;
sourceStream = tempStream;
}
// Calculate MD5 if necessary
// Note that we cannot do this while we encrypt, it must be a separate step, because we want the MD5 of the encrypted data,
// not the unencrypted data.
string contentMD5 = null;
if (modifiedOptions.StoreBlobContentMD5.Value)
{
using (ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions))
{
StreamDescriptor streamCopyState = new StreamDescriptor();
long startPosition = sourceStream.Position;
sourceStream.WriteToSync(Stream.Null, length, null /* maxLength */, true, true, tempExecutionState, streamCopyState);
sourceStream.Position = startPosition;
contentMD5 = streamCopyState.Md5;
}
}
else
{
// Throw exception if we need to use Transactional MD5 but cannot store it
if (modifiedOptions.UseTransactionalMD5.Value)
{
throw new ArgumentException(SR.PutBlobNeedsStoreBlobContentMD5, "options");
}
}
// Execute the put blob.
Executor.ExecuteSync(
this.PutBlobImpl(sourceStream, length, contentMD5, accessCondition, modifiedOptions),
modifiedOptions.RetryPolicy,
operationContext);
}
}
else
{
using (CloudBlobStream blobStream = this.OpenWrite(accessCondition, modifiedOptions, operationContext))
{
using (ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions))
{
source.WriteToSync(blobStream, length, null /* maxLength */, false, true, tempExecutionState, null /* streamCopyState */);
blobStream.Commit();
}
}
}
}
internal ICancellableAsyncResult BeginUploadFromStreamHelper(Stream source, long? length, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext, AsyncCallback callback, object state)
{
CommonUtility.AssertNotNull("source", source);
if (length.HasValue)
{
CommonUtility.AssertInBounds("length", length.Value, 1);
if (source.CanSeek && length > source.Length - source.Position)
{
throw new ArgumentOutOfRangeException("length", SR.StreamLengthShortError);
}
}
this.attributes.AssertNoSnapshot();
BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient);
ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions);
StorageAsyncResult<NullType> storageAsyncResult = new StorageAsyncResult<NullType>(callback, state);
bool lessThanSingleBlobThreshold = CloudBlockBlob.IsLessThanSingleBlobThreshold(source, length, modifiedOptions, false);
modifiedOptions.AssertPolicyIfRequired();
if (modifiedOptions.ParallelOperationThreadCount.Value == 1 && lessThanSingleBlobThreshold)
{
// Because we may or may not want to calculate the MD5, and we may or may not want to encrypt, rather than have four branching code
// paths, here we have an action that we will run, which continually gets added to, depending on which operations we need to do.
// The confusing part is that we have to build it from the bottom up.
string md5 = null;
Stream sourceStream = source;
Action actionToRun = null;
Action uploadAction = () =>
{
if (md5 == null && modifiedOptions.UseTransactionalMD5.Value)
{
throw new ArgumentException(SR.PutBlobNeedsStoreBlobContentMD5, "options");
}
this.UploadFromStreamHandler(
sourceStream,
length,
md5,
accessCondition,
operationContext,
modifiedOptions,
storageAsyncResult);
};
actionToRun = uploadAction;
if (modifiedOptions.StoreBlobContentMD5.Value)
{
Action<Action> calculateMD5 = (continuation) =>
{
long startPosition = sourceStream.Position;
StreamDescriptor streamCopyState = new StreamDescriptor();
sourceStream.WriteToAsync(
Stream.Null,
length,
null /* maxLength */,
true,
tempExecutionState,
streamCopyState,
completedState =>
{
ContinueAsyncOperation(storageAsyncResult, completedState, () =>
{
if (completedState.ExceptionRef != null)
{
storageAsyncResult.OnComplete(completedState.ExceptionRef);
}
else
{
sourceStream.Position = startPosition;
md5 = streamCopyState.Md5;
continuation();
}
});
});
storageAsyncResult.CancelDelegate = tempExecutionState.Cancel;
if (storageAsyncResult.CancelRequested)
{
storageAsyncResult.Cancel();
}
};
Action oldActionToRun = actionToRun;
actionToRun = () => calculateMD5(oldActionToRun);
}
if (modifiedOptions.EncryptionPolicy != null)
{
Action<Action> encryptStream = continuation =>
{
SyncMemoryStream syncMemoryStream = new SyncMemoryStream();
options.AssertPolicyIfRequired();
sourceStream = syncMemoryStream;
if (modifiedOptions.EncryptionPolicy.EncryptionMode != BlobEncryptionMode.FullBlob)
{
throw new InvalidOperationException(SR.InvalidEncryptionMode, null);
}
ICryptoTransform transform = options.EncryptionPolicy.CreateAndSetEncryptionContext(this.Metadata, false /* noPadding */);
CryptoStream cryptoStream = new CryptoStream(syncMemoryStream, transform, CryptoStreamMode.Write);
StreamDescriptor streamCopyState = new StreamDescriptor();
source.WriteToAsync(cryptoStream, length, null, false, tempExecutionState, streamCopyState, completedState =>
{
ContinueAsyncOperation(storageAsyncResult, completedState, () =>
{
if (completedState.ExceptionRef != null)
{
storageAsyncResult.OnComplete(completedState.ExceptionRef);
}
else
{
// Flush the CryptoStream in order to make sure that the last block of data is flushed. This call is a sync call
// but it is ok to have it because we're just writing to a memory stream.
cryptoStream.FlushFinalBlock();
// After the tempStream has been written to, we need to seek back to the beginning, so that it can be read from.
sourceStream.Seek(0, SeekOrigin.Begin);
length = syncMemoryStream.Length;
continuation();
}
});
});
storageAsyncResult.CancelDelegate = tempExecutionState.Cancel;
if (storageAsyncResult.CancelRequested)
{
storageAsyncResult.Cancel();
}
};
Action oldActionToRun = actionToRun;
actionToRun = () => encryptStream(oldActionToRun);
}
actionToRun();
}
else
{
ICancellableAsyncResult result = this.BeginOpenWrite(
accessCondition,
modifiedOptions,
operationContext,
ar =>
{
ContinueAsyncOperation(storageAsyncResult, ar, () =>
{
CloudBlobStream blobStream = this.EndOpenWrite(ar);
storageAsyncResult.OperationState = blobStream;
source.WriteToAsync(
blobStream,
length,
null /* maxLength */,
false,
tempExecutionState,
null /* streamCopyState */,
completedState =>
{
ContinueAsyncOperation(storageAsyncResult, completedState, () =>
{
if (completedState.ExceptionRef != null)
{
storageAsyncResult.OnComplete(completedState.ExceptionRef);
}
else
{
ICancellableAsyncResult commitResult = blobStream.BeginCommit(
CloudBlob.BlobOutputStreamCommitCallback,
storageAsyncResult);
storageAsyncResult.CancelDelegate = commitResult.Cancel;
if (storageAsyncResult.CancelRequested)
{
storageAsyncResult.Cancel();
}
}
});
});
storageAsyncResult.CancelDelegate = tempExecutionState.Cancel;
if (storageAsyncResult.CancelRequested)
{
storageAsyncResult.Cancel();
}
});
},
null /* state */);
// We do not need to do this inside a lock, as storageAsyncResult is
// not returned to the user yet.
storageAsyncResult.CancelDelegate = result.Cancel;
}
return storageAsyncResult;
}
internal static Task WriteToAsync <T>(this Stream stream, Stream toStream, IBufferManager bufferManager, long?copyLength, long?maxLength, bool calculateMd5, ExecutionState <T> executionState, StreamDescriptor streamCopyState, CancellationToken cancellationToken, Action <ExecutionState <T> > completed = null)
{
AsyncStreamCopier <T> copier = new AsyncStreamCopier <T>(stream, toStream, executionState, bufferManager, GetBufferSize(stream), calculateMd5, streamCopyState);
return(copier.StartCopyStream(completed, copyLength, maxLength, cancellationToken));
}
