/// <summary> /// Creates and initializes a new asynchronous copy operation. /// </summary> /// <param name="src">The source stream.</param> /// <param name="dest">The destination stream.</param> /// <param name="state">An ExecutionState used to coordinate copy operation.</param> /// <param name="buffSize">Size of read and write buffers used to move data.</param> /// <param name="calculateMd5">Boolean value indicating whether the MD-5 should be calculated.</param> /// <param name="streamCopyState">An object that represents the state for the current operation.</param> public AsyncStreamCopier(Stream src, Stream dest, ExecutionState <T> state, int buffSize, bool calculateMd5, StreamDescriptor streamCopyState) { this.src = src; this.dest = dest; this.state = state; this.buffSize = buffSize; this.streamCopyState = streamCopyState; if (streamCopyState != null && calculateMd5 && streamCopyState.Md5HashRef == null) { streamCopyState.Md5HashRef = new MD5Wrapper(); } }
/// <summary> /// Creates and initializes a new asynchronous copy operation. /// </summary> /// <param name="src">The source stream.</param> /// <param name="dest">The destination stream.</param> /// <param name="state">An ExecutionState used to coordinate copy operation.</param> /// <param name="bufferManager">IBufferManager instance to use. May be null.</param> /// <param name="buffSize">Size of read and write buffers used to move data. Overrides the default buffer size of bufferManager.</param> /// <param name="calculateMd5">Boolean value indicating whether the MD-5 should be calculated.</param> /// <param name="streamCopyState">An object that represents the state for the current operation.</param> public AsyncStreamCopier(Stream src, Stream dest, ExecutionState <T> state, IBufferManager bufferManager, int?buffSize, bool calculateMd5, StreamDescriptor streamCopyState) { this.src = src; this.dest = dest; this.state = state; this.bufferManager = bufferManager; this.buffSize = buffSize ?? (bufferManager != null ? bufferManager.GetDefaultBufferSize() : Constants.DefaultBufferSize); this.streamCopyState = streamCopyState; if (streamCopyState != null && calculateMd5 && streamCopyState.Md5HashRef == null) { streamCopyState.Md5HashRef = new MD5Wrapper(); } }
public AsyncStreamCopier(Stream src, Stream dest, ExecutionState <T> state, int buffSize, bool calculateMd5, OperationContext operationContext, StreamDescriptor streamCopyState) { this.src = src; this.dest = dest; this.state = state; this.currentReadBuff = new byte[buffSize]; this.currentWriteBuff = new byte[buffSize]; this.operationContext = operationContext; this.streamCopyState = streamCopyState; if (streamCopyState != null && calculateMd5 && streamCopyState.Md5HashRef == null) { streamCopyState.Md5HashRef = new MD5Wrapper(); } }
private static void ValidateCopier(int bufferLength, bool inputSync, int inputDelayInMs, bool outputSync, int outputDelayInMs) { byte[] buffer = new byte[bufferLength]; new Random().NextBytes(buffer); int expectedCallCount = buffer.Length / Constants.DefaultBufferSize; int totalDelayInMs = (expectedCallCount + 1) * inputDelayInMs + expectedCallCount * outputDelayInMs; DataValidationStream input = new DataValidationStream(buffer, inputSync, inputDelayInMs); DataValidationStream output = new DataValidationStream(buffer, outputSync, outputDelayInMs); OperationContext opContext = new OperationContext(); ExecutionState <NullType> state = new ExecutionState <NullType>(null, new NoRetry(), opContext); StreamDescriptor copyState = new StreamDescriptor(); using (ManualResetEvent waitHandle = new ManualResetEvent(false)) { input.WriteToAsync(output, null, null, false, state, opContext, copyState, _ => { waitHandle.Set(); }); Assert.IsTrue(waitHandle.WaitOne(totalDelayInMs + 10 * 1000)); } Assert.AreEqual(output.LastException, state.ExceptionRef); if (output.LastException != null) { throw output.LastException; } Assert.AreEqual(buffer.Length, copyState.Length); Assert.AreEqual(expectedCallCount + 1, input.ReadCallCount); Assert.AreEqual(expectedCallCount, output.WriteCallCount); }
private static void ValidateCopier(int bufferLength, long? copyLength, long? maxLength, bool inputSync, int inputDelayInMs, int inputFailRequest, bool outputSync, int outputDelayInMs, int outputFailRequest, DateTime? copyTimeout, bool seekable) { byte[] buffer = GetRandomBuffer(bufferLength); // Finds ceiling of division operation int expectedCallCount = (int) (copyLength.HasValue && buffer.Length > copyLength ? (-1L + copyLength + Constants.DefaultBufferSize) / Constants.DefaultBufferSize : (-1L + buffer.Length + Constants.DefaultBufferSize) / Constants.DefaultBufferSize); int totalDelayInMs = (expectedCallCount + 1) * inputDelayInMs + expectedCallCount * outputDelayInMs; DataValidationStream input = new DataValidationStream(buffer, inputSync, inputDelayInMs, inputFailRequest, seekable); DataValidationStream output = new DataValidationStream(buffer, outputSync, outputDelayInMs, outputFailRequest, seekable); RESTCommand<NullType> cmdWithTimeout = new RESTCommand<NullType>(new StorageCredentials(), null) { OperationExpiryTime = copyTimeout }; ExecutionState<NullType> state = new ExecutionState<NullType>(cmdWithTimeout, null, null); StreamDescriptor copyState = new StreamDescriptor(); using (ManualResetEvent waitHandle = new ManualResetEvent(false)) { input.WriteToAsync(output, copyLength, maxLength, false, state, copyState, _ => waitHandle.Set()); Assert.IsTrue(waitHandle.WaitOne(totalDelayInMs + 10 * 1000)); } if (inputFailRequest >= 0) { Assert.AreEqual(input.LastException, state.ExceptionRef); Assert.AreEqual(inputFailRequest, input.ReadCallCount); } if (outputFailRequest >= 0) { Assert.AreEqual(output.LastException, state.ExceptionRef); Assert.AreEqual(outputFailRequest, output.WriteCallCount); } if (state.ExceptionRef != null) { throw state.ExceptionRef; } Assert.AreEqual(copyLength.HasValue ? copyLength : buffer.Length, copyState.Length); Assert.AreEqual(copyLength.HasValue ? expectedCallCount : expectedCallCount + 1, input.ReadCallCount); Assert.AreEqual(expectedCallCount, output.WriteCallCount); }
public IAsyncAction WritePagesAsync(IInputStream pageData, long startOffset, string contentMD5, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext) { BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.PageBlob, this.ServiceClient); bool requiresContentMD5 = (contentMD5 == null) && modifiedOptions.UseTransactionalMD5.Value; operationContext = operationContext ?? new OperationContext(); return AsyncInfo.Run(async (token) => { Stream pageDataAsStream = pageData.AsStreamForRead(); Stream seekableStream = pageDataAsStream; if (!pageDataAsStream.CanSeek || requiresContentMD5) { DateTime streamCopyStartTime = DateTime.Now; CancellationToken streamCopyToken = token; if (modifiedOptions.MaximumExecutionTime.HasValue) { // Setup token CancellationTokenSource cts = new CancellationTokenSource(modifiedOptions.MaximumExecutionTime.Value); CancellationToken tokenWithTimeout = cts.Token; // Hookup users token token.Register(() => cts.Cancel()); streamCopyToken = tokenWithTimeout; } Stream writeToStream; if (pageDataAsStream.CanSeek) { writeToStream = Stream.Null; } else { seekableStream = new MemoryStream(); writeToStream = seekableStream; } OperationContext tempOperationContext = new OperationContext(); StreamDescriptor streamCopyState = new StreamDescriptor(); long startPosition = seekableStream.Position; await pageDataAsStream.WriteToAsync(writeToStream, Constants.MaxBlockSize, requiresContentMD5, tempOperationContext, streamCopyState, streamCopyToken); seekableStream.Position = startPosition; if (requiresContentMD5) { contentMD5 = streamCopyState.Md5; } if (modifiedOptions.MaximumExecutionTime.HasValue) { modifiedOptions.MaximumExecutionTime -= DateTime.Now.Subtract(streamCopyStartTime); } } await Executor.ExecuteAsyncNullReturn( this.PutPageImpl(seekableStream, startOffset, contentMD5, accessCondition, modifiedOptions), modifiedOptions.RetryPolicy, operationContext, token); }); }
/// <summary> /// Asynchronously reads the entire content of the stream and writes it to the given output stream. /// </summary> /// <param name="stream">The origin stream.</param> /// <param name="toStream">The destination stream.</param> /// <param name="copyLength">Number of bytes to copy from source stream to destination stream. Cannot be passed with a value for maxLength.</param> /// <param name="maxLength">Maximum length of the source stream. Cannot be passed with a value for copyLength.</param> /// <param name="calculateMd5">Bool value indicating whether the Md5 should be calculated.</param> /// <param name="executionState">An object that stores state of the operation.</param> /// <param name="streamCopyState">An object that represents the current state for the copy operation.</param> /// <param name="token">A CancellationToken to observe while waiting for the copy to complete.</param> /// <returns>The task object representing the asynchronous operation.</returns> internal static async Task WriteToAsync <T>(this Stream stream, Stream toStream, long?copyLength, long?maxLength, bool calculateMd5, ExecutionState <T> executionState, StreamDescriptor streamCopyState, CancellationToken token) { if (copyLength.HasValue && maxLength.HasValue) { throw new ArgumentException(SR.StreamLengthMismatch); } if (stream.CanSeek && maxLength.HasValue && stream.Length - stream.Position > maxLength) { throw new InvalidOperationException(SR.StreamLengthError); } if (stream.CanSeek && copyLength.HasValue && stream.Length - stream.Position < copyLength) { throw new ArgumentOutOfRangeException("copyLength", SR.StreamLengthShortError); } if (streamCopyState != null && calculateMd5 && streamCopyState.Md5HashRef == null) { streamCopyState.Md5HashRef = new MD5Wrapper(); } CancellationTokenSource cts = null; try { if (executionState.OperationExpiryTime.HasValue) { // Setup token for timeout cts = CancellationTokenSource.CreateLinkedTokenSource(token); cts.CancelAfter(executionState.RemainingTimeout); // Switch tokens token = cts.Token; } byte[] buffer = new byte[GetBufferSize(stream)]; long? bytesRemaining = copyLength; int readCount; do { // Determine how many bytes to read this time so that no more than count bytes are read int bytesToRead = bytesRemaining.HasValue && bytesRemaining < buffer.Length ? (int)bytesRemaining : buffer.Length; if (bytesToRead == 0) { break; } readCount = await stream.ReadAsync(buffer, 0, bytesToRead, token); if (bytesRemaining.HasValue) { bytesRemaining -= readCount; } if (readCount > 0) { await toStream.WriteAsync(buffer, 0, readCount, token); // Update the StreamDescriptor after the bytes are successfully committed to the output stream if (streamCopyState != null) { streamCopyState.Length += readCount; if (maxLength.HasValue && streamCopyState.Length > maxLength.Value) { throw new InvalidOperationException(SR.StreamLengthError); } if (streamCopyState.Md5HashRef != null) { streamCopyState.Md5HashRef.UpdateHash(buffer, 0, readCount); } } } }while (readCount > 0); if (bytesRemaining.HasValue && bytesRemaining != 0) { throw new ArgumentOutOfRangeException("copyLength", SR.StreamLengthShortError); } } finally { if (cts != null) { cts.Dispose(); cts = null; } } // Streams opened with AsStreamForWrite extension need to be flushed // to write all buffered data to the underlying Windows Runtime stream. await toStream.FlushAsync(); if (streamCopyState != null && streamCopyState.Md5HashRef != null) { streamCopyState.Md5 = streamCopyState.Md5HashRef.ComputeHash(); streamCopyState.Md5HashRef = null; } }
internal static void WriteToSync <T>(this Stream stream, Stream toStream, long?copyLength, long?maxLength, bool calculateMd5, bool syncRead, ExecutionState <T> executionState, StreamDescriptor streamCopyState) { if (copyLength.HasValue && maxLength.HasValue) { throw new ArgumentException(SR.StreamLengthMismatch); } if (stream.CanSeek && maxLength.HasValue && stream.Length - stream.Position > maxLength) { throw new InvalidOperationException(SR.StreamLengthError); } if (stream.CanSeek && copyLength.HasValue && stream.Length - stream.Position < copyLength) { throw new ArgumentOutOfRangeException("copyLength", SR.StreamLengthShortError); } byte[] buffer = new byte[GetBufferSize(stream)]; if (streamCopyState != null && calculateMd5 && streamCopyState.Md5HashRef == null) { streamCopyState.Md5HashRef = new MD5Wrapper(); } RegisteredWaitHandle waitHandle = null; ManualResetEvent completedEvent = null; if (!syncRead && executionState.OperationExpiryTime.HasValue) { completedEvent = new ManualResetEvent(false); waitHandle = ThreadPool.RegisterWaitForSingleObject( completedEvent, StreamExtensions.MaximumCopyTimeCallback <T>, executionState, executionState.RemainingTimeout, true); } try { long?bytesRemaining = copyLength; int readCount; do { if (executionState.OperationExpiryTime.HasValue && DateTime.Now.CompareTo(executionState.OperationExpiryTime.Value) > 0) { throw Exceptions.GenerateTimeoutException(executionState.Cmd != null ? executionState.Cmd.CurrentResult : null, null); } // Determine how many bytes to read this time so that no more than copyLength bytes are read int bytesToRead = MinBytesToRead(bytesRemaining, buffer.Length); if (bytesToRead == 0) { break; } // Read synchronously or asynchronously readCount = syncRead ? stream.Read(buffer, 0, bytesToRead) : stream.EndRead(stream.BeginRead(buffer, 0, bytesToRead, null /* Callback */, null /* State */)); // Decrement bytes to write from bytes read if (bytesRemaining.HasValue) { bytesRemaining -= readCount; } // Write if (readCount > 0) { toStream.Write(buffer, 0, readCount); // Update the StreamDescriptor after the bytes are successfully committed to the output stream if (streamCopyState != null) { streamCopyState.Length += readCount; if (maxLength.HasValue && streamCopyState.Length > maxLength.Value) { throw new InvalidOperationException(SR.StreamLengthError); } if (streamCopyState.Md5HashRef != null) { streamCopyState.Md5HashRef.UpdateHash(buffer, 0, readCount); } } } }while (readCount != 0); if (bytesRemaining.HasValue && bytesRemaining != 0) { throw new ArgumentOutOfRangeException("copyLength", SR.StreamLengthShortError); } } catch (Exception) { if (executionState.OperationExpiryTime.HasValue && DateTime.Now.CompareTo(executionState.OperationExpiryTime.Value) > 0) { throw Exceptions.GenerateTimeoutException(executionState.Cmd != null ? executionState.Cmd.CurrentResult : null, null); } else { throw; } } finally { if (waitHandle != null) { waitHandle.Unregister(null); } if (completedEvent != null) { completedEvent.Close(); } } if (streamCopyState != null && streamCopyState.Md5HashRef != null) { streamCopyState.Md5 = streamCopyState.Md5HashRef.ComputeHash(); streamCopyState.Md5HashRef = null; } }
public ICancellableAsyncResult BeginWritePages(Stream pageData, long startOffset, string contentMD5, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext, AsyncCallback callback, object state) { BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.PageBlob, this.ServiceClient); bool requiresContentMD5 = (contentMD5 == null) && modifiedOptions.UseTransactionalMD5.Value; operationContext = operationContext ?? new OperationContext(); ChainedAsyncResult<NullType> chainedResult = new ChainedAsyncResult<NullType>(callback, state); if (pageData.CanSeek && !requiresContentMD5) { this.WritePagesHandler(pageData, startOffset, contentMD5, accessCondition, modifiedOptions, operationContext, chainedResult); } else { DateTime? expiryTime = modifiedOptions.MaximumExecutionTime.HasValue ? DateTime.Now + modifiedOptions.MaximumExecutionTime.Value : (DateTime?)null; OperationContext tempOperationContext = new OperationContext(); ExecutionState<NullType> executionState = new ExecutionState<NullType>(null /* cmd */, modifiedOptions.RetryPolicy, tempOperationContext); chainedResult.CancelDelegate = executionState.Cancel; Stream seekableStream; Stream writeToStream; if (pageData.CanSeek) { seekableStream = pageData; writeToStream = Stream.Null; } else { seekableStream = new MemoryStream(); writeToStream = seekableStream; } long startPosition = seekableStream.Position; StreamDescriptor streamCopyState = new StreamDescriptor(); pageData.WriteToAsync( writeToStream, Constants.MaxBlockSize, expiryTime, requiresContentMD5, executionState, tempOperationContext, streamCopyState, _ => { chainedResult.UpdateCompletedSynchronously(executionState.CompletedSynchronously); if (executionState.ExceptionRef != null) { chainedResult.OnComplete(executionState.ExceptionRef); } else { try { if (requiresContentMD5) { contentMD5 = streamCopyState.Md5; } seekableStream.Position = startPosition; this.WritePagesHandler(seekableStream, startOffset, contentMD5, accessCondition, modifiedOptions, operationContext, chainedResult); } catch (Exception e) { chainedResult.OnComplete(e); } } }); } return chainedResult; }
internal static void WriteToAsync <T>(this Stream stream, Stream toStream, long?maxLength, DateTime?expiryTime, bool calculateMd5, ExecutionState <T> executionState, OperationContext operationContext, StreamDescriptor streamCopyState, Action <ExecutionState <T> > completed) { AsyncStreamCopier <T> copier = new AsyncStreamCopier <T>(stream, toStream, executionState, GetBufferSize(stream), calculateMd5, operationContext, streamCopyState); copier.StartCopyStream(completed, maxLength, expiryTime); }
public ICancellableAsyncResult BeginPutBlock(string blockId, Stream blockData, string contentMD5, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext, AsyncCallback callback, object state) { CommonUtility.AssertNotNull("blockData", blockData); BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient); bool requiresContentMD5 = (contentMD5 == null) && modifiedOptions.UseTransactionalMD5.Value; operationContext = operationContext ?? new OperationContext(); StorageAsyncResult<NullType> storageAsyncResult = new StorageAsyncResult<NullType>(callback, state); if (blockData.CanSeek && !requiresContentMD5) { this.PutBlockHandler(blockId, blockData, contentMD5, accessCondition, modifiedOptions, operationContext, storageAsyncResult); } else { ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions); storageAsyncResult.CancelDelegate = tempExecutionState.Cancel; Stream seekableStream; Stream writeToStream; if (blockData.CanSeek) { seekableStream = blockData; writeToStream = Stream.Null; } else { seekableStream = new MultiBufferMemoryStream(this.ServiceClient.BufferManager); writeToStream = seekableStream; } long startPosition = seekableStream.Position; StreamDescriptor streamCopyState = new StreamDescriptor(); blockData.WriteToAsync( writeToStream, null /* copyLength */, Constants.MaxBlockSize, requiresContentMD5, tempExecutionState, streamCopyState, completedState => { storageAsyncResult.UpdateCompletedSynchronously(completedState.CompletedSynchronously); if (completedState.ExceptionRef != null) { storageAsyncResult.OnComplete(completedState.ExceptionRef); } else { try { if (requiresContentMD5) { contentMD5 = streamCopyState.Md5; } seekableStream.Position = startPosition; this.PutBlockHandler(blockId, seekableStream, contentMD5, accessCondition, modifiedOptions, operationContext, storageAsyncResult); } catch (Exception e) { storageAsyncResult.OnComplete(e); } } }); } return storageAsyncResult; }
internal ICancellableAsyncResult BeginUploadFromStreamHelper(Stream source, long? length, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext, AsyncCallback callback, object state) { CommonUtility.AssertNotNull("source", source); if (length.HasValue) { CommonUtility.AssertInBounds("length", length.Value, 1); if (source.CanSeek && length > source.Length - source.Position) { throw new ArgumentOutOfRangeException("length", SR.StreamLengthShortError); } } this.attributes.AssertNoSnapshot(); BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient); ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions); StorageAsyncResult<NullType> storageAsyncResult = new StorageAsyncResult<NullType>(callback, state); bool lessThanSingleBlobThreshold = source.CanSeek && (length ?? source.Length - source.Position) <= this.ServiceClient.SingleBlobUploadThresholdInBytes; if (this.ServiceClient.ParallelOperationThreadCount == 1 && lessThanSingleBlobThreshold) { if (modifiedOptions.StoreBlobContentMD5.Value) { long startPosition = source.Position; StreamDescriptor streamCopyState = new StreamDescriptor(); source.WriteToAsync( Stream.Null, length, null /* maxLength */, true, tempExecutionState, streamCopyState, completedState => { storageAsyncResult.UpdateCompletedSynchronously(completedState.CompletedSynchronously); try { lock (storageAsyncResult.CancellationLockerObject) { storageAsyncResult.CancelDelegate = null; if (completedState.ExceptionRef != null) { storageAsyncResult.OnComplete(completedState.ExceptionRef); } else { source.Position = startPosition; this.UploadFromStreamHandler( source, length, streamCopyState.Md5, accessCondition, operationContext, modifiedOptions, storageAsyncResult); } } } catch (Exception e) { storageAsyncResult.OnComplete(e); } }); // We do not need to do this inside a lock, as storageAsyncResult is // not returned to the user yet. storageAsyncResult.CancelDelegate = tempExecutionState.Cancel; } else { if (modifiedOptions.UseTransactionalMD5.Value) { throw new ArgumentException(SR.PutBlobNeedsStoreBlobContentMD5, "options"); } this.UploadFromStreamHandler( source, length, null /* contentMD5 */, accessCondition, operationContext, modifiedOptions, storageAsyncResult); } } else { ICancellableAsyncResult result = this.BeginOpenWrite( accessCondition, modifiedOptions, operationContext, ar => { storageAsyncResult.UpdateCompletedSynchronously(ar.CompletedSynchronously); lock (storageAsyncResult.CancellationLockerObject) { storageAsyncResult.CancelDelegate = null; try { CloudBlobStream blobStream = this.EndOpenWrite(ar); storageAsyncResult.OperationState = blobStream; source.WriteToAsync( blobStream, length, null /* maxLength */, false, tempExecutionState, null /* streamCopyState */, completedState => { storageAsyncResult.UpdateCompletedSynchronously(completedState.CompletedSynchronously); if (completedState.ExceptionRef != null) { storageAsyncResult.OnComplete(completedState.ExceptionRef); } else { try { lock (storageAsyncResult.CancellationLockerObject) { storageAsyncResult.CancelDelegate = null; ICancellableAsyncResult commitResult = blobStream.BeginCommit( CloudBlobSharedImpl.BlobOutputStreamCommitCallback, storageAsyncResult); storageAsyncResult.CancelDelegate = commitResult.Cancel; if (storageAsyncResult.CancelRequested) { storageAsyncResult.Cancel(); } } } catch (Exception e) { storageAsyncResult.OnComplete(e); } } }); storageAsyncResult.CancelDelegate = tempExecutionState.Cancel; if (storageAsyncResult.CancelRequested) { storageAsyncResult.Cancel(); } } catch (Exception e) { storageAsyncResult.OnComplete(e); } } }, null /* state */); // We do not need to do this inside a lock, as storageAsyncResult is // not returned to the user yet. storageAsyncResult.CancelDelegate = result.Cancel; } return storageAsyncResult; }
public virtual Task WritePagesAsync(Stream pageData, long startOffset, string contentMD5, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken) { BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.PageBlob, this.ServiceClient); bool requiresContentMD5 = (contentMD5 == null) && modifiedOptions.UseTransactionalMD5.Value; operationContext = operationContext ?? new OperationContext(); ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions); return Task.Run(async () => { Stream pageDataAsStream = pageData; Stream seekableStream = pageDataAsStream; bool seekableStreamCreated = false; try { if (!pageDataAsStream.CanSeek || requiresContentMD5) { Stream writeToStream; if (pageDataAsStream.CanSeek) { writeToStream = Stream.Null; } else { seekableStream = new MultiBufferMemoryStream(this.ServiceClient.BufferManager); seekableStreamCreated = true; writeToStream = seekableStream; } StreamDescriptor streamCopyState = new StreamDescriptor(); long startPosition = seekableStream.Position; await pageDataAsStream.WriteToAsync(writeToStream, null /* copyLength */, Constants.MaxBlockSize, requiresContentMD5, tempExecutionState, streamCopyState, cancellationToken); seekableStream.Position = startPosition; if (requiresContentMD5) { contentMD5 = streamCopyState.Md5; } } await Executor.ExecuteAsyncNullReturn( this.PutPageImpl(seekableStream, startOffset, contentMD5, accessCondition, modifiedOptions), modifiedOptions.RetryPolicy, operationContext, cancellationToken); } finally { if (seekableStreamCreated) { seekableStream.Dispose(); } } }, cancellationToken); }
private static void ValidateCopier(int bufferLength, bool inputSync, int inputDelayInMs, bool outputSync, int outputDelayInMs) { byte[] buffer = new byte[bufferLength]; new Random().NextBytes(buffer); int expectedCallCount = buffer.Length / Constants.DefaultBufferSize; int totalDelayInMs = (expectedCallCount + 1) * inputDelayInMs + expectedCallCount * outputDelayInMs; DataValidationStream input = new DataValidationStream(buffer, inputSync, inputDelayInMs); DataValidationStream output = new DataValidationStream(buffer, outputSync, outputDelayInMs); OperationContext opContext = new OperationContext(); ExecutionState<NullType> state = new ExecutionState<NullType>(null, new NoRetry(), opContext); StreamDescriptor copyState = new StreamDescriptor(); using (ManualResetEvent waitHandle = new ManualResetEvent(false)) { input.WriteToAsync(output, null, null, false, state, opContext, copyState, _ => { waitHandle.Set(); }); Assert.IsTrue(waitHandle.WaitOne(totalDelayInMs + 10 * 1000)); } Assert.AreEqual(output.LastException, state.ExceptionRef); if (output.LastException != null) { throw output.LastException; } Assert.AreEqual(buffer.Length, copyState.Length); Assert.AreEqual(expectedCallCount + 1, input.ReadCallCount); Assert.AreEqual(expectedCallCount, output.WriteCallCount); }
public IAsyncAction PutBlockAsync(string blockId, IInputStream blockData, string contentMD5, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext) { BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient); bool requiresContentMD5 = (contentMD5 == null) && modifiedOptions.UseTransactionalMD5.Value; operationContext = operationContext ?? new OperationContext(); ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions); return AsyncInfo.Run(async (token) => { Stream blockDataAsStream = blockData.AsStreamForRead(); Stream seekableStream = blockDataAsStream; if (!blockDataAsStream.CanSeek || requiresContentMD5) { Stream writeToStream; if (blockDataAsStream.CanSeek) { writeToStream = Stream.Null; } else { seekableStream = new MultiBufferMemoryStream(this.ServiceClient.BufferManager); writeToStream = seekableStream; } StreamDescriptor streamCopyState = new StreamDescriptor(); long startPosition = seekableStream.Position; await blockDataAsStream.WriteToAsync(writeToStream, null /* copyLength */, Constants.MaxBlockSize, requiresContentMD5, tempExecutionState, streamCopyState, token); seekableStream.Position = startPosition; if (requiresContentMD5) { contentMD5 = streamCopyState.Md5; } } await Executor.ExecuteAsyncNullReturn( this.PutBlockImpl(seekableStream, blockId, contentMD5, accessCondition, modifiedOptions), modifiedOptions.RetryPolicy, operationContext, token); }); }
internal IAsyncAction UploadFromStreamAsyncHelper(IInputStream source, long? length, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext) { CommonUtility.AssertNotNull("source", source); Stream sourceAsStream = source.AsStreamForRead(); if (length.HasValue) { CommonUtility.AssertInBounds("length", length.Value, 1); if (sourceAsStream.CanSeek && length > sourceAsStream.Length - sourceAsStream.Position) { throw new ArgumentOutOfRangeException("length", SR.StreamLengthShortError); } } this.attributes.AssertNoSnapshot(); BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient); operationContext = operationContext ?? new OperationContext(); ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions); return AsyncInfo.Run(async (token) => { bool lessThanSingleBlobThreshold = sourceAsStream.CanSeek && (length ?? sourceAsStream.Length - sourceAsStream.Position) <= this.ServiceClient.SingleBlobUploadThresholdInBytes; if (this.ServiceClient.ParallelOperationThreadCount == 1 && lessThanSingleBlobThreshold) { string contentMD5 = null; if (modifiedOptions.StoreBlobContentMD5.Value) { StreamDescriptor streamCopyState = new StreamDescriptor(); long startPosition = sourceAsStream.Position; await sourceAsStream.WriteToAsync(Stream.Null, length, null /* maxLength */, true, tempExecutionState, streamCopyState, token); sourceAsStream.Position = startPosition; contentMD5 = streamCopyState.Md5; } else { if (modifiedOptions.UseTransactionalMD5.Value) { throw new ArgumentException(SR.PutBlobNeedsStoreBlobContentMD5, "options"); } } await Executor.ExecuteAsyncNullReturn( this.PutBlobImpl(sourceAsStream, length, contentMD5, accessCondition, modifiedOptions), modifiedOptions.RetryPolicy, operationContext, token); } else { using (ICloudBlobStream blobStream = await this.OpenWriteAsync(accessCondition, options, operationContext).AsTask(token)) { // We should always call AsStreamForWrite with bufferSize=0 to prevent buffering. Our // stream copier only writes 64K buffers at a time anyway, so no buffering is needed. await sourceAsStream.WriteToAsync(blobStream.AsStreamForWrite(0), length, null /* maxLength */, false, tempExecutionState, null /* streamCopyState */, token); await blobStream.CommitAsync().AsTask(token); } } }); }
public void WriteToMultiBufferMemoryStreamTestAPM() { using (AutoResetEvent waitHandle = new AutoResetEvent(false)) { byte[] buffer = GetRandomBuffer(1 * 1024 * 1024); MemoryStream stream1 = new MemoryStream(buffer); RESTCommand<NullType> cmd = new RESTCommand<NullType>(TestBase.StorageCredentials, null); ExecutionState<NullType> state = new ExecutionState<NullType>(cmd, new NoRetry(), new OperationContext()); StreamDescriptor copyState = new StreamDescriptor(); MultiBufferMemoryStream stream2 = new MultiBufferMemoryStream(null /* bufferManager */); stream1.WriteToAsync(stream2, null, null, false, state, copyState, _ => waitHandle.Set()); waitHandle.WaitOne(); if (state.ExceptionRef != null) { throw state.ExceptionRef; } stream1.Seek(0, SeekOrigin.Begin); stream2.Seek(0, SeekOrigin.Begin); TestHelper.AssertStreamsAreEqual(stream1, stream2); MultiBufferMemoryStream stream3 = new MultiBufferMemoryStream(null /* bufferManager */); IAsyncResult ar = stream2.BeginFastCopyTo(stream3, DateTime.Now.AddMinutes(-1), _ => waitHandle.Set(), null); waitHandle.WaitOne(); TestHelper.ExpectedException<TimeoutException>( () => stream3.EndFastCopyTo(ar), "Past expiration time should immediately fail"); stream2.Seek(0, SeekOrigin.Begin); stream3.Seek(0, SeekOrigin.Begin); ar = stream2.BeginFastCopyTo(stream3, DateTime.Now.AddHours(1), _ => waitHandle.Set(), null); waitHandle.WaitOne(); stream2.EndFastCopyTo(ar); stream2.Seek(0, SeekOrigin.Begin); stream3.Seek(0, SeekOrigin.Begin); TestHelper.AssertStreamsAreEqual(stream2, stream3); MultiBufferMemoryStream stream4 = new MultiBufferMemoryStream(null, 12345); ar = stream3.BeginFastCopyTo(stream4, null, _ => waitHandle.Set(), null); waitHandle.WaitOne(); stream3.EndFastCopyTo(ar); stream3.Seek(0, SeekOrigin.Begin); stream4.Seek(0, SeekOrigin.Begin); TestHelper.AssertStreamsAreEqual(stream3, stream4); state = new ExecutionState<NullType>(cmd, new NoRetry(), new OperationContext()); copyState = new StreamDescriptor(); MemoryStream stream5 = new MemoryStream(); stream4.WriteToAsync(stream5, null, null, false, state, copyState, _ => waitHandle.Set()); waitHandle.WaitOne(); if (state.ExceptionRef != null) { throw state.ExceptionRef; } stream4.Seek(0, SeekOrigin.Begin); stream5.Seek(0, SeekOrigin.Begin); TestHelper.AssertStreamsAreEqual(stream4, stream5); TestHelper.AssertStreamsAreEqual(stream1, stream5); } }
public void PutBlock(string blockId, Stream blockData, string contentMD5, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) { CommonUtility.AssertNotNull("blockData", blockData); BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient); bool requiresContentMD5 = (contentMD5 == null) && modifiedOptions.UseTransactionalMD5.Value; operationContext = operationContext ?? new OperationContext(); Stream seekableStream = blockData; if (!blockData.CanSeek || requiresContentMD5) { ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions); Stream writeToStream; if (blockData.CanSeek) { writeToStream = Stream.Null; } else { seekableStream = new MultiBufferMemoryStream(this.ServiceClient.BufferManager); writeToStream = seekableStream; } long startPosition = seekableStream.Position; StreamDescriptor streamCopyState = new StreamDescriptor(); blockData.WriteToSync(writeToStream, null /* copyLength */, Constants.MaxBlockSize, requiresContentMD5, true, tempExecutionState, streamCopyState); seekableStream.Position = startPosition; if (requiresContentMD5) { contentMD5 = streamCopyState.Md5; } } Executor.ExecuteSync( this.PutBlockImpl(seekableStream, blockId, contentMD5, accessCondition, modifiedOptions), modifiedOptions.RetryPolicy, operationContext); }
public void UploadFromStream(Stream source, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) { CommonUtils.AssertNotNull("source", source); this.attributes.AssertNoSnapshot(); BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient); operationContext = operationContext ?? new OperationContext(); DateTime? expiryTime = modifiedOptions.MaximumExecutionTime.HasValue ? DateTime.Now + modifiedOptions.MaximumExecutionTime.Value : (DateTime?)null; if ((this.ServiceClient.ParallelOperationThreadCount == 1) && source.CanSeek && ((source.Length - source.Position) <= this.ServiceClient.SingleBlobUploadThresholdInBytes)) { string contentMD5 = null; if (modifiedOptions.StoreBlobContentMD5.Value) { OperationContext tempOperationContext = new OperationContext(); StreamDescriptor streamCopyState = new StreamDescriptor(); long startPosition = source.Position; source.WriteToSync(Stream.Null, null /* maxLength */, expiryTime, true, true, tempOperationContext, streamCopyState); source.Position = startPosition; contentMD5 = streamCopyState.Md5; } Executor.ExecuteSync( this.PutBlobImpl(source, contentMD5, accessCondition, modifiedOptions), modifiedOptions.RetryPolicy, operationContext); } else { using (Stream blobStream = this.OpenWrite(accessCondition, modifiedOptions, operationContext)) { source.WriteToSync(blobStream, null /* maxLength */, expiryTime, false, true, new OperationContext(), null /* streamCopyState */); } } }
internal void UploadFromStreamHelper(Stream source, long? length, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext) { CommonUtility.AssertNotNull("source", source); if (length.HasValue) { CommonUtility.AssertInBounds("length", length.Value, 1); if (source.CanSeek && length > source.Length - source.Position) { throw new ArgumentOutOfRangeException("length", SR.StreamLengthShortError); } } this.attributes.AssertNoSnapshot(); BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient); operationContext = operationContext ?? new OperationContext(); bool lessThanSingleBlobThreshold = source.CanSeek && (length ?? source.Length - source.Position) <= this.ServiceClient.SingleBlobUploadThresholdInBytes; if (this.ServiceClient.ParallelOperationThreadCount == 1 && lessThanSingleBlobThreshold) { string contentMD5 = null; if (modifiedOptions.StoreBlobContentMD5.Value) { using (ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions)) { StreamDescriptor streamCopyState = new StreamDescriptor(); long startPosition = source.Position; source.WriteToSync(Stream.Null, length, null /* maxLength */, true, true, tempExecutionState, streamCopyState); source.Position = startPosition; contentMD5 = streamCopyState.Md5; } } else { // Throw exception if we need to use Transactional MD5 but cannot store it if (modifiedOptions.UseTransactionalMD5.Value) { throw new ArgumentException(SR.PutBlobNeedsStoreBlobContentMD5, "options"); } } Executor.ExecuteSync( this.PutBlobImpl(source, length, contentMD5, accessCondition, modifiedOptions), modifiedOptions.RetryPolicy, operationContext); } else { using (CloudBlobStream blobStream = this.OpenWrite(accessCondition, modifiedOptions, operationContext)) { using (ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions)) { source.WriteToSync(blobStream, length, null /* maxLength */, false, true, tempExecutionState, null /* streamCopyState */); blobStream.Commit(); } } } }
public ICancellableAsyncResult BeginUploadFromStream(Stream source, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext, AsyncCallback callback, object state) { CommonUtils.AssertNotNull("source", source); this.attributes.AssertNoSnapshot(); BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient); DateTime? expiryTime = modifiedOptions.MaximumExecutionTime.HasValue ? DateTime.Now + modifiedOptions.MaximumExecutionTime.Value : (DateTime?)null; OperationContext tempOperationContext = new OperationContext(); ExecutionState<NullType> executionState = new ExecutionState<NullType>(null /* cmd */, modifiedOptions.RetryPolicy, tempOperationContext); ChainedAsyncResult<NullType> chainedResult = new ChainedAsyncResult<NullType>(callback, state) { CancelDelegate = executionState.Cancel, }; if ((this.ServiceClient.ParallelOperationThreadCount == 1) && source.CanSeek && ((source.Length - source.Position) <= this.ServiceClient.SingleBlobUploadThresholdInBytes)) { if (modifiedOptions.StoreBlobContentMD5.Value) { long startPosition = source.Position; StreamDescriptor streamCopyState = new StreamDescriptor(); source.WriteToAsync( Stream.Null, null /* maxLength */, expiryTime, true, executionState, tempOperationContext, streamCopyState, completedState => { chainedResult.UpdateCompletedSynchronously(executionState.CompletedSynchronously); try { lock (chainedResult.CancellationLockerObject) { chainedResult.CancelDelegate = null; if (completedState.ExceptionRef != null) { chainedResult.OnComplete(completedState.ExceptionRef); } else { source.Position = startPosition; UploadFromStreamHandler(source, streamCopyState.Md5, accessCondition, operationContext, modifiedOptions, chainedResult); } } } catch (Exception e) { chainedResult.OnComplete(e); } }); } else { this.UploadFromStreamHandler(source, null /* contentMD5 */, accessCondition, operationContext, modifiedOptions, chainedResult); } } else { Stream blobStream = this.OpenWrite(accessCondition, modifiedOptions, operationContext); source.WriteToAsync( blobStream, null /* maxLength */, expiryTime, false, executionState, tempOperationContext, null /* streamCopyState */, completedState => { chainedResult.UpdateCompletedSynchronously(executionState.CompletedSynchronously); try { blobStream.Close(); chainedResult.OnComplete(executionState.ExceptionRef); } catch (Exception e) { chainedResult.OnComplete(e); } }); } return chainedResult; }
public void WritePages(Stream pageData, long startOffset, string contentMD5 = null, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) { BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.PageBlob, this.ServiceClient); bool requiresContentMD5 = (contentMD5 == null) && modifiedOptions.UseTransactionalMD5.Value; operationContext = operationContext ?? new OperationContext(); Stream seekableStream = pageData; if (!pageData.CanSeek || requiresContentMD5) { OperationContext tempOperationContext = new OperationContext(); DateTime? expiryTime = modifiedOptions.MaximumExecutionTime.HasValue ? DateTime.Now + modifiedOptions.MaximumExecutionTime.Value : (DateTime?)null; Stream writeToStream; if (pageData.CanSeek) { writeToStream = Stream.Null; } else { seekableStream = new MemoryStream(); writeToStream = seekableStream; } long startPosition = seekableStream.Position; StreamDescriptor streamCopyState = new StreamDescriptor(); pageData.WriteToSync(writeToStream, Constants.MaxBlockSize, expiryTime, requiresContentMD5, true, tempOperationContext, streamCopyState); seekableStream.Position = startPosition; if (requiresContentMD5) { contentMD5 = streamCopyState.Md5; } } Executor.ExecuteSync( this.PutPageImpl(seekableStream, startOffset, contentMD5, accessCondition, modifiedOptions), modifiedOptions.RetryPolicy, operationContext); }
public virtual void WriteRange(Stream rangeData, long startOffset, string contentMD5 = null, AccessCondition accessCondition = null, FileRequestOptions options = null, OperationContext operationContext = null) { CommonUtility.AssertNotNull("rangeData", rangeData); FileRequestOptions modifiedOptions = FileRequestOptions.ApplyDefaults(options, this.ServiceClient); bool requiresContentMD5 = (contentMD5 == null) && modifiedOptions.UseTransactionalMD5.Value; operationContext = operationContext ?? new OperationContext(); Stream seekableStream = rangeData; bool seekableStreamCreated = false; try { if (!rangeData.CanSeek || requiresContentMD5) { ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions); Stream writeToStream; if (rangeData.CanSeek) { writeToStream = Stream.Null; } else { seekableStream = new MultiBufferMemoryStream(this.ServiceClient.BufferManager); seekableStreamCreated = true; writeToStream = seekableStream; } long startPosition = seekableStream.Position; StreamDescriptor streamCopyState = new StreamDescriptor(); rangeData.WriteToSync(writeToStream, null /* copyLength */, Constants.MaxBlockSize, requiresContentMD5, true, tempExecutionState, streamCopyState); seekableStream.Position = startPosition; if (requiresContentMD5) { contentMD5 = streamCopyState.Md5; } } Executor.ExecuteSync( this.PutRangeImpl(seekableStream, startOffset, contentMD5, accessCondition, modifiedOptions), modifiedOptions.RetryPolicy, operationContext); } finally { if (seekableStreamCreated) { seekableStream.Dispose(); } } }
internal static async Task WriteToAsync(this Stream stream, Stream toStream, long?maxLength, bool calculateMd5, OperationContext operationContext, StreamDescriptor streamCopyState, CancellationToken token) { byte[] buffer = new byte[GetBufferSize(stream)]; if (streamCopyState != null && calculateMd5 && streamCopyState.Md5HashRef == null) { streamCopyState.Md5HashRef = new MD5Wrapper(); } int readCount; do { readCount = await stream.ReadAsync(buffer, 0, buffer.Length, token); await toStream.WriteAsync(buffer, 0, readCount, token); // Update the StreamDescriptor after the bytes are successfully committed to the output stream if (streamCopyState != null) { streamCopyState.Length += readCount; if (maxLength.HasValue && streamCopyState.Length > maxLength.Value) { throw new ArgumentOutOfRangeException("stream"); } if (streamCopyState.Md5HashRef != null) { streamCopyState.Md5HashRef.UpdateHash(buffer, 0, readCount); } } }while (readCount != 0); if (streamCopyState != null && streamCopyState.Md5HashRef != null) { streamCopyState.Md5 = streamCopyState.Md5HashRef.ComputeHash(); streamCopyState.Md5HashRef = null; } }
public IAsyncAction WriteRangeAsync(IInputStream rangeData, long startOffset, string contentMD5, AccessCondition accessCondition, FileRequestOptions options, OperationContext operationContext) #endif { CommonUtility.AssertNotNull("rangeData", rangeData); FileRequestOptions modifiedOptions = FileRequestOptions.ApplyDefaults(options, this.ServiceClient); bool requiresContentMD5 = (contentMD5 == null) && modifiedOptions.UseTransactionalMD5.Value; operationContext = operationContext ?? new OperationContext(); ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions); #if ASPNET_K return Task.Run(async () => #else return AsyncInfo.Run(async (cancellationToken) => #endif { DateTime streamCopyStartTime = DateTime.Now; Stream rangeDataAsStream = rangeData.AsStreamForRead(); Stream seekableStream = rangeDataAsStream; if (!rangeDataAsStream.CanSeek || requiresContentMD5) { Stream writeToStream; if (rangeDataAsStream.CanSeek) { writeToStream = Stream.Null; } else { seekableStream = new MultiBufferMemoryStream(this.ServiceClient.BufferManager); writeToStream = seekableStream; } StreamDescriptor streamCopyState = new StreamDescriptor(); long startPosition = seekableStream.Position; await rangeDataAsStream.WriteToAsync(writeToStream, null /* copyLength */, Constants.MaxBlockSize, requiresContentMD5, tempExecutionState, streamCopyState, cancellationToken); seekableStream.Position = startPosition; if (requiresContentMD5) { contentMD5 = streamCopyState.Md5; } if (modifiedOptions.MaximumExecutionTime.HasValue) { modifiedOptions.MaximumExecutionTime -= DateTime.Now.Subtract(streamCopyStartTime); } } await Executor.ExecuteAsyncNullReturn( this.PutRangeImpl(seekableStream, startOffset, contentMD5, accessCondition, modifiedOptions), modifiedOptions.RetryPolicy, operationContext, cancellationToken); #if ASPNET_K }, cancellationToken); #else });
internal static void WriteToSync(this Stream stream, Stream toStream, long?maxLength, DateTime?expiryTime, bool calculateMd5, bool syncRead, OperationContext operationContext, StreamDescriptor streamCopyState) { byte[] buffer = new byte[GetBufferSize(stream)]; if (streamCopyState != null && calculateMd5 && streamCopyState.Md5HashRef == null) { streamCopyState.Md5HashRef = new MD5Wrapper(); } int readCount; do { if (expiryTime.HasValue && DateTime.Now.CompareTo(expiryTime.Value) > 0) { throw Exceptions.GenerateTimeoutException(operationContext.LastResult, null); } if (syncRead) { readCount = stream.Read(buffer, 0, buffer.Length); } else { // Use an async read since Sync read on ConnectStream will not throw for prematurely closed connections readCount = stream.EndRead(stream.BeginRead(buffer, 0, buffer.Length, null /* Callback */, null /* State */)); } toStream.Write(buffer, 0, readCount); // Update the StreamDescriptor after the bytes are successfully committed to the output stream if (streamCopyState != null) { streamCopyState.Length += readCount; if (maxLength.HasValue && streamCopyState.Length > maxLength.Value) { throw new ArgumentOutOfRangeException("stream"); } if (streamCopyState.Md5HashRef != null) { streamCopyState.Md5HashRef.UpdateHash(buffer, 0, readCount); } } }while (readCount != 0); if (streamCopyState != null && streamCopyState.Md5HashRef != null) { streamCopyState.Md5 = streamCopyState.Md5HashRef.ComputeHash(); streamCopyState.Md5HashRef = null; } }
public IAsyncAction UploadFromStreamAsync(IInputStream source, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext) { CommonUtils.AssertNotNull("source", source); this.attributes.AssertNoSnapshot(); BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient); operationContext = operationContext ?? new OperationContext(); Stream sourceAsStream = source.AsStreamForRead(); return AsyncInfo.Run(async (token) => { DateTime streamCopyStartTime = DateTime.Now; CancellationToken streamCopyToken = token; if (modifiedOptions.MaximumExecutionTime.HasValue) { // Setup token CancellationTokenSource cts = new CancellationTokenSource(modifiedOptions.MaximumExecutionTime.Value); CancellationToken tokenWithTimeout = cts.Token; // Hookup users token token.Register(() => cts.Cancel()); streamCopyToken = tokenWithTimeout; } if ((this.ServiceClient.ParallelOperationThreadCount == 1) && sourceAsStream.CanSeek && ((sourceAsStream.Length - sourceAsStream.Position) <= this.ServiceClient.SingleBlobUploadThresholdInBytes)) { string contentMD5 = null; if (modifiedOptions.StoreBlobContentMD5.Value) { OperationContext tempOperationContext = new OperationContext(); StreamDescriptor streamCopyState = new StreamDescriptor(); long startPosition = sourceAsStream.Position; await sourceAsStream.WriteToAsync(Stream.Null, null /* maxLength */, true, tempOperationContext, streamCopyState, streamCopyToken); sourceAsStream.Position = startPosition; contentMD5 = streamCopyState.Md5; } if (modifiedOptions.MaximumExecutionTime.HasValue) { modifiedOptions.MaximumExecutionTime -= DateTime.Now.Subtract(streamCopyStartTime); } await Executor.ExecuteAsyncNullReturn( this.PutBlobImpl(sourceAsStream, contentMD5, accessCondition, modifiedOptions), modifiedOptions.RetryPolicy, operationContext, token); } else { await Task.Run(async () => { IOutputStream writeStream = await this.OpenWriteAsync(accessCondition, options, operationContext); // We should always call AsStreamForWrite with bufferSize=0 to prevent buffering. Our // stream copier only writes 64K buffers at a time anyway, so no buffering is needed. using (Stream blobStream = writeStream.AsStreamForWrite(0)) { await sourceAsStream.WriteToAsync(blobStream, null /* maxLength */, false, new OperationContext(), null /* streamCopyState */, streamCopyToken); } }); } }); }
internal static void WriteToAsync <T>(this Stream stream, Stream toStream, long?copyLength, long?maxLength, bool calculateMd5, ExecutionState <T> executionState, StreamDescriptor streamCopyState, Action <ExecutionState <T> > completed) { AsyncStreamCopier <T> copier = new AsyncStreamCopier <T>(stream, toStream, executionState, GetBufferSize(stream), calculateMd5, streamCopyState); copier.StartCopyStream(completed, copyLength, maxLength); }
internal void UploadFromStreamHelper(Stream source, long? length, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext) { CommonUtility.AssertNotNull("source", source); if (length.HasValue) { CommonUtility.AssertInBounds("length", length.Value, 1); if (source.CanSeek && length > source.Length - source.Position) { throw new ArgumentOutOfRangeException("length", SR.StreamLengthShortError); } } this.attributes.AssertNoSnapshot(); BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient); operationContext = operationContext ?? new OperationContext(); bool lessThanSingleBlobThreshold = CloudBlockBlob.IsLessThanSingleBlobThreshold(source, length, modifiedOptions, false); modifiedOptions.AssertPolicyIfRequired(); if (modifiedOptions.ParallelOperationThreadCount.Value == 1 && lessThanSingleBlobThreshold) { bool usingEncryption = modifiedOptions.EncryptionPolicy != null; Stream sourceStream = source; using (MemoryStream tempStream = !usingEncryption ? null : new MemoryStream()) { // Encrypt if necessary if (usingEncryption) { modifiedOptions.AssertPolicyIfRequired(); if (modifiedOptions.EncryptionPolicy.EncryptionMode != BlobEncryptionMode.FullBlob) { throw new InvalidOperationException(SR.InvalidEncryptionMode, null); } ICryptoTransform transform = modifiedOptions.EncryptionPolicy.CreateAndSetEncryptionContext(this.Metadata, false /* noPadding */); CryptoStream cryptoStream = new CryptoStream(tempStream, transform, CryptoStreamMode.Write); using (ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(options)) { source.WriteToSync(cryptoStream, length, null, false, true, tempExecutionState, null); cryptoStream.FlushFinalBlock(); } // After the tempStream has been written to, we need to seek back to the beginning, so that it can be read from. tempStream.Seek(0, SeekOrigin.Begin); length = tempStream.Length; sourceStream = tempStream; } // Calculate MD5 if necessary // Note that we cannot do this while we encrypt, it must be a separate step, because we want the MD5 of the encrypted data, // not the unencrypted data. string contentMD5 = null; if (modifiedOptions.StoreBlobContentMD5.Value) { using (ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions)) { StreamDescriptor streamCopyState = new StreamDescriptor(); long startPosition = sourceStream.Position; sourceStream.WriteToSync(Stream.Null, length, null /* maxLength */, true, true, tempExecutionState, streamCopyState); sourceStream.Position = startPosition; contentMD5 = streamCopyState.Md5; } } else { // Throw exception if we need to use Transactional MD5 but cannot store it if (modifiedOptions.UseTransactionalMD5.Value) { throw new ArgumentException(SR.PutBlobNeedsStoreBlobContentMD5, "options"); } } // Execute the put blob. Executor.ExecuteSync( this.PutBlobImpl(sourceStream, length, contentMD5, accessCondition, modifiedOptions), modifiedOptions.RetryPolicy, operationContext); } } else { using (CloudBlobStream blobStream = this.OpenWrite(accessCondition, modifiedOptions, operationContext)) { using (ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions)) { source.WriteToSync(blobStream, length, null /* maxLength */, false, true, tempExecutionState, null /* streamCopyState */); blobStream.Commit(); } } } }
internal ICancellableAsyncResult BeginUploadFromStreamHelper(Stream source, long? length, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext, AsyncCallback callback, object state) { CommonUtility.AssertNotNull("source", source); if (length.HasValue) { CommonUtility.AssertInBounds("length", length.Value, 1); if (source.CanSeek && length > source.Length - source.Position) { throw new ArgumentOutOfRangeException("length", SR.StreamLengthShortError); } } this.attributes.AssertNoSnapshot(); BlobRequestOptions modifiedOptions = BlobRequestOptions.ApplyDefaults(options, BlobType.BlockBlob, this.ServiceClient); ExecutionState<NullType> tempExecutionState = CommonUtility.CreateTemporaryExecutionState(modifiedOptions); StorageAsyncResult<NullType> storageAsyncResult = new StorageAsyncResult<NullType>(callback, state); bool lessThanSingleBlobThreshold = CloudBlockBlob.IsLessThanSingleBlobThreshold(source, length, modifiedOptions, false); modifiedOptions.AssertPolicyIfRequired(); if (modifiedOptions.ParallelOperationThreadCount.Value == 1 && lessThanSingleBlobThreshold) { // Because we may or may not want to calculate the MD5, and we may or may not want to encrypt, rather than have four branching code // paths, here we have an action that we will run, which continually gets added to, depending on which operations we need to do. // The confusing part is that we have to build it from the bottom up. string md5 = null; Stream sourceStream = source; Action actionToRun = null; Action uploadAction = () => { if (md5 == null && modifiedOptions.UseTransactionalMD5.Value) { throw new ArgumentException(SR.PutBlobNeedsStoreBlobContentMD5, "options"); } this.UploadFromStreamHandler( sourceStream, length, md5, accessCondition, operationContext, modifiedOptions, storageAsyncResult); }; actionToRun = uploadAction; if (modifiedOptions.StoreBlobContentMD5.Value) { Action<Action> calculateMD5 = (continuation) => { long startPosition = sourceStream.Position; StreamDescriptor streamCopyState = new StreamDescriptor(); sourceStream.WriteToAsync( Stream.Null, length, null /* maxLength */, true, tempExecutionState, streamCopyState, completedState => { ContinueAsyncOperation(storageAsyncResult, completedState, () => { if (completedState.ExceptionRef != null) { storageAsyncResult.OnComplete(completedState.ExceptionRef); } else { sourceStream.Position = startPosition; md5 = streamCopyState.Md5; continuation(); } }); }); storageAsyncResult.CancelDelegate = tempExecutionState.Cancel; if (storageAsyncResult.CancelRequested) { storageAsyncResult.Cancel(); } }; Action oldActionToRun = actionToRun; actionToRun = () => calculateMD5(oldActionToRun); } if (modifiedOptions.EncryptionPolicy != null) { Action<Action> encryptStream = continuation => { SyncMemoryStream syncMemoryStream = new SyncMemoryStream(); options.AssertPolicyIfRequired(); sourceStream = syncMemoryStream; if (modifiedOptions.EncryptionPolicy.EncryptionMode != BlobEncryptionMode.FullBlob) { throw new InvalidOperationException(SR.InvalidEncryptionMode, null); } ICryptoTransform transform = options.EncryptionPolicy.CreateAndSetEncryptionContext(this.Metadata, false /* noPadding */); CryptoStream cryptoStream = new CryptoStream(syncMemoryStream, transform, CryptoStreamMode.Write); StreamDescriptor streamCopyState = new StreamDescriptor(); source.WriteToAsync(cryptoStream, length, null, false, tempExecutionState, streamCopyState, completedState => { ContinueAsyncOperation(storageAsyncResult, completedState, () => { if (completedState.ExceptionRef != null) { storageAsyncResult.OnComplete(completedState.ExceptionRef); } else { // Flush the CryptoStream in order to make sure that the last block of data is flushed. This call is a sync call // but it is ok to have it because we're just writing to a memory stream. cryptoStream.FlushFinalBlock(); // After the tempStream has been written to, we need to seek back to the beginning, so that it can be read from. sourceStream.Seek(0, SeekOrigin.Begin); length = syncMemoryStream.Length; continuation(); } }); }); storageAsyncResult.CancelDelegate = tempExecutionState.Cancel; if (storageAsyncResult.CancelRequested) { storageAsyncResult.Cancel(); } }; Action oldActionToRun = actionToRun; actionToRun = () => encryptStream(oldActionToRun); } actionToRun(); } else { ICancellableAsyncResult result = this.BeginOpenWrite( accessCondition, modifiedOptions, operationContext, ar => { ContinueAsyncOperation(storageAsyncResult, ar, () => { CloudBlobStream blobStream = this.EndOpenWrite(ar); storageAsyncResult.OperationState = blobStream; source.WriteToAsync( blobStream, length, null /* maxLength */, false, tempExecutionState, null /* streamCopyState */, completedState => { ContinueAsyncOperation(storageAsyncResult, completedState, () => { if (completedState.ExceptionRef != null) { storageAsyncResult.OnComplete(completedState.ExceptionRef); } else { ICancellableAsyncResult commitResult = blobStream.BeginCommit( CloudBlob.BlobOutputStreamCommitCallback, storageAsyncResult); storageAsyncResult.CancelDelegate = commitResult.Cancel; if (storageAsyncResult.CancelRequested) { storageAsyncResult.Cancel(); } } }); }); storageAsyncResult.CancelDelegate = tempExecutionState.Cancel; if (storageAsyncResult.CancelRequested) { storageAsyncResult.Cancel(); } }); }, null /* state */); // We do not need to do this inside a lock, as storageAsyncResult is // not returned to the user yet. storageAsyncResult.CancelDelegate = result.Cancel; } return storageAsyncResult; }
internal static Task WriteToAsync <T>(this Stream stream, Stream toStream, IBufferManager bufferManager, long?copyLength, long?maxLength, bool calculateMd5, ExecutionState <T> executionState, StreamDescriptor streamCopyState, CancellationToken cancellationToken, Action <ExecutionState <T> > completed = null) { AsyncStreamCopier <T> copier = new AsyncStreamCopier <T>(stream, toStream, executionState, bufferManager, GetBufferSize(stream), calculateMd5, streamCopyState); return(copier.StartCopyStream(completed, copyLength, maxLength, cancellationToken)); }