/// <summary>
/// Returns an enumerable collection of tables segmented implementation.
/// </summary>
/// <param name="prefix">The prefix.</param>
/// <param name="continuationToken">The continuation token.</param>
/// <param name="pagination">The pagination.</param>
/// <param name="lastResult">The last result.</param>
/// <param name="setResult">The set result.</param>
/// <returns>A sequence of tasks to do the operation.</returns>
private TaskSequence ListTablesSegmentedImplCore(
string prefix,
ResultContinuation continuationToken,
ResultPagination pagination,
ResultSegment <TableServiceTable> lastResult,
Action <ResultSegment <string> > setResult)
{
CommonUtils.AssertContinuationType(continuationToken, ResultContinuation.ContinuationType.Table);
InvokeTaskSequenceTask <ResultSegment <TableServiceTable> > listTablesSegmentedTask;
if (lastResult == null)
{
var svc = this.GetDataServiceContext();
var query = from table in svc.CreateQuery <TableServiceTable>(Protocol.Constants.TableServiceTablesName)
select table;
if (prefix != string.Empty)
{
// Append Max char to end '{' is 1 + 'z' in AsciiTable
string uppperBound = prefix + '{';
query = query.Where((table) => table.TableName.CompareTo(prefix) >= 0 && table.TableName.CompareTo(uppperBound) < 0);
}
if (pagination.IsPagingEnabled)
{
query = query.Take(pagination.GetNextRequestPageSize().Value);
}
var listTablesQuery = query.AsTableServiceQuery();
listTablesSegmentedTask = new InvokeTaskSequenceTask <ResultSegment <TableServiceTable> >(
(setResultInner) =>
listTablesQuery.ExecuteSegmentedImpl(continuationToken, setResultInner));
}
else
{
listTablesSegmentedTask = new InvokeTaskSequenceTask <ResultSegment <TableServiceTable> >(lastResult.GetNextImpl);
}
yield return(listTablesSegmentedTask);
if (GetResultOrDefault <ResultSegment <TableServiceTable> >(listTablesSegmentedTask, out lastResult))
{
setResult(new ResultSegment <string>(
lastResult.Results.Select((table) => table.TableName),
lastResult.HasMoreResults,
(setResultInner) =>
this.ListTablesSegmentedImplCore(prefix, lastResult.ContinuationToken, pagination, lastResult, setResultInner),
RetryPolicy)
{
ContinuationToken = lastResult.ContinuationToken
});
}
else
{
setResult(new ResultSegment <string>(new List <string>(), false, null, RetryPolicy));
}
}
/// <summary>
/// Generates a task sequence for getting the properties of the queue service.
/// </summary>
/// <param name="setResult">A delegate to receive the service properties.</param>
/// <returns>A task sequence that gets the properties of the queue service.</returns>
private TaskSequence GetServicePropertiesImpl(Action <ServiceProperties> setResult)
{
HttpWebRequest request = QueueRequest.GetServiceProperties(this.BaseUri, this.Timeout.RoundUpToSeconds());
CommonUtils.ApplyRequestOptimizations(request, -1);
this.Credentials.SignRequest(request);
// Get the web response.
Task <WebResponse> responseTask = request.GetResponseAsyncWithTimeout(this, this.Timeout);
yield return(responseTask);
using (HttpWebResponse response = responseTask.Result as HttpWebResponse)
using (Stream responseStream = response.GetResponseStream())
using (MemoryStream memoryStream = new MemoryStream())
{
// Download the service properties.
Task <NullTaskReturn> downloadTask = new InvokeTaskSequenceTask(() => { return(responseStream.WriteTo(memoryStream)); });
yield return(downloadTask);
// Materialize any exceptions.
NullTaskReturn scratch = downloadTask.Result;
// Get the result from the memory stream.
memoryStream.Seek(0, SeekOrigin.Begin);
setResult(QueueResponse.ReadServiceProperties(memoryStream));
}
}
/// <summary>
/// Implements the flushing sequence.
/// </summary>
/// <returns>The sequence of events for a flush.</returns>
private TaskSequence FlushInternal()
{
// If there's nothing to flush, just return;
if (this.blockBuffer == null)
{
yield break;
}
// Rewind the stream before upload
this.blockBuffer.Position = 0;
InvokeTaskSequenceTask newTask;
if (this.UseBlocks)
{
newTask = new InvokeTaskSequenceTask(this.UploadBlock);
}
else
{
newTask = new InvokeTaskSequenceTask(this.UploadBlob);
}
yield return(newTask);
var result = newTask.Result; // Materialize the results to ensure exception propagation
}
internal static IAsyncResult BeginImpl <T>(Func <Action <T>, TaskSequence> impl, AsyncCallback callback, object state)
{
CommonUtils.AssertNotNull("impl", impl);
var invokerTask = new InvokeTaskSequenceTask <T>(impl);
return(invokerTask.ToAsyncResult(callback, state));
}
internal static T ExecuteImpl <T>(Func <Action <T>, TaskSequence> impl)
{
CommonUtils.AssertNotNull("impl", impl);
var invokerTask = new InvokeTaskSequenceTask <T>(impl);
return(invokerTask.ExecuteAndWait());
}
internal static void ExecuteImpl(Func <TaskSequence> impl)
{
CommonUtils.AssertNotNull("impl", impl);
InvokeTaskSequenceTask invokerTask = new InvokeTaskSequenceTask(impl);
invokerTask.ExecuteAndWait();
}
/// <summary>
/// Uploads the block list.
/// </summary>
/// <param name="blocks">The blocks to upload.</param>
/// <param name="options">An object that specifies any additional options for the request.</param>
/// <returns>A <see cref="TaskSequence"/> that uploads the block list.</returns>
internal TaskSequence UploadBlockList(List <PutBlockListItem> blocks, BlobRequestOptions options)
{
if (options == null)
{
throw new ArgumentNullException("modifers");
}
var request = ProtocolHelper.GetWebRequest(this.ServiceClient, options, (timeout) => BlobRequest.PutBlockList(this.TransformedAddress, timeout, this.Properties, null));
options.AccessCondition.ApplyCondition(request);
BlobRequest.AddMetadata(request, this.Metadata);
using (var memoryStream = new SmallBlockMemoryStream(Constants.DefaultBufferSize))
{
BlobRequest.WriteBlockListBody(blocks, memoryStream);
CommonUtils.ApplyRequestOptimizations(request, memoryStream.Length);
memoryStream.Seek(0, SeekOrigin.Begin);
// Compute the MD5
var md5 = System.Security.Cryptography.MD5.Create();
request.Headers[HttpRequestHeader.ContentMd5] = Convert.ToBase64String(md5.ComputeHash(memoryStream));
this.ServiceClient.Credentials.SignRequest(request);
memoryStream.Seek(0, SeekOrigin.Begin);
// Retrieve the stream
var requestStreamTask = request.GetRequestStreamAsync();
yield return(requestStreamTask);
using (Stream requestStream = requestStreamTask.Result)
{
// Copy the data
var copyTask = new InvokeTaskSequenceTask(() => { return(memoryStream.WriteTo(requestStream)); });
yield return(copyTask);
// Materialize any exceptions
var scratch = copyTask.Result;
Console.WriteLine(scratch);
}
}
// Get the response
var responseTask = request.GetResponseAsyncWithTimeout(this.ServiceClient, options.Timeout);
yield return(responseTask);
using (var response = responseTask.Result as HttpWebResponse)
{
ParseSizeAndLastModified(response);
this.Properties.Length = 0;
}
}
internal static Task <T> GetRetryableAsyncTask <T>(Func <Action <T>, TaskSequence> impl, RetryPolicy policy)
{
CommonUtils.AssertNotNull("impl", impl);
CommonUtils.AssertNotNull("policy", policy);
ShouldRetry oracle = policy();
InvokeTaskSequenceTask <T> retryableTask = new InvokeTaskSequenceTask <T>(
(setResult) => RequestWithRetry.RequestWithRetryImpl <T>(oracle, impl, setResult));
return(retryableTask);
}
/// <summary>
/// Creates the table if not exist implementation.
/// </summary>
/// <param name="tableName">The table name.</param>
/// <param name="setResult">The set result.</param>
/// <returns>A sequence of tasks to do the operation.</returns>
private TaskSequence CreateTableIfNotExistImpl(string tableName, Action <bool> setResult)
{
CommonUtils.CheckStringParameter(tableName, false, "tableName", Protocol.Constants.TableServiceMaxStringPropertySizeInChars);
TableServiceUtilities.CheckTableName(tableName, "tableName");
var doesTableExistTask = new InvokeTaskSequenceTask <bool>((set) => this.DoesTableExistImpl(tableName, set));
yield return(doesTableExistTask);
if (doesTableExistTask.Result)
{
setResult(false);
}
else
{
var createTableTask = TaskImplHelper.GetRetryableAsyncTask <InvalidOperationException>((resultSetter) => this.CreateTableImpl(tableName, resultSetter), RetryPolicy);
yield return(createTableTask);
// wrap any exceptions
try
{
if (createTableTask.Result == null)
{
setResult(true);
}
else
{
StorageClientException exception = Utilities.TranslateDataServiceClientException(createTableTask.Result) as StorageClientException;
if (exception != null &&
exception.ErrorCode == StorageErrorCode.ResourceAlreadyExists &&
exception.ExtendedErrorInformation != null &&
exception.ExtendedErrorInformation.ErrorCode == TableErrorCodeStrings.TableAlreadyExists)
{
setResult(false);
}
else
{
throw createTableTask.Result;
}
}
}
catch (InvalidOperationException ex)
{
throw Utilities.TranslateDataServiceClientException(ex);
}
}
}
/// <summary>
/// Generates a task sequence for setting the properties of the queue service.
/// </summary>
/// <param name="properties">The queue service properties to set.</param>
/// <returns>A task sequence that sets the properties of the queue service.</returns>
private TaskSequence SetServicePropertiesImpl(ServiceProperties properties)
{
CommonUtils.AssertNotNull("properties", properties);
HttpWebRequest request = QueueRequest.SetServiceProperties(this.BaseUri, this.Timeout.RoundUpToSeconds());
using (MemoryStream memoryStream = new MemoryStream())
{
try
{
QueueRequest.WriteServiceProperties(properties, memoryStream);
}
catch (InvalidOperationException invalidOpException)
{
throw new ArgumentException(invalidOpException.Message, "properties");
}
memoryStream.Seek(0, SeekOrigin.Begin);
CommonUtils.ApplyRequestOptimizations(request, memoryStream.Length);
this.Credentials.SignRequest(request);
// Get the request stream
Task <Stream> getStreamTask = request.GetRequestStreamAsync();
yield return(getStreamTask);
using (Stream requestStream = getStreamTask.Result)
{
// Upload the service properties.
Task <NullTaskReturn> uploadTask = new InvokeTaskSequenceTask(() => { return((memoryStream as Stream).WriteTo(requestStream)); });
yield return(uploadTask);
// Materialize any exceptions.
NullTaskReturn scratch = uploadTask.Result;
Console.WriteLine(scratch);
}
}
// Get the web response.
Task <WebResponse> responseTask = request.GetResponseAsyncWithTimeout(this, this.Timeout);
yield return(responseTask);
// Materialize any exceptions.
using (HttpWebResponse response = responseTask.Result as HttpWebResponse)
{
}
}
internal static Task <NullTaskReturn> GetRetryableAsyncTask(Func <TaskSequence> impl, RetryPolicy policy)
{
CommonUtils.AssertNotNull("impl", impl);
CommonUtils.AssertNotNull("policy", policy);
ShouldRetry oracle = policy();
InvokeTaskSequenceTask retryableTask = new InvokeTaskSequenceTask(
() => RequestWithRetry.RequestWithRetryImpl <NullTaskReturn>(
oracle,
(setResult) => impl(),
(scratch) =>
{
}));
return(retryableTask);
}
/// <summary>
/// Commits the blob by uploading any remaining data and the blocklist asynchronously.
/// </summary>
/// <returns>A sequence of events required for commit.</returns>
private TaskSequence CommitImpl()
{
this.CheckWriteState();
if (this.blockBuffer != null && this.blockBuffer.Length != 0)
{
var task = new InvokeTaskSequenceTask(this.FlushInternal);
yield return(task);
var result = task.Result; // Materialize the errors
Console.WriteLine(result);
}
// If all blocks are uploaded, commit
if (this.UseBlocks)
{
this.SetBlobMD5();
var task = TaskImplHelper.GetRetryableAsyncTask(
() =>
{
// At the convenience layer we always upload uncommitted blocks
List <PutBlockListItem> putBlockList = new List <PutBlockListItem>();
foreach (var id in this.blockList)
{
putBlockList.Add(new PutBlockListItem(id, BlockSearchMode.Uncommitted));
}
return(this.Blob.ToBlockBlob.UploadBlockList(putBlockList, this.currentModifier));
},
this.currentModifier.RetryPolicy);
yield return(task);
var result = task.Result;
Console.WriteLine(result);
}
// Now we can set the full size.
this.Blob.Properties.Length = this.Length;
// Clear the internal state
this.Abort();
}
/// <summary>
/// Begins an asynchronous write operation.
/// </summary>
/// <param name="buffer">The buffer to write data from.</param>
/// <param name="offset">The byte offset in buffer from which to begin writing.</param>
/// <param name="count">The number of bytes to write.</param>
/// <param name="callback">An optional asynchronous callback, to be called when the write is complete.</param>
/// <param name="state">A user-provided object that distinguishes this particular asynchronous write request from other requests.</param>
/// <returns>An IAsyncResult that represents the asynchronous write, which could still be pending.</returns>
/// <exception cref="System.ArgumentNullException"><paramref name="buffer"/> is null.</exception>
/// <exception cref="System.ArgumentOutOfRangeException">offset or count is negative.</exception>
/// <exception cref="System.ObjectDisposedException">Thrown if blob is already committed/closed</exception>
/// <remarks>The operation will be completed synchronously if the buffer is not filled</remarks>
public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback callback, object state)
{
this.CheckWriteState();
StreamUtilities.CheckBufferArguments(buffer, offset, count);
Task <NullTaskReturn> task = null;
if (this.UseBlocks)
{
task = new InvokeTaskSequenceTask(() =>
{
return(this.WriteBlockBlobImpl(buffer, offset, count));
});
}
else
{
task = this.WriteNonBlockedBlobImpl(buffer, offset, count);
}
return(task.ToAsyncResult(callback, state));
}
/// <summary>
/// Implements the block writing task.
/// </summary>
/// <param name="buffer">The buffer to write data from.</param>
/// <param name="offset">The byte offset in buffer from which to begin writing.</param>
/// <param name="count">The number of bytes to write.</param>
/// <returns>The sequence representing the uploading of the blocks.</returns>
private TaskSequence WriteBlockBlobImpl(byte[] buffer, int offset, int count)
{
if (this.blockList.Count + (count / this.blockSize) > Constants.MaxBlockNumber)
{
throw new ArgumentOutOfRangeException("count", String.Format(CultureInfo.CurrentCulture, SR.TooManyBlocksError, Constants.MaxBlockNumber));
}
while (count != 0)
{
// Create a buffer if we don't have one
if (this.blockBuffer == null)
{
this.CreateNewBlock();
}
// Copy enough data to fill the buffer.
int numCopied = (int)Math.Min(count, this.blockSize - this.blockBuffer.Length);
this.blockBuffer.Write(buffer, offset, numCopied);
StreamUtilities.ComputeHash(buffer, offset, numCopied, this.blockHash);
StreamUtilities.ComputeHash(buffer, offset, numCopied, this.blobHash);
// Advance the location
this.position += numCopied;
offset += numCopied;
count -= numCopied;
// If buffer is full, flush
if (this.blockBuffer.Length == this.blockSize)
{
var newTask = new InvokeTaskSequenceTask(this.FlushInternal);
yield return(newTask);
// Make sure task completed successfully by materializing the exception
var result = newTask.Result;
Console.WriteLine(result);
}
}
}
/// <summary>
/// Uploads the data into the web request.
/// </summary>
/// <param name="request">The request that is setup for a put.</param>
/// <param name="source">The source of data.</param>
/// <param name="result">The response from the server.</param>
/// <returns>The sequence used for uploading data.</returns>
internal TaskSequence UploadData(HttpWebRequest request, Stream source, Action<WebResponse> result)
{
// Retrieve the stream
var requestStreamTask = request.GetRequestStreamAsync();
yield return requestStreamTask;
// Copy the data
using (var outputStream = requestStreamTask.Result)
{
var copyTask = new InvokeTaskSequenceTask(() => { return source.WriteTo(outputStream); });
yield return copyTask;
// Materialize any exceptions
var scratch = copyTask.Result;
}
// Get the response
var responseTask = request.GetResponseAsyncWithTimeout(this.ServiceClient, this.ServiceClient.Timeout);
yield return responseTask;
// Return the response object
var response = responseTask.Result;
result(response);
}
/// <summary>
/// Perform a parallel upload of blocks for a blob from a given stream.
/// </summary>
/// <param name="uploadFunc">The upload func.</param>
/// <returns>A <see cref="TaskSequence"/> that uploads the blob in parallel.</returns>
/// <remarks>
/// The operation is done as a series of alternate producer and consumer tasks. The producer tasks dispense out
/// chunks of source stream as fixed size blocks. This is done in serial order on a thread using InvokeTaskSequence's
/// serial execution. The consumer tasks upload each block in parallel on multiple thread. The producer thread waits
/// for at least one consumer task to finish before adding more producer tasks. The producer thread quits when no
/// more data can be read from the stream and no other pending consumer tasks.
/// </remarks>
internal TaskSequence ParallelExecute(
Func <SmallBlockMemoryStream, string, string, BlobRequestOptions, Task <NullTaskReturn> > uploadFunc)
{
bool moreToUpload = true;
List <IAsyncResult> asyncResults = new List <IAsyncResult>();
Random rand = new Random();
long blockIdSequenceNumber = (long)rand.Next() << 32;
blockIdSequenceNumber += rand.Next();
do
{
int currentPendingTasks = asyncResults.Count;
// Step 1
// Create producer tasks in a serial order as stream can only be read sequentially
for (int i = currentPendingTasks; i < this.parellelism && moreToUpload; i++)
{
string blockId = null;
string blockHash = null;
SmallBlockMemoryStream blockAsStream = null;
blockIdSequenceNumber++;
InvokeTaskSequenceTask producerTask = new InvokeTaskSequenceTask(() =>
this.DispenseBlockStream(
blockIdSequenceNumber,
(stream, id, hashVal) =>
{
blockAsStream = stream;
blockId = id;
blockHash = hashVal;
}));
yield return(producerTask);
this.producerTasksCreated++;
var scatch = producerTask.Result;
Console.WriteLine(scatch);
if (blockAsStream == null)
{
TraceHelper.WriteLine("No more upload tasks");
moreToUpload = false;
}
else
{
// Step 2
// Fire off consumer tasks that may finish on other threads;
var task = uploadFunc(blockAsStream, blockId, blockHash, this.options);
IAsyncResult asyncresult = task.ToAsyncResult(null, null);
this.consumerTasksCreated++;
asyncResults.Add(asyncresult);
}
}
// Step 3
// Wait for 1 or more consumer tasks to finish inorder to bound set of parallel tasks
if (asyncResults.Count > 0)
{
int waitTimeout = GetWaitTimeout(this.options);
TraceHelper.WriteLine("Starting wait");
int waitResult = WaitHandle.WaitAny(asyncResults.Select(result => result.AsyncWaitHandle).ToArray(), waitTimeout);
TraceHelper.WriteLine("Ending wait");
if (waitResult == WaitHandle.WaitTimeout)
{
throw TimeoutHelper.ThrowTimeoutError(this.options.Timeout.Value);
}
CompleteAsyncresult(asyncResults, waitResult);
// Optimize away any other completed tasks
for (int index = 0; index < asyncResults.Count; index++)
{
IAsyncResult result = asyncResults[index];
if (result.IsCompleted)
{
CompleteAsyncresult(asyncResults, index);
index--;
}
}
}
}while (moreToUpload || asyncResults.Count != 0);
TraceHelper.WriteLine(
"Total producer tasks created {0}, consumer tasks created {1} ",
this.producerTasksCreated,
this.consumerTasksCreated);
var commitTask = TaskImplHelper.GetRetryableAsyncTask(this.CommitBlob, this.options.RetryPolicy);
yield return(commitTask);
var commitTaskResult = commitTask.Result;
Console.WriteLine(commitTaskResult);
}
/// <summary>
/// Implementation method for the WritePage methods.
/// </summary>
/// <param name="pageData">The page data.</param>
/// <param name="startOffset">The start offset.</param>
/// <param name="sourceStreamPosition">The beginning position of the source stream prior to execution, negative if stream is unseekable.</param>
/// <param name="options">An object that specifies any additional options for the request.</param>
/// <returns>A <see cref="TaskSequence"/> that writes the pages.</returns>
private TaskSequence WritePageImpl(Stream pageData, long startOffset, long sourceStreamPosition, BlobRequestOptions options)
{
CommonUtils.AssertNotNull("options", options);
long length = pageData.Length;
// Logic to rewind stream on a retry
// For non seekable streams we need a way to detect the retry iteration so as to only attempt to execute once.
// The first attempt will have SourceStreamPosition = -1, which means the first iteration on a non seekable stream.
// The second attempt will have SourceStreamPosition = -2, anything below -1 is considered an abort. Since the Impl method
// does not have an execution context to be aware of what iteration is used the SourceStreamPosition is utilized as counter to
// differentiate between the first attempt and a retry.
if (sourceStreamPosition >= 0 && pageData.CanSeek)
{
if (sourceStreamPosition != pageData.Position)
{
pageData.Seek(sourceStreamPosition, 0);
}
}
else if (sourceStreamPosition < -1)
{
throw new NotSupportedException(string.Format(CultureInfo.CurrentCulture, SR.CannotRetryNonSeekableStreamError));
}
if (startOffset % Protocol.Constants.PageSize != 0)
{
CommonUtils.ArgumentOutOfRange("startOffset", startOffset);
}
long rangeStreamOffset = pageData.CanSeek ? pageData.Position : 0;
PutPageProperties properties = new PutPageProperties()
{
Range = new PageRange(startOffset, startOffset + length - rangeStreamOffset - 1),
PageWrite = PageWrite.Update,
};
if ((1 + properties.Range.EndOffset - properties.Range.StartOffset) % Protocol.Constants.PageSize != 0 ||
(1 + properties.Range.EndOffset - properties.Range.StartOffset) == 0)
{
CommonUtils.ArgumentOutOfRange("pageData", pageData);
}
var webRequest = ProtocolHelper.GetWebRequest(
this.ServiceClient,
options,
(timeout) => BlobRequest.PutPage(this.TransformedAddress, timeout, properties, null));
////BlobRequest.AddMetadata(webRequest, this.Metadata);
// Retrieve the stream
var requestStreamTask = webRequest.GetRequestStreamAsync();
yield return(requestStreamTask);
// Copy the data
using (var outputStream = requestStreamTask.Result)
{
var copyTask = new InvokeTaskSequenceTask(() => { return(pageData.WriteTo(outputStream)); });
yield return(copyTask);
// Materialize any exceptions
var scratch = copyTask.Result;
}
// signing request needs Size to be set
this.ServiceClient.Credentials.SignRequest(webRequest);
// Get the response
var responseTask = webRequest.GetResponseAsyncWithTimeout(this.ServiceClient, options.Timeout);
yield return(responseTask);
// Parse the response
using (HttpWebResponse webResponse = responseTask.Result as HttpWebResponse)
{
this.ParseSizeAndLastModified(webResponse);
}
}
/// <summary>
/// Reads the data from the service starting at the specified location. Verifies the block's signature (if required) and adds it to the buffered data.
/// </summary>
/// <param name="startPosition">The starting position of the read-ahead.</param>
/// <param name="length">The number of bytes to read ahead.</param>
/// <returns> An TaskSequence that represents the asynchronous read action. </returns>
private TaskSequence ReadAheadImpl(long startPosition, long length)
{
var webResponseTask = new InvokeTaskSequenceTask<Stream>((result) => { return this.Blob.GetStreamImpl(options, startPosition, length, result); });
yield return webResponseTask;
using (var stream = webResponseTask.Result)
{
this.LockToEtag();
if (this.IntegrityControlVerificationEnabled && this.Blob.Properties.BlobType == BlobType.BlockBlob)
{
long blockStartPosition = 0;
foreach (var block in this.blockList)
{
var blockSize = block.Size;
// Find the starting block
if (blockStartPosition < startPosition)
{
blockStartPosition += blockSize;
continue;
}
// Start creating blocks
var memoryStream = new SmallBlockMemoryStream(Constants.DefaultBufferSize);
var md5Check = MD5.Create();
int totalCopied = 0, numRead = 0;
do
{
byte[] buffer = new byte[Constants.DefaultBufferSize];
var numToRead = (int)Math.Min(buffer.Length, blockSize - totalCopied);
var readTask = stream.ReadAsyncEx(buffer, 0, numToRead);
yield return readTask;
numRead = readTask.Result;
if (numRead != 0)
{
// Verify the content
StreamUtilities.ComputeHash(buffer, 0, numRead, md5Check);
var writeTask = memoryStream.WriteAsyncEx(buffer, 0, numRead);
yield return writeTask;
//var scratch = writeTask.Result; // Materialize any exceptions
totalCopied += numRead;
}
}
while (numRead != 0 && totalCopied < blockSize);
// If we read something, act on it
if (totalCopied != 0)
{
// Verify the hash
string blockNameMD5Value = Utilities.ExtractMD5ValueFromBlockID(block.Name);
if (blockNameMD5Value != StreamUtilities.GetHashValue(md5Check))
{
throw new InvalidDataException("Blob data corrupted (integrity check failed)");
}
memoryStream.Position = 0; // Rewind the stream to allow for reading
this.downloadedBlocksList.Add(new DownloadedBlock(startPosition, memoryStream));
startPosition += blockSize;
blockStartPosition += blockSize;
}
else
{
break;
}
}
}
else
{
var memoryStream = new SmallBlockMemoryStream(Constants.DefaultBufferSize);
var copyTask = new InvokeTaskSequenceTask(() => { return stream.WriteTo(memoryStream); });
yield return copyTask;
//var scratch = copyTask.Result; // Materialize any errors
memoryStream.Position = 0; // Rewind the stream to allow for reading
this.downloadedBlocksList.Add(new DownloadedBlock(startPosition, memoryStream));
}
}
}
/// <summary>
/// Implementation method for the WritePage methods.
/// </summary>
/// <param name="pageData">The page data.</param>
/// <param name="startOffset">The start offset.</param>
/// <param name="sourceStreamPosition">The beginning position of the source stream prior to execution, negative if stream is unseekable.</param>
/// <param name="options">An object that specifies any additional options for the request.</param>
/// <returns>A <see cref="TaskSequence"/> that writes the pages.</returns>
private TaskSequence WritePageImpl(Stream pageData, long startOffset, long sourceStreamPosition, BlobRequestOptions options)
{
CommonUtils.AssertNotNull("options", options);
long length = pageData.Length;
// Logic to rewind stream on a retry
// HACK : for non seekable streams we need a way to detect the retry iteration so as to only attempt to execute once.
// The first attempt will have SourceStreamPosition = -1, which means the first iteration on a non seekable stream.
// The second attempt will have SourceStreamPosition = -2, anything below -1 is considered an abort. Since the Impl method
// does not have an executino context to be aware of what iteration is used the SourceStreamPosition is utilized as counter to
// differentiate between the first attempt and a retry.
if (sourceStreamPosition >= 0 && pageData.CanSeek)
{
if (sourceStreamPosition != pageData.Position)
{
pageData.Seek(sourceStreamPosition, 0);
}
}
else if (sourceStreamPosition < -1)
{
// TODO : Need to rewrite this to support buffering in XSCL2 so that retries can work on non seekable streams
throw new NotSupportedException(string.Format(CultureInfo.CurrentCulture, SR.CannotRetryNonSeekableStreamError));
}
if (startOffset % Protocol.Constants.PageSize != 0)
{
CommonUtils.ArgumentOutOfRange("startOffset", startOffset);
}
// TODO should reuse sourceStreamPoisition when the HACK above is removed, for readability using a new local variable
long rangeStreamOffset = pageData.CanSeek ? pageData.Position : 0;
PutPageProperties properties = new PutPageProperties()
{
Range = new PageRange(startOffset, startOffset + length - rangeStreamOffset - 1),
PageWrite = PageWrite.Update,
};
if ((1 + properties.Range.EndOffset - properties.Range.StartOffset) % Protocol.Constants.PageSize != 0 ||
(1 + properties.Range.EndOffset - properties.Range.StartOffset) == 0)
{
CommonUtils.ArgumentOutOfRange("pageData", pageData);
}
var webRequest = ProtocolHelper.GetWebRequest(
this.ServiceClient,
options,
(timeout) => BlobRequest.PutPage(this.TransformedAddress, timeout, properties, null));
////BlobRequest.AddMetadata(webRequest, this.Metadata);
// Retrieve the stream
var requestStreamTask = webRequest.GetRequestStreamAsync();
yield return requestStreamTask;
// Copy the data
using (var outputStream = requestStreamTask.Result)
{
var copyTask = new InvokeTaskSequenceTask(() => { return pageData.WriteTo(outputStream); });
yield return copyTask;
// Materialize any exceptions
var scratch = copyTask.Result;
}
// signing request needs Size to be set
this.ServiceClient.Credentials.SignRequest(webRequest);
// Get the response
var responseTask = webRequest.GetResponseAsyncWithTimeout(this.ServiceClient, options.Timeout);
yield return responseTask;
// Parse the response
using (HttpWebResponse webResponse = responseTask.Result as HttpWebResponse)
{
this.ParseSizeAndLastModified(webResponse);
}
}
/// <summary>
/// Implements the flushing sequence.
/// </summary>
/// <returns>The sequence of events for a flush.</returns>
private TaskSequence FlushInternal()
{
// If there's nothing to flush, just return;
if (this.blockBuffer == null)
{
yield break;
}
// Rewind the stream before upload
this.blockBuffer.Position = 0;
InvokeTaskSequenceTask newTask;
if (this.UseBlocks)
{
newTask = new InvokeTaskSequenceTask(this.UploadBlock);
}
else
{
newTask = new InvokeTaskSequenceTask(this.UploadBlob);
}
yield return newTask;
var result = newTask.Result; // Materialize the results to ensure exception propagation
Console.WriteLine(result);
}
/// <summary>
/// Begins an asynchronous write operation.
/// </summary>
/// <param name="buffer">The buffer to write data from.</param>
/// <param name="offset">The byte offset in buffer from which to begin writing.</param>
/// <param name="count">The number of bytes to write.</param>
/// <param name="callback">An optional asynchronous callback, to be called when the write is complete.</param>
/// <param name="state">A user-provided object that distinguishes this particular asynchronous write request from other requests.</param>
/// <returns>An IAsyncResult that represents the asynchronous write, which could still be pending.</returns>
/// <exception cref="System.ArgumentNullException"><paramref name="buffer"/> is null.</exception>
/// <exception cref="System.ArgumentOutOfRangeException">offset or count is negative.</exception>
/// <exception cref="System.ObjectDisposedException">Thrown if blob is already committed/closed</exception>
/// <remarks>The operation will be completed synchronously if the buffer is not filled</remarks>
public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback callback, object state)
{
this.CheckWriteState();
StreamUtilities.CheckBufferArguments(buffer, offset, count);
Task<NullTaskReturn> task = null;
if (this.UseBlocks)
{
task = new InvokeTaskSequenceTask(() =>
{
return this.WriteBlockBlobImpl(buffer, offset, count);
});
}
else
{
task = this.WriteNonBlockedBlobImpl(buffer, offset, count);
}
return task.ToAsyncResult(callback, state);
}
/// <summary>
/// Implementation for the DeleteIfExists method.
/// </summary>
/// <param name="options">An object that specifies any additional options for the request.</param>
/// <param name="setResult">The set result.</param>
/// <returns>A <see cref="TaskSequence"/> that deletes the blob if it exists.</returns>
private TaskSequence DeleteBlobIfExistsImpl(BlobRequestOptions options, Action<bool> setResult)
{
CommonUtils.AssertNotNull("options", options);
var task = new InvokeTaskSequenceTask(() => this.DeleteBlobImpl(options));
yield return task;
try
{
// Materialize exceptions
var scratch = task.Result;
setResult(true);
}
catch (StorageClientException e)
{
if (e.StatusCode == HttpStatusCode.NotFound)
{
setResult(false);
}
else
{
throw;
}
}
}
/// <summary>
/// Implements the DownloadToStream method.
/// </summary>
/// <param name="target">The target stream.</param>
/// <param name="options">An object that specifies any additional options for the request.</param>
/// <returns>A <see cref="TaskSequence"/> that downloads the blob to the stream.</returns>
internal TaskSequence DownloadToStreamImpl(Stream target, BlobRequestOptions options)
{
CommonUtils.AssertNotNull("options", options);
var webResponseTask = new InvokeTaskSequenceTask<Stream>((result) => { return GetStreamImpl(options, result); });
yield return webResponseTask;
using (var responseStream = webResponseTask.Result)
{
var copyTask = new InvokeTaskSequenceTask(() => { return responseStream.WriteTo(target); });
yield return copyTask;
// Materialize any exceptions
var scratch = copyTask.Result;
}
}
/// <summary>
/// Uploads the block.
/// </summary>
/// <param name="source">The source stream.</param>
/// <param name="blockId">The block ID.</param>
/// <param name="contentMD5">The content MD5.</param>
/// <param name="options">An object that specifies any additional options for the request.</param>
/// <returns>A <see cref="TaskSequence"/> that uploads the block.</returns>
internal TaskSequence UploadBlock(Stream source, string blockId, string contentMD5, BlobRequestOptions options)
{
int id = Environment.TickCount;
TraceHelper.WriteLine("Starting upload for id {0}", id);
CommonUtils.AssertNotNull("options", options);
var request = ProtocolHelper.GetWebRequest(this.ServiceClient, options, (timeout) => BlobRequest.PutBlock(this.TransformedAddress, timeout, blockId, null));
options.AccessCondition.ApplyCondition(request);
var length = source.Length - source.Position;
CommonUtils.ApplyRequestOptimizations(request, length);
if (!string.IsNullOrEmpty(contentMD5))
{
request.Headers.Set(HttpRequestHeader.ContentMd5, contentMD5);
}
this.ServiceClient.Credentials.SignRequest(request);
var uploadTask = new InvokeTaskSequenceTask<WebResponse>((result) =>
{
return UploadData(request, source, result);
});
yield return uploadTask;
uploadTask.Result.Close();
TraceHelper.WriteLine("Ending upload for id {0}", id);
}
/// <summary>
/// Generates a task sequence for setting the properties of the blob service.
/// </summary>
/// <param name="properties">The blob service properties to set.</param>
/// <returns>A task sequence that sets the properties of the blob service.</returns>
private TaskSequence SetServicePropertiesImpl(ServiceProperties properties)
{
CommonUtils.AssertNotNull("properties", properties);
HttpWebRequest request = BlobRequest.SetServiceProperties(this.BaseUri, this.Timeout.RoundUpToSeconds());
using (MemoryStream memoryStream = new MemoryStream())
{
try
{
BlobRequest.WriteServiceProperties(properties, memoryStream);
}
catch (InvalidOperationException invalidOpException)
{
throw new ArgumentException(invalidOpException.Message, "properties");
}
memoryStream.Seek(0, SeekOrigin.Begin);
CommonUtils.ApplyRequestOptimizations(request, memoryStream.Length);
this.Credentials.SignRequest(request);
// Get the request stream
StorageTask<Stream> getStreamTask = request.GetRequestStreamAsyncEx();
yield return getStreamTask;
using (Stream requestStream = getStreamTask.Result)
{
// Upload the service properties.
StorageTask<NullTaskReturn> uploadTask = new InvokeTaskSequenceTask(() => { return (memoryStream as Stream).WriteTo(requestStream); });
yield return uploadTask;
// Materialize any exceptions.
NullTaskReturn scratch = uploadTask.Result;
Console.WriteLine(scratch);
}
}
// Get the web response.
StorageTask<WebResponse> responseTask = request.GetResponseAsyncWithTimeout(this, this.Timeout);
yield return responseTask;
// Materialize any exceptions.
using (HttpWebResponse response = responseTask.Result as HttpWebResponse)
{
}
}
/// <summary>
/// Uploads the full blob.
/// </summary>
/// <param name="source">The source stream.</param>
/// <param name="options">An object that specifies any additional options for the request.</param>
/// <returns>A <see cref="TaskSequence"/> that uploads the blob.</returns>
internal TaskSequence UploadFullBlobImpl(Stream source, BlobRequestOptions options)
{
CommonUtils.AssertNotNull("modifers", options);
this.Properties.BlobType = BlobType.BlockBlob;
var request = ProtocolHelper.GetWebRequest(
this.ServiceClient,
options,
(timeout) => BlobRequest.Put(this.TransformedAddress, timeout, this.Properties, this.Properties.BlobType, null, 0));
BlobRequest.AddMetadata(request, this.Metadata);
var length = source.Length - source.Position;
CommonUtils.ApplyRequestOptimizations(request, length);
this.ServiceClient.Credentials.SignRequest(request);
var uploadTask = new InvokeTaskSequenceTask<WebResponse>((result) =>
{
return UploadData(request, source, result);
});
yield return uploadTask;
using (var response = uploadTask.Result as HttpWebResponse)
{
// Retrieve the ETag/LastModified
this.ParseSizeAndLastModified(response);
this.Properties.Length = length;
}
}
/// <summary>
/// Generates a task sequence for getting the properties of the blob service.
/// </summary>
/// <param name="setResult">A delegate to receive the service properties.</param>
/// <returns>A task sequence that gets the properties of the blob service.</returns>
private TaskSequence GetServicePropertiesImpl(Action<ServiceProperties> setResult)
{
HttpWebRequest request = BlobRequest.GetServiceProperties(this.BaseUri, this.Timeout.RoundUpToSeconds());
CommonUtils.ApplyRequestOptimizations(request, -1);
this.Credentials.SignRequest(request);
// Get the web response.
StorageTask<WebResponse> responseTask = request.GetResponseAsyncWithTimeout(this, this.Timeout);
yield return responseTask;
using (HttpWebResponse response = responseTask.Result as HttpWebResponse)
using (Stream responseStream = response.GetResponseStream())
using (MemoryStream memoryStream = new MemoryStream())
{
// Download the service properties.
StorageTask<NullTaskReturn> downloadTask = new InvokeTaskSequenceTask(() => { return responseStream.WriteTo(memoryStream); });
yield return downloadTask;
// Materialize any exceptions.
NullTaskReturn scratch = downloadTask.Result;
Console.WriteLine(scratch);
// Get the result from the memory stream.
memoryStream.Seek(0, SeekOrigin.Begin);
setResult(BlobResponse.ReadServiceProperties(memoryStream));
}
}
/// <summary>
/// Uploads the full blob with a retry sequence. The stream must be seekable.
/// </summary>
/// <param name="source">The source stream, which must be seekable.</param>
/// <param name="options">An object that specifies any additional options for the request.</param>
/// <returns>A <see cref="TaskSequence"/> that uploads the blob.</returns>
private TaskSequence UploadFullBlobWithRetrySequenceImpl(Stream source, BlobRequestOptions options)
{
// Save the stream position
long streamPos = source.Position;
// Compute the MD5
Task<string> md5Task = new InvokeTaskSequenceTask<string>(source.ComputeMD5);
yield return md5Task;
// Store the MD5 and rewind the stream
this.Properties.ContentMD5 = md5Task.Result;
source.Seek(streamPos, SeekOrigin.Begin);
// Upload the blob
Task<NullTaskReturn> uploadTask = this.UploadFullBlobWithRetryImpl(source, options);
yield return uploadTask;
// Materialize any exceptions
NullTaskReturn scratch = uploadTask.Result;
}
/// <summary>
/// Reads the data from the service starting at the specified location. Verifies the block's signature (if required) and adds it to the buffered data.
/// </summary>
/// <param name="startPosition">The starting position of the read-ahead.</param>
/// <param name="length">The number of bytes to read ahead.</param>
/// <returns> An TaskSequence that represents the asynchronous read action. </returns>
private TaskSequence ReadAheadImpl(long startPosition, long length)
{
var webResponseTask = new InvokeTaskSequenceTask <Stream>((result) => { return(this.Blob.GetStreamImpl(options, startPosition, length, result)); });
yield return(webResponseTask);
using (var stream = webResponseTask.Result)
{
this.LockToEtag();
if (this.IntegrityControlVerificationEnabled && this.Blob.Properties.BlobType == BlobType.BlockBlob)
{
long blockStartPosition = 0;
foreach (var block in this.blockList)
{
var blockSize = block.Size;
// Find the starting block
if (blockStartPosition < startPosition)
{
blockStartPosition += blockSize;
continue;
}
// Start creating blocks
var memoryStream = new SmallBlockMemoryStream(Constants.DefaultBufferSize);
var md5Check = MD5.Create();
int totalCopied = 0, numRead = 0;
do
{
byte[] buffer = new byte[Constants.DefaultBufferSize];
var numToRead = (int)Math.Min(buffer.Length, blockSize - totalCopied);
var readTask = stream.ReadAsync(buffer, 0, numToRead);
yield return(readTask);
numRead = readTask.Result;
if (numRead != 0)
{
// Verify the content
StreamUtilities.ComputeHash(buffer, 0, numRead, md5Check);
var writeTask = memoryStream.WriteAsync(buffer, 0, numRead);
yield return(writeTask);
var scratch = writeTask.Result; // Materialize any exceptions
totalCopied += numRead;
}
}while (numRead != 0 && totalCopied < blockSize);
// If we read something, act on it
if (totalCopied != 0)
{
// Verify the hash
string blockNameMD5Value = Utilities.ExtractMD5ValueFromBlockID(block.Name);
if (blockNameMD5Value != StreamUtilities.GetHashValue(md5Check))
{
throw new InvalidDataException("Blob data corrupted (integrity check failed)");
}
memoryStream.Position = 0; // Rewind the stream to allow for reading
this.downloadedBlocksList.Add(new DownloadedBlock(startPosition, memoryStream));
startPosition += blockSize;
blockStartPosition += blockSize;
}
else
{
break;
}
}
}
else
{
var memoryStream = new SmallBlockMemoryStream(Constants.DefaultBufferSize);
var copyTask = new InvokeTaskSequenceTask(() => { return(stream.WriteTo(memoryStream)); });
yield return(copyTask);
var scratch = copyTask.Result; // Materialize any errors
memoryStream.Position = 0; // Rewind the stream to allow for reading
this.downloadedBlocksList.Add(new DownloadedBlock(startPosition, memoryStream));
}
}
}
/// <summary>
/// Commits the blob by uploading any remaining data and the blocklist asynchronously.
/// </summary>
/// <returns>A sequence of events required for commit.</returns>
private TaskSequence CommitImpl()
{
this.CheckWriteState();
if (this.blockBuffer != null && this.blockBuffer.Length != 0)
{
var task = new InvokeTaskSequenceTask(this.FlushInternal);
yield return task;
var result = task.Result; // Materialize the errors
Console.WriteLine(result);
}
// If all blocks are uploaded, commit
if (this.UseBlocks)
{
this.SetBlobMD5();
var task = TaskImplHelper.GetRetryableAsyncTask(
() =>
{
// At the convenience layer we always upload uncommitted blocks
List<PutBlockListItem> putBlockList = new List<PutBlockListItem>();
foreach (var id in this.blockList)
{
putBlockList.Add(new PutBlockListItem(id, BlockSearchMode.Uncommitted));
}
return this.Blob.ToBlockBlob.UploadBlockList(putBlockList, this.currentModifier);
},
this.currentModifier.RetryPolicy);
yield return task;
var result = task.Result;
Console.WriteLine(result);
}
// Now we can set the full size.
this.Blob.Properties.Length = this.Length;
// Clear the internal state
this.Abort();
}
/// <summary>
/// Implementation for the SetPermissions method.
/// </summary>
/// <param name="acl">The permissions to set.</param>
/// <param name="options">An object that specifies any additional options for the request.</param>
/// <returns>A <see cref="TaskSequence"/> that sets the permissions.</returns>
private TaskSequence SetPermissionsImpl(BlobContainerPermissions acl, BlobRequestOptions options)
{
CommonUtils.AssertNotNull("options", options);
var webRequest = ProtocolHelper.GetWebRequest(
this.ServiceClient,
options,
(timeout) => ContainerRequest.SetAcl(this.TransformedAddress, timeout, acl.PublicAccess));
using (var memoryStream = new SmallBlockMemoryStream(Constants.DefaultBufferSize))
{
ContainerRequest.WriteSharedAccessIdentifiers(acl.SharedAccessPolicies, memoryStream);
memoryStream.Seek(0, System.IO.SeekOrigin.Begin);
CommonUtils.ApplyRequestOptimizations(webRequest, memoryStream.Length);
this.ServiceClient.Credentials.SignRequest(webRequest);
var requestStreamTask = webRequest.GetRequestStreamAsync();
yield return requestStreamTask;
using (var requestStream = requestStreamTask.Result)
{
// Copy the data
var copyTask = new InvokeTaskSequenceTask(() => { return memoryStream.WriteTo(requestStream); });
yield return copyTask;
// Materialize any exceptions
var scratch = copyTask.Result;
}
}
var task = webRequest.GetResponseAsyncWithTimeout(this.ServiceClient, options.Timeout);
yield return task;
using (var webResponse = task.Result as HttpWebResponse)
{
this.ParseETagAndLastModified(webResponse);
}
}
/// <summary>
/// Implements the block writing task.
/// </summary>
/// <param name="buffer">The buffer to write data from.</param>
/// <param name="offset">The byte offset in buffer from which to begin writing.</param>
/// <param name="count">The number of bytes to write.</param>
/// <returns>The sequence representing the uploading of the blocks.</returns>
private TaskSequence WriteBlockBlobImpl(byte[] buffer, int offset, int count)
{
if (this.blockList.Count + (count / this.blockSize) > Constants.MaxBlockNumber)
{
throw new ArgumentOutOfRangeException("count", String.Format(CultureInfo.CurrentCulture, SR.TooManyBlocksError, Constants.MaxBlockNumber));
}
while (count != 0)
{
// Create a buffer if we don't have one
if (this.blockBuffer == null)
{
this.CreateNewBlock();
}
// Copy enough data to fill the buffer.
int numCopied = (int)Math.Min(count, this.blockSize - this.blockBuffer.Length);
this.blockBuffer.Write(buffer, offset, numCopied);
StreamUtilities.ComputeHash(buffer, offset, numCopied, this.blockHash);
StreamUtilities.ComputeHash(buffer, offset, numCopied, this.blobHash);
// Advance the location
this.position += numCopied;
offset += numCopied;
count -= numCopied;
// If buffer is full, flush
if (this.blockBuffer.Length == this.blockSize)
{
var newTask = new InvokeTaskSequenceTask(this.FlushInternal);
yield return newTask;
// Make sure task completed successfully by materializing the exception
var result = newTask.Result;
Console.WriteLine(result);
}
}
}
/// <summary>
/// Implementation for the CreateIfNotExist method.
/// </summary>
/// <param name="options">An object that specifies any additional options for the request.</param>
/// <param name="setResult">The set result.</param>
/// <returns>A <see cref="TaskSequence"/> that creates the container if it does not exist.</returns>
private TaskSequence CreateContainerIfNotExistImpl(BlobRequestOptions options, Action<bool> setResult)
{
CommonUtils.AssertNotNull("options", options);
var task = new InvokeTaskSequenceTask(() => this.CreateContainerImpl(options));
yield return task;
try
{
// Materialize exceptions
var scratch = task.Result;
setResult(true);
}
catch (StorageClientException e)
{
if (e.StatusCode == HttpStatusCode.Conflict && e.ExtendedErrorInformation.ErrorCode == StorageErrorCodeStrings.ContainerAlreadyExists)
{
setResult(false);
}
else
{
throw;
}
}
}
/// <summary>
/// Implementation of the *RequestWithRetry methods.
/// </summary>
/// <typeparam name="T">The result type of the task.</typeparam>
/// <param name="retryOracle">The retry oracle.</param>
/// <param name="impl">The task implementation.</param>
/// <param name="setResult">The result report delegate.</param>
/// <returns>A <see cref="TaskSequence"/> that performs the request with retries.</returns>
internal static TaskSequence RequestWithRetryImpl <T>(ShouldRetry retryOracle, Func <Action <T>, TaskSequence> impl, Action <T> setResult)
{
int retryCount = 0;
bool succeeded = false;
bool shouldRetry = false;
do
{
var task = new InvokeTaskSequenceTask <T>(impl);
yield return(task);
TimeSpan delay = TimeSpan.FromMilliseconds(-1);
try
{
var result = task.Result;
succeeded = true;
setResult(result);
}
catch (TimeoutException e)
{
shouldRetry = retryOracle != null?retryOracle(retryCount ++, e, out delay) : false;
// We should just throw out the exception if we are not retrying
if (!shouldRetry)
{
throw;
}
}
catch (StorageServerException e)
{
if (e.StatusCode == HttpStatusCode.NotImplemented || e.StatusCode == HttpStatusCode.HttpVersionNotSupported)
{
throw;
}
shouldRetry = retryOracle != null?retryOracle(retryCount ++, e, out delay) : false;
// We should just throw out the exception if we are not retrying
if (!shouldRetry)
{
throw;
}
}
catch (InvalidOperationException e)
{
//DataServiceClientException dsce = CommonUtils.FindInnerDataServiceClientException(e);
/*
* // rethrow 400 class errors immediately as they can't be retried
* // 501 (Not Implemented) and 505 (HTTP Version Not Supported) shouldn't be retried either.
* if (dsce != null &&
* ((dsce.StatusCode >= 400 && dsce.StatusCode < 500)
|| dsce.StatusCode == (int)HttpStatusCode.NotImplemented
|| dsce.StatusCode == (int)HttpStatusCode.HttpVersionNotSupported))
||{
||throw;
||}*/
// If it is BlobTypeMismatchExceptionMessage, we should throw without retry
if (e.Message == SR.BlobTypeMismatchExceptionMessage)
{
throw;
}
shouldRetry = retryOracle != null?retryOracle(retryCount ++, e, out delay) : false;
// We should just throw out the exception if we are not retrying
if (!shouldRetry)
{
throw;
}
}
if (!succeeded && shouldRetry && delay > TimeSpan.Zero)
{
using (DelayTask delayTask = new DelayTask(delay))
{
yield return(delayTask);
// Materialize exceptions
var scratch = delayTask.Result;
Console.WriteLine(scratch);
}
}
}while (!succeeded && shouldRetry);
}
/// <summary>
/// Uploads the block list.
/// </summary>
/// <param name="blocks">The blocks to upload.</param>
/// <param name="options">An object that specifies any additional options for the request.</param>
/// <returns>A <see cref="TaskSequence"/> that uploads the block list.</returns>
internal TaskSequence UploadBlockList(List<PutBlockListItem> blocks, BlobRequestOptions options)
{
if (options == null)
{
throw new ArgumentNullException("modifers");
}
var request = ProtocolHelper.GetWebRequest(this.ServiceClient, options, (timeout) => BlobRequest.PutBlockList(this.TransformedAddress, timeout, this.Properties, null));
options.AccessCondition.ApplyCondition(request);
BlobRequest.AddMetadata(request, this.Metadata);
using (var memoryStream = new SmallBlockMemoryStream(Constants.DefaultBufferSize))
{
BlobRequest.WriteBlockListBody(blocks, memoryStream);
CommonUtils.ApplyRequestOptimizations(request, memoryStream.Length);
memoryStream.Seek(0, SeekOrigin.Begin);
// Compute the MD5
var md5 = System.Security.Cryptography.MD5.Create();
request.Headers[HttpRequestHeader.ContentMd5] = Convert.ToBase64String(md5.ComputeHash(memoryStream));
this.ServiceClient.Credentials.SignRequest(request);
memoryStream.Seek(0, SeekOrigin.Begin);
// Retrieve the stream
var requestStreamTask = request.GetRequestStreamAsync();
yield return requestStreamTask;
using (Stream requestStream = requestStreamTask.Result)
{
// Copy the data
var copyTask = new InvokeTaskSequenceTask(() => { return memoryStream.WriteTo(requestStream); });
yield return copyTask;
// Materialize any exceptions
var scratch = copyTask.Result;
Console.WriteLine(scratch);
}
}
// Get the response
var responseTask = request.GetResponseAsyncWithTimeout(this.ServiceClient, options.Timeout);
yield return responseTask;
using (var response = responseTask.Result as HttpWebResponse)
{
ParseSizeAndLastModified(response);
this.Properties.Length = 0;
}
}
/// <summary>
/// Perform a parallel upload of blocks for a blob from a given stream.
/// </summary>
/// <param name="uploadFunc">The upload func.</param>
/// <returns>A <see cref="TaskSequence"/> that uploads the blob in parallel.</returns>
/// <remarks>
/// The operation is done as a series of alternate producer and consumer tasks. The producer tasks dispense out
/// chunks of source stream as fixed size blocks. This is done in serial order on a thread using InvokeTaskSequence's
/// serial execution. The consumer tasks upload each block in parallel on multiple thread. The producer thread waits
/// for at least one consumer task to finish before adding more producer tasks. The producer thread quits when no
/// more data can be read from the stream and no other pending consumer tasks.
/// </remarks>
internal TaskSequence ParallelExecute(
Func<SmallBlockMemoryStream, string, string, BlobRequestOptions, Task<NullTaskReturn>> uploadFunc)
{
bool moreToUpload = true;
List<IAsyncResult> asyncResults = new List<IAsyncResult>();
Random rand = new Random();
long blockIdSequenceNumber = (long)rand.Next() << 32;
blockIdSequenceNumber += rand.Next();
do
{
int currentPendingTasks = asyncResults.Count;
// Step 1
// Create producer tasks in a serial order as stream can only be read sequentially
for (int i = currentPendingTasks; i < this.parellelism && moreToUpload; i++)
{
string blockId = null;
string blockHash = null;
SmallBlockMemoryStream blockAsStream = null;
blockIdSequenceNumber++;
InvokeTaskSequenceTask producerTask = new InvokeTaskSequenceTask(() =>
this.DispenseBlockStream(
blockIdSequenceNumber,
(stream, id, hashVal) =>
{
blockAsStream = stream;
blockId = id;
blockHash = hashVal;
}));
yield return producerTask;
this.producerTasksCreated++;
var scatch = producerTask.Result;
if (blockAsStream == null)
{
TraceHelper.WriteLine("No more upload tasks");
moreToUpload = false;
}
else
{
// Step 2
// Fire off consumer tasks that may finish on other threads;
var task = uploadFunc(blockAsStream, blockId, blockHash, this.options);
IAsyncResult asyncresult = task.ToAsyncResult(null, null);
this.consumerTasksCreated++;
asyncResults.Add(asyncresult);
}
}
// Step 3
// Wait for 1 or more consumer tasks to finish inorder to bound set of parallel tasks
if (asyncResults.Count > 0)
{
int waitTimeout = GetWaitTimeout(this.options);
TraceHelper.WriteLine("Starting wait");
int waitResult = WaitHandle.WaitAny(asyncResults.Select(result => result.AsyncWaitHandle).ToArray(), waitTimeout);
TraceHelper.WriteLine("Ending wait");
if (waitResult == WaitHandle.WaitTimeout)
{
throw TimeoutHelper.ThrowTimeoutError(this.options.Timeout.Value);
}
CompleteAsyncresult(asyncResults, waitResult);
// Optimize away any other completed tasks
for (int index = 0; index < asyncResults.Count; index++)
{
IAsyncResult result = asyncResults[index];
if (result.IsCompleted)
{
CompleteAsyncresult(asyncResults, index);
index--;
}
}
}
}
while (moreToUpload || asyncResults.Count != 0);
TraceHelper.WriteLine(
"Total producer tasks created {0}, consumer tasks created {1} ",
this.producerTasksCreated,
this.consumerTasksCreated);
var commitTask = TaskImplHelper.GetRetryableAsyncTask(this.CommitBlob, this.options.RetryPolicy);
yield return commitTask;
var commitTaskResult = commitTask.Result;
}
/// <summary>Generates a task sequence for getting the properties of the queue service.</summary>
/// <param name="setResult">A delegate to receive the service properties. </param>
/// <returns>A task sequence that gets the properties of the queue service. </returns>
private TaskSequence GetServicePropertiesImpl(Action<ServiceProperties> setResult)
{
var request = QueueRequest.GetServiceProperties(this.BaseUri, this.Timeout.RoundUpToSeconds());
CommonUtils.ApplyRequestOptimizations(request, -1);
this.Credentials.SignRequest(request);
// Get the web response.
var responseTask = request.GetResponseAsyncWithTimeout(this, this.Timeout);
yield return responseTask;
using (var response = responseTask.Result as HttpWebResponse)
{
Debug.Assert(response != null, "response != null");
using (var responseStream = response.GetResponseStream())
using (var memoryStream = new MemoryStream())
{
// Download the service properties.
Task<NullTaskReturn> downloadTask =
new InvokeTaskSequenceTask(() => responseStream.WriteTo(memoryStream));
yield return downloadTask;
// Materialize any exceptions.
var scratch = downloadTask.Result;
// Get the result from the memory stream.
memoryStream.Seek(0, SeekOrigin.Begin);
setResult(QueueResponse.ReadServiceProperties(memoryStream));
}
}
}
