/// <summary>
/// Upload a single block. This can happen on parallel threads.
/// </summary>
/// <param name="blockIdSequenceNumber">The block sequence prefix value.</param>
/// <param name="setResult">The set result.</param>
/// <returns>A <see cref="TaskSequence"/> that dispenses a block stream.</returns>
private TaskSequence DispenseBlockStream(long blockIdSequenceNumber, Action <SmallBlockMemoryStream, string, string> setResult)
{
int currentCallIndex = this.dispenserCallCount++;
TraceHelper.WriteLine("Staring dispensBlockStream for id {0}", currentCallIndex);
SmallBlockMemoryStream memoryStream = new SmallBlockMemoryStream(Constants.DefaultBufferSize);
var md5Check = MD5.Create();
int totalCopied = 0, numRead = 0;
do
{
byte[] buffer = new byte[Constants.DefaultBufferSize];
var numToRead = (int)Math.Min(buffer.Length, this.blockSize - totalCopied);
var readTask = this.sourceStream.ReadAsync(buffer, 0, numToRead);
yield return(readTask);
numRead = readTask.Result;
if (numRead != 0)
{
// Verify the content
StreamUtilities.ComputeHash(buffer, 0, numRead, md5Check);
StreamUtilities.ComputeHash(buffer, 0, numRead, this.blobHash);
var writeTask = memoryStream.WriteAsync(buffer, 0, numRead);
yield return(writeTask);
// Materialize any exceptions
var scratch = writeTask.Result;
Console.WriteLine(scratch);
totalCopied += numRead;
}
}while (numRead != 0 && totalCopied < this.blockSize);
// No locking necessary as only once active Dispense Task
this.dispensizedStreamSize += totalCopied;
if (totalCopied != 0)
{
string hashVal = StreamUtilities.GetHashValue(md5Check);
string blockId = Utilities.GenerateBlockIDWithHash(hashVal, blockIdSequenceNumber);
this.blockList.Add(blockId);
memoryStream.Position = 0;
setResult(memoryStream, blockId, hashVal);
}
else
{
memoryStream.Close();
setResult(null, null, null);
}
TraceHelper.WriteLine("Ending dispensBlockStream for id {0}", currentCallIndex);
}
/// <summary>
/// Uploads the block list.
/// </summary>
/// <param name="blocks">The blocks to upload.</param>
/// <param name="options">An object that specifies any additional options for the request.</param>
/// <returns>A <see cref="TaskSequence"/> that uploads the block list.</returns>
internal TaskSequence UploadBlockList(List <PutBlockListItem> blocks, BlobRequestOptions options)
{
if (options == null)
{
throw new ArgumentNullException("modifers");
}
var request = ProtocolHelper.GetWebRequest(this.ServiceClient, options, (timeout) => BlobRequest.PutBlockList(this.TransformedAddress, timeout, this.Properties, null));
options.AccessCondition.ApplyCondition(request);
BlobRequest.AddMetadata(request, this.Metadata);
using (var memoryStream = new SmallBlockMemoryStream(Constants.DefaultBufferSize))
{
BlobRequest.WriteBlockListBody(blocks, memoryStream);
CommonUtils.ApplyRequestOptimizations(request, memoryStream.Length);
memoryStream.Seek(0, SeekOrigin.Begin);
// Compute the MD5
var md5 = System.Security.Cryptography.MD5.Create();
request.Headers[HttpRequestHeader.ContentMd5] = Convert.ToBase64String(md5.ComputeHash(memoryStream));
this.ServiceClient.Credentials.SignRequest(request);
memoryStream.Seek(0, SeekOrigin.Begin);
// Retrieve the stream
var requestStreamTask = request.GetRequestStreamAsync();
yield return(requestStreamTask);
using (Stream requestStream = requestStreamTask.Result)
{
// Copy the data
var copyTask = new InvokeTaskSequenceTask(() => { return(memoryStream.WriteTo(requestStream)); });
yield return(copyTask);
// Materialize any exceptions
var scratch = copyTask.Result;
Console.WriteLine(scratch);
}
}
// Get the response
var responseTask = request.GetResponseAsyncWithTimeout(this.ServiceClient, options.Timeout);
yield return(responseTask);
using (var response = responseTask.Result as HttpWebResponse)
{
ParseSizeAndLastModified(response);
this.Properties.Length = 0;
}
}
/// <summary>
/// Resets the block and the block hash.
/// </summary>
private void ResetBlock()
{
if (this.blockBuffer != null)
{
this.blockBuffer.Dispose();
}
this.blockBuffer = null;
this.blockHash = null;
}
/// <summary>
/// Reads the data from the service starting at the specified location. Verifies the block's signature (if required) and adds it to the buffered data.
/// </summary>
/// <param name="startPosition">The starting position of the read-ahead.</param>
/// <param name="length">The number of bytes to read ahead.</param>
/// <returns> An TaskSequence that represents the asynchronous read action. </returns>
private TaskSequence ReadAheadImpl(long startPosition, long length)
{
var webResponseTask = new InvokeTaskSequenceTask<Stream>((result) => { return this.Blob.GetStreamImpl(options, startPosition, length, result); });
yield return webResponseTask;
using (var stream = webResponseTask.Result)
{
this.LockToEtag();
if (this.IntegrityControlVerificationEnabled && this.Blob.Properties.BlobType == BlobType.BlockBlob)
{
long blockStartPosition = 0;
foreach (var block in this.blockList)
{
var blockSize = block.Size;
// Find the starting block
if (blockStartPosition < startPosition)
{
blockStartPosition += blockSize;
continue;
}
// Start creating blocks
var memoryStream = new SmallBlockMemoryStream(Constants.DefaultBufferSize);
var md5Check = MD5.Create();
int totalCopied = 0, numRead = 0;
do
{
byte[] buffer = new byte[Constants.DefaultBufferSize];
var numToRead = (int)Math.Min(buffer.Length, blockSize - totalCopied);
var readTask = stream.ReadAsyncEx(buffer, 0, numToRead);
yield return readTask;
numRead = readTask.Result;
if (numRead != 0)
{
// Verify the content
StreamUtilities.ComputeHash(buffer, 0, numRead, md5Check);
var writeTask = memoryStream.WriteAsyncEx(buffer, 0, numRead);
yield return writeTask;
//var scratch = writeTask.Result; // Materialize any exceptions
totalCopied += numRead;
}
}
while (numRead != 0 && totalCopied < blockSize);
// If we read something, act on it
if (totalCopied != 0)
{
// Verify the hash
string blockNameMD5Value = Utilities.ExtractMD5ValueFromBlockID(block.Name);
if (blockNameMD5Value != StreamUtilities.GetHashValue(md5Check))
{
throw new InvalidDataException("Blob data corrupted (integrity check failed)");
}
memoryStream.Position = 0; // Rewind the stream to allow for reading
this.downloadedBlocksList.Add(new DownloadedBlock(startPosition, memoryStream));
startPosition += blockSize;
blockStartPosition += blockSize;
}
else
{
break;
}
}
}
else
{
var memoryStream = new SmallBlockMemoryStream(Constants.DefaultBufferSize);
var copyTask = new InvokeTaskSequenceTask(() => { return stream.WriteTo(memoryStream); });
yield return copyTask;
//var scratch = copyTask.Result; // Materialize any errors
memoryStream.Position = 0; // Rewind the stream to allow for reading
this.downloadedBlocksList.Add(new DownloadedBlock(startPosition, memoryStream));
}
}
}
/// <summary>
/// Resets the block and the block hash.
/// </summary>
private void ResetBlock()
{
if (this.blockBuffer != null)
{
this.blockBuffer.Dispose();
}
this.blockBuffer = null;
this.blockHash = null;
}
/// <summary>
/// Creates the new block and the block hash.
/// </summary>
private void CreateNewBlock()
{
this.blockBuffer = new SmallBlockMemoryStream(Constants.DefaultBufferSize);
this.blockHash = MD5.Create();
}
/// <summary>
/// Perform a parallel upload of blocks for a blob from a given stream.
/// </summary>
/// <param name="uploadFunc">The upload func.</param>
/// <returns>A <see cref="TaskSequence"/> that uploads the blob in parallel.</returns>
/// <remarks>
/// The operation is done as a series of alternate producer and consumer tasks. The producer tasks dispense out
/// chunks of source stream as fixed size blocks. This is done in serial order on a thread using InvokeTaskSequence's
/// serial execution. The consumer tasks upload each block in parallel on multiple thread. The producer thread waits
/// for at least one consumer task to finish before adding more producer tasks. The producer thread quits when no
/// more data can be read from the stream and no other pending consumer tasks.
/// </remarks>
internal TaskSequence ParallelExecute(
Func <SmallBlockMemoryStream, string, string, BlobRequestOptions, Task <NullTaskReturn> > uploadFunc)
{
bool moreToUpload = true;
List <IAsyncResult> asyncResults = new List <IAsyncResult>();
Random rand = new Random();
long blockIdSequenceNumber = (long)rand.Next() << 32;
blockIdSequenceNumber += rand.Next();
do
{
int currentPendingTasks = asyncResults.Count;
// Step 1
// Create producer tasks in a serial order as stream can only be read sequentially
for (int i = currentPendingTasks; i < this.parellelism && moreToUpload; i++)
{
string blockId = null;
string blockHash = null;
SmallBlockMemoryStream blockAsStream = null;
blockIdSequenceNumber++;
InvokeTaskSequenceTask producerTask = new InvokeTaskSequenceTask(() =>
this.DispenseBlockStream(
blockIdSequenceNumber,
(stream, id, hashVal) =>
{
blockAsStream = stream;
blockId = id;
blockHash = hashVal;
}));
yield return(producerTask);
this.producerTasksCreated++;
var scatch = producerTask.Result;
Console.WriteLine(scatch);
if (blockAsStream == null)
{
TraceHelper.WriteLine("No more upload tasks");
moreToUpload = false;
}
else
{
// Step 2
// Fire off consumer tasks that may finish on other threads;
var task = uploadFunc(blockAsStream, blockId, blockHash, this.options);
IAsyncResult asyncresult = task.ToAsyncResult(null, null);
this.consumerTasksCreated++;
asyncResults.Add(asyncresult);
}
}
// Step 3
// Wait for 1 or more consumer tasks to finish inorder to bound set of parallel tasks
if (asyncResults.Count > 0)
{
int waitTimeout = GetWaitTimeout(this.options);
TraceHelper.WriteLine("Starting wait");
int waitResult = WaitHandle.WaitAny(asyncResults.Select(result => result.AsyncWaitHandle).ToArray(), waitTimeout);
TraceHelper.WriteLine("Ending wait");
if (waitResult == WaitHandle.WaitTimeout)
{
throw TimeoutHelper.ThrowTimeoutError(this.options.Timeout.Value);
}
CompleteAsyncresult(asyncResults, waitResult);
// Optimize away any other completed tasks
for (int index = 0; index < asyncResults.Count; index++)
{
IAsyncResult result = asyncResults[index];
if (result.IsCompleted)
{
CompleteAsyncresult(asyncResults, index);
index--;
}
}
}
}while (moreToUpload || asyncResults.Count != 0);
TraceHelper.WriteLine(
"Total producer tasks created {0}, consumer tasks created {1} ",
this.producerTasksCreated,
this.consumerTasksCreated);
var commitTask = TaskImplHelper.GetRetryableAsyncTask(this.CommitBlob, this.options.RetryPolicy);
yield return(commitTask);
var commitTaskResult = commitTask.Result;
Console.WriteLine(commitTaskResult);
}
/// <summary>
/// Upload a single block. This can happen on parallel threads.
/// </summary>
/// <param name="blockIdSequenceNumber">The block sequence prefix value.</param>
/// <param name="setResult">The set result.</param>
/// <returns>A <see cref="TaskSequence"/> that dispenses a block stream.</returns>
private TaskSequence DispenseBlockStream(long blockIdSequenceNumber, Action<SmallBlockMemoryStream, string, string> setResult)
{
int currentCallIndex = this.dispenserCallCount++;
TraceHelper.WriteLine("Staring dispensBlockStream for id {0}", currentCallIndex);
SmallBlockMemoryStream memoryStream = new SmallBlockMemoryStream(Constants.DefaultBufferSize);
var md5Check = MD5.Create();
int totalCopied = 0, numRead = 0;
do
{
byte[] buffer = new byte[Constants.DefaultBufferSize];
var numToRead = (int)Math.Min(buffer.Length, this.blockSize - totalCopied);
var readTask = this.sourceStream.ReadAsync(buffer, 0, numToRead);
yield return readTask;
numRead = readTask.Result;
if (numRead != 0)
{
// Verify the content
StreamUtilities.ComputeHash(buffer, 0, numRead, md5Check);
StreamUtilities.ComputeHash(buffer, 0, numRead, this.blobHash);
var writeTask = memoryStream.WriteAsync(buffer, 0, numRead);
yield return writeTask;
// Materialize any exceptions
var scratch = writeTask.Result;
totalCopied += numRead;
}
}
while (numRead != 0 && totalCopied < this.blockSize);
// No locking necessary as only once active Dispense Task
this.dispensizedStreamSize += totalCopied;
if (totalCopied != 0)
{
string hashVal = StreamUtilities.GetHashValue(md5Check);
string blockId = Utilities.GenerateBlockIDWithHash(hashVal, blockIdSequenceNumber);
this.blockList.Add(blockId);
memoryStream.Position = 0;
setResult(memoryStream, blockId, hashVal);
}
else
{
memoryStream.Close();
setResult(null, null, null);
}
TraceHelper.WriteLine("Ending dispensBlockStream for id {0}", currentCallIndex);
}
/// <summary>
/// Uploads the block list.
/// </summary>
/// <param name="blocks">The blocks to upload.</param>
/// <param name="options">An object that specifies any additional options for the request.</param>
/// <returns>A <see cref="TaskSequence"/> that uploads the block list.</returns>
internal TaskSequence UploadBlockList(List<PutBlockListItem> blocks, BlobRequestOptions options)
{
if (options == null)
{
throw new ArgumentNullException("modifers");
}
var request = ProtocolHelper.GetWebRequest(this.ServiceClient, options, (timeout) => BlobRequest.PutBlockList(this.TransformedAddress, timeout, this.Properties, null));
options.AccessCondition.ApplyCondition(request);
BlobRequest.AddMetadata(request, this.Metadata);
using (var memoryStream = new SmallBlockMemoryStream(Constants.DefaultBufferSize))
{
BlobRequest.WriteBlockListBody(blocks, memoryStream);
CommonUtils.ApplyRequestOptimizations(request, memoryStream.Length);
memoryStream.Seek(0, SeekOrigin.Begin);
// Compute the MD5
var md5 = System.Security.Cryptography.MD5.Create();
request.Headers[HttpRequestHeader.ContentMd5] = Convert.ToBase64String(md5.ComputeHash(memoryStream));
this.ServiceClient.Credentials.SignRequest(request);
memoryStream.Seek(0, SeekOrigin.Begin);
// Retrieve the stream
var requestStreamTask = request.GetRequestStreamAsync();
yield return requestStreamTask;
using (Stream requestStream = requestStreamTask.Result)
{
// Copy the data
var copyTask = new InvokeTaskSequenceTask(() => { return memoryStream.WriteTo(requestStream); });
yield return copyTask;
// Materialize any exceptions
var scratch = copyTask.Result;
Console.WriteLine(scratch);
}
}
// Get the response
var responseTask = request.GetResponseAsyncWithTimeout(this.ServiceClient, options.Timeout);
yield return responseTask;
using (var response = responseTask.Result as HttpWebResponse)
{
ParseSizeAndLastModified(response);
this.Properties.Length = 0;
}
}
/// <summary>
/// Implementation for the SetPermissions method.
/// </summary>
/// <param name="acl">The permissions to set.</param>
/// <param name="options">An object that specifies any additional options for the request.</param>
/// <returns>A <see cref="TaskSequence"/> that sets the permissions.</returns>
private TaskSequence SetPermissionsImpl(BlobContainerPermissions acl, BlobRequestOptions options)
{
CommonUtils.AssertNotNull("options", options);
var webRequest = ProtocolHelper.GetWebRequest(
this.ServiceClient,
options,
(timeout) => ContainerRequest.SetAcl(this.TransformedAddress, timeout, acl.PublicAccess));
using (var memoryStream = new SmallBlockMemoryStream(Constants.DefaultBufferSize))
{
ContainerRequest.WriteSharedAccessIdentifiers(acl.SharedAccessPolicies, memoryStream);
memoryStream.Seek(0, System.IO.SeekOrigin.Begin);
CommonUtils.ApplyRequestOptimizations(webRequest, memoryStream.Length);
this.ServiceClient.Credentials.SignRequest(webRequest);
var requestStreamTask = webRequest.GetRequestStreamAsync();
yield return requestStreamTask;
using (var requestStream = requestStreamTask.Result)
{
// Copy the data
var copyTask = new InvokeTaskSequenceTask(() => { return memoryStream.WriteTo(requestStream); });
yield return copyTask;
// Materialize any exceptions
var scratch = copyTask.Result;
}
}
var task = webRequest.GetResponseAsyncWithTimeout(this.ServiceClient, options.Timeout);
yield return task;
using (var webResponse = task.Result as HttpWebResponse)
{
this.ParseETagAndLastModified(webResponse);
}
}
/// <summary>
/// Reads the data from the service starting at the specified location. Verifies the block's signature (if required) and adds it to the buffered data.
/// </summary>
/// <param name="startPosition">The starting position of the read-ahead.</param>
/// <param name="length">The number of bytes to read ahead.</param>
/// <returns> An TaskSequence that represents the asynchronous read action. </returns>
private TaskSequence ReadAheadImpl(long startPosition, long length)
{
var webResponseTask = new InvokeTaskSequenceTask <Stream>((result) => { return(this.Blob.GetStreamImpl(options, startPosition, length, result)); });
yield return(webResponseTask);
using (var stream = webResponseTask.Result)
{
this.LockToEtag();
if (this.IntegrityControlVerificationEnabled && this.Blob.Properties.BlobType == BlobType.BlockBlob)
{
long blockStartPosition = 0;
foreach (var block in this.blockList)
{
var blockSize = block.Size;
// Find the starting block
if (blockStartPosition < startPosition)
{
blockStartPosition += blockSize;
continue;
}
// Start creating blocks
var memoryStream = new SmallBlockMemoryStream(Constants.DefaultBufferSize);
var md5Check = MD5.Create();
int totalCopied = 0, numRead = 0;
do
{
byte[] buffer = new byte[Constants.DefaultBufferSize];
var numToRead = (int)Math.Min(buffer.Length, blockSize - totalCopied);
var readTask = stream.ReadAsync(buffer, 0, numToRead);
yield return(readTask);
numRead = readTask.Result;
if (numRead != 0)
{
// Verify the content
StreamUtilities.ComputeHash(buffer, 0, numRead, md5Check);
var writeTask = memoryStream.WriteAsync(buffer, 0, numRead);
yield return(writeTask);
var scratch = writeTask.Result; // Materialize any exceptions
totalCopied += numRead;
}
}while (numRead != 0 && totalCopied < blockSize);
// If we read something, act on it
if (totalCopied != 0)
{
// Verify the hash
string blockNameMD5Value = Utilities.ExtractMD5ValueFromBlockID(block.Name);
if (blockNameMD5Value != StreamUtilities.GetHashValue(md5Check))
{
throw new InvalidDataException("Blob data corrupted (integrity check failed)");
}
memoryStream.Position = 0; // Rewind the stream to allow for reading
this.downloadedBlocksList.Add(new DownloadedBlock(startPosition, memoryStream));
startPosition += blockSize;
blockStartPosition += blockSize;
}
else
{
break;
}
}
}
else
{
var memoryStream = new SmallBlockMemoryStream(Constants.DefaultBufferSize);
var copyTask = new InvokeTaskSequenceTask(() => { return(stream.WriteTo(memoryStream)); });
yield return(copyTask);
var scratch = copyTask.Result; // Materialize any errors
memoryStream.Position = 0; // Rewind the stream to allow for reading
this.downloadedBlocksList.Add(new DownloadedBlock(startPosition, memoryStream));
}
}
}
/// <summary>
/// Creates the new block and the block hash.
/// </summary>
private void CreateNewBlock()
{
this.blockBuffer = new SmallBlockMemoryStream(Constants.DefaultBufferSize);
this.blockHash = MD5.Create();
}
