/// <summary>
/// Verifies whether the table exist implementation.
/// </summary>
/// <param name="tableName">The table name.</param>
/// <param name="setResult">The set result.</param>
/// <returns>A sequence of tasks to do the operation.</returns>
private TaskSequence DoesTableExistImpl(string tableName, Action <bool> setResult)
{
CommonUtils.CheckStringParameter(tableName, false, "tableName", Protocol.Constants.TableServiceMaxStringPropertySizeInChars);
TableServiceUtilities.CheckTableName(tableName, "tableName");
var svc = this.GetDataServiceContext();
var tableExistsQuery = (from table in svc.CreateQuery <TableServiceTable>(Protocol.Constants.TableServiceTablesName)
where table.TableName == tableName
select table).AsTableServiceQuery();
ResultSegment <TableServiceTable> segment = null;
while (true)
{
Task <ResultSegment <TableServiceTable> > tableExistsSegmentTask;
if (segment == null)
{
tableExistsSegmentTask = TaskImplHelper.GetRetryableAsyncTask <ResultSegment <TableServiceTable> >(
(setResultInner) => tableExistsQuery.ExecuteSegmentedImpl(null, setResultInner), RetryPolicy);
}
else
{
tableExistsSegmentTask = TaskImplHelper.GetRetryableAsyncTask <ResultSegment <TableServiceTable> >(segment.GetNextImpl, RetryPolicy);
}
yield return(tableExistsSegmentTask);
if (GetResultOrDefault(tableExistsSegmentTask, out segment))
{
if (segment.Results.Any())
{
setResult(true);
break;
}
else
{
setResult(false);
break;
}
}
else
{
setResult(false);
break;
}
}
}
/// <summary>
/// Creates the table if not exist implementation.
/// </summary>
/// <param name="tableName">The table name.</param>
/// <param name="setResult">The set result.</param>
/// <returns>A sequence of tasks to do the operation.</returns>
private TaskSequence CreateTableIfNotExistImpl(string tableName, Action <bool> setResult)
{
CommonUtils.CheckStringParameter(tableName, false, "tableName", Protocol.Constants.TableServiceMaxStringPropertySizeInChars);
TableServiceUtilities.CheckTableName(tableName, "tableName");
var doesTableExistTask = new InvokeTaskSequenceTask <bool>((set) => this.DoesTableExistImpl(tableName, set));
yield return(doesTableExistTask);
if (doesTableExistTask.Result)
{
setResult(false);
}
else
{
var createTableTask = TaskImplHelper.GetRetryableAsyncTask <InvalidOperationException>((resultSetter) => this.CreateTableImpl(tableName, resultSetter), RetryPolicy);
yield return(createTableTask);
// wrap any exceptions
try
{
if (createTableTask.Result == null)
{
setResult(true);
}
else
{
StorageClientException exception = Utilities.TranslateDataServiceClientException(createTableTask.Result) as StorageClientException;
if (exception != null &&
exception.ErrorCode == StorageErrorCode.ResourceAlreadyExists &&
exception.ExtendedErrorInformation != null &&
exception.ExtendedErrorInformation.ErrorCode == TableErrorCodeStrings.TableAlreadyExists)
{
setResult(false);
}
else
{
throw createTableTask.Result;
}
}
}
catch (InvalidOperationException ex)
{
throw Utilities.TranslateDataServiceClientException(ex);
}
}
}
/// <summary>
/// Commits the blob by uploading any remaining data and the blocklist asynchronously.
/// </summary>
/// <returns>A sequence of events required for commit.</returns>
private TaskSequence CommitImpl()
{
this.CheckWriteState();
if (this.blockBuffer != null && this.blockBuffer.Length != 0)
{
var task = new InvokeTaskSequenceTask(this.FlushInternal);
yield return(task);
var result = task.Result; // Materialize the errors
Console.WriteLine(result);
}
// If all blocks are uploaded, commit
if (this.UseBlocks)
{
this.SetBlobMD5();
var task = TaskImplHelper.GetRetryableAsyncTask(
() =>
{
// At the convenience layer we always upload uncommitted blocks
List <PutBlockListItem> putBlockList = new List <PutBlockListItem>();
foreach (var id in this.blockList)
{
putBlockList.Add(new PutBlockListItem(id, BlockSearchMode.Uncommitted));
}
return(this.Blob.ToBlockBlob.UploadBlockList(putBlockList, this.currentModifier));
},
this.currentModifier.RetryPolicy);
yield return(task);
var result = task.Result;
Console.WriteLine(result);
}
// Now we can set the full size.
this.Blob.Properties.Length = this.Length;
// Clear the internal state
this.Abort();
}
/// <summary>
/// Deletes table if exists implementation.
/// </summary>
/// <param name="tableName">The table name.</param>
/// <param name="setResult">The set result.</param>
/// <returns>A sequence of tasks to do the operation.</returns>
private TaskSequence DeleteTableIfExistImpl(string tableName, Action <bool> setResult)
{
var doesTableExistTask = TaskImplHelper.GetRetryableAsyncTask <bool>(
(set) => this.DoesTableExistImpl(tableName, set),
RetryPolicy);
yield return(doesTableExistTask);
if (!doesTableExistTask.Result)
{
setResult(false);
}
else
{
var deleteTableTask = TaskImplHelper.GetRetryableAsyncTask(() => this.DeleteTableImpl(tableName), RetryPolicy);
yield return(deleteTableTask);
var result = deleteTableTask.Result;
setResult(true);
}
}
/// <summary>
/// Perform a parallel upload of blocks for a blob from a given stream.
/// </summary>
/// <param name="uploadFunc">The upload func.</param>
/// <returns>A <see cref="TaskSequence"/> that uploads the blob in parallel.</returns>
/// <remarks>
/// The operation is done as a series of alternate producer and consumer tasks. The producer tasks dispense out
/// chunks of source stream as fixed size blocks. This is done in serial order on a thread using InvokeTaskSequence's
/// serial execution. The consumer tasks upload each block in parallel on multiple thread. The producer thread waits
/// for at least one consumer task to finish before adding more producer tasks. The producer thread quits when no
/// more data can be read from the stream and no other pending consumer tasks.
/// </remarks>
internal TaskSequence ParallelExecute(
Func <SmallBlockMemoryStream, string, string, BlobRequestOptions, Task <NullTaskReturn> > uploadFunc)
{
bool moreToUpload = true;
List <IAsyncResult> asyncResults = new List <IAsyncResult>();
Random rand = new Random();
long blockIdSequenceNumber = (long)rand.Next() << 32;
blockIdSequenceNumber += rand.Next();
do
{
int currentPendingTasks = asyncResults.Count;
// Step 1
// Create producer tasks in a serial order as stream can only be read sequentially
for (int i = currentPendingTasks; i < this.parellelism && moreToUpload; i++)
{
string blockId = null;
string blockHash = null;
SmallBlockMemoryStream blockAsStream = null;
blockIdSequenceNumber++;
InvokeTaskSequenceTask producerTask = new InvokeTaskSequenceTask(() =>
this.DispenseBlockStream(
blockIdSequenceNumber,
(stream, id, hashVal) =>
{
blockAsStream = stream;
blockId = id;
blockHash = hashVal;
}));
yield return(producerTask);
this.producerTasksCreated++;
var scatch = producerTask.Result;
Console.WriteLine(scatch);
if (blockAsStream == null)
{
TraceHelper.WriteLine("No more upload tasks");
moreToUpload = false;
}
else
{
// Step 2
// Fire off consumer tasks that may finish on other threads;
var task = uploadFunc(blockAsStream, blockId, blockHash, this.options);
IAsyncResult asyncresult = task.ToAsyncResult(null, null);
this.consumerTasksCreated++;
asyncResults.Add(asyncresult);
}
}
// Step 3
// Wait for 1 or more consumer tasks to finish inorder to bound set of parallel tasks
if (asyncResults.Count > 0)
{
int waitTimeout = GetWaitTimeout(this.options);
TraceHelper.WriteLine("Starting wait");
int waitResult = WaitHandle.WaitAny(asyncResults.Select(result => result.AsyncWaitHandle).ToArray(), waitTimeout);
TraceHelper.WriteLine("Ending wait");
if (waitResult == WaitHandle.WaitTimeout)
{
throw TimeoutHelper.ThrowTimeoutError(this.options.Timeout.Value);
}
CompleteAsyncresult(asyncResults, waitResult);
// Optimize away any other completed tasks
for (int index = 0; index < asyncResults.Count; index++)
{
IAsyncResult result = asyncResults[index];
if (result.IsCompleted)
{
CompleteAsyncresult(asyncResults, index);
index--;
}
}
}
}while (moreToUpload || asyncResults.Count != 0);
TraceHelper.WriteLine(
"Total producer tasks created {0}, consumer tasks created {1} ",
this.producerTasksCreated,
this.consumerTasksCreated);
var commitTask = TaskImplHelper.GetRetryableAsyncTask(this.CommitBlob, this.options.RetryPolicy);
yield return(commitTask);
var commitTaskResult = commitTask.Result;
Console.WriteLine(commitTaskResult);
}
/// <summary>
/// Performs the read operation.
/// </summary>
/// <param name="buffer">The buffer to read the data into.</param>
/// <param name="offset">The zero-based byte offset in buffer at which to begin storing the data read from the current stream.</param>
/// <param name="count">The maximum number of bytes to be read from the current stream.</param>
/// <param name="setResult">The action to be done upon completion to return the number of bytes read.</param>
/// <returns>A task sequence representing the operation.</returns>
/// <remarks>
/// If verification is on, retrieve the blocklist.
/// If there are downloaded blocks, read from there.
/// Otherwise, if there's an outstanding request, wait for completion and read from there.
/// Otherwise perform a new request.
/// </remarks>
private TaskSequence ReadImpl(byte[] buffer, int offset, int count, Action <int> setResult)
{
if (this.Blob.Properties.BlobType == BlobType.Unspecified)
{
this.Blob.FetchAttributes(this.options);
}
var origCount = count;
// If we don't have a blocklist and need it, get it.
if (this.Blob.Properties.BlobType == BlobType.BlockBlob && this.IntegrityControlVerificationEnabled && this.blockList == null)
{
var blockListTask = TaskImplHelper.GetRetryableAsyncTask <IEnumerable <ListBlockItem> >(
(result) =>
{
return(this.Blob.ToBlockBlob.GetDownloadBlockList(
BlockListingFilter.Committed,
this.options,
result));
},
this.options.RetryPolicy);
yield return(blockListTask);
this.blockList = blockListTask.Result.ToList();
// Verify we got a blocklist and it's in valid state. This will disable verification if it's invalid state
this.VerifyBlocks();
}
bool readComplete = false;
// Clear any pending read-aheads
if (this.readAheadResult != null && this.readAheadResult.IsCompleted)
{
this.readAheadResult = null;
}
// If we have any data, read existing data
if (this.downloadedBlocksList.Any())
{
readComplete = this.ReadBufferedData(buffer, ref offset, ref count);
}
// If we didn't complete our read and have outstanding readAhead, wait on it
if (!readComplete && this.readAheadResult != null && !this.readAheadResult.IsCompleted)
{
this.readAheadResult.AsyncWaitHandle.WaitOne();
this.readAheadResult = null;
// We now have more data, so we can do some read
readComplete = this.ReadBufferedData(buffer, ref offset, ref count);
}
long gapStart, gapEnd;
this.downloadedBlocksList.CalculateGapLength(this.position, count + this.readAheadSize, out gapStart, out gapEnd);
// Figure out if we need to do a server request. There are two reasons:
// * We have satisfied the read request, but the remaining data is low enough to warrant a ReadAhead
// * We didn't satisfy any of the read request and therefore must do it now
if (this.CalculateIfBufferTooLow(gapStart) || !readComplete)
{
long startReadAhead;
long readAheadCount;
this.CalculateReadAheadBounds(gapStart, gapEnd, count, out startReadAhead, out readAheadCount);
var blocksLeft = startReadAhead != -1;
var outsideBounds = this.LengthAvailable && (this.Position >= this.Length || startReadAhead >= this.Length);
// If we didn't find any blocks, that means there's no data left.
// If we have length, we can ensure we are within bounds. This will prevent extra round trips
if (!blocksLeft || outsideBounds)
{
setResult(origCount - count);
yield break;
}
// If we are doing an optional read-ahead, save the async result for future use
if (readComplete)
{
// We should only have a single outstanding ReadAhed
if (this.readAheadResult == null)
{
this.readAheadResult = TaskImplHelper.BeginImplWithRetry(
() =>
{
return(this.ReadAheadImpl(startReadAhead, readAheadCount));
},
this.options.RetryPolicy,
(result) =>
{
this.EndReadAhead(result);
},
null);
}
}
else
{
var task = TaskImplHelper.GetRetryableAsyncTask(() => this.ReadAheadImpl(startReadAhead, readAheadCount), this.options.RetryPolicy);
yield return(task);
var scratch = task.Result;
// We now have data, so we read it
readComplete = this.ReadBufferedData(buffer, ref offset, ref count);
}
}
setResult(origCount - count);
}
