/// <summary> /// Verifies whether the table exist implementation. /// </summary> /// <param name="tableName">The table name.</param> /// <param name="setResult">The set result.</param> /// <returns>A sequence of tasks to do the operation.</returns> private TaskSequence DoesTableExistImpl(string tableName, Action <bool> setResult) { CommonUtils.CheckStringParameter(tableName, false, "tableName", Protocol.Constants.TableServiceMaxStringPropertySizeInChars); TableServiceUtilities.CheckTableName(tableName, "tableName"); var svc = this.GetDataServiceContext(); var tableExistsQuery = (from table in svc.CreateQuery <TableServiceTable>(Protocol.Constants.TableServiceTablesName) where table.TableName == tableName select table).AsTableServiceQuery(); ResultSegment <TableServiceTable> segment = null; while (true) { Task <ResultSegment <TableServiceTable> > tableExistsSegmentTask; if (segment == null) { tableExistsSegmentTask = TaskImplHelper.GetRetryableAsyncTask <ResultSegment <TableServiceTable> >( (setResultInner) => tableExistsQuery.ExecuteSegmentedImpl(null, setResultInner), RetryPolicy); } else { tableExistsSegmentTask = TaskImplHelper.GetRetryableAsyncTask <ResultSegment <TableServiceTable> >(segment.GetNextImpl, RetryPolicy); } yield return(tableExistsSegmentTask); if (GetResultOrDefault(tableExistsSegmentTask, out segment)) { if (segment.Results.Any()) { setResult(true); break; } else { setResult(false); break; } } else { setResult(false); break; } } }
/// <summary> /// Creates the table if not exist implementation. /// </summary> /// <param name="tableName">The table name.</param> /// <param name="setResult">The set result.</param> /// <returns>A sequence of tasks to do the operation.</returns> private TaskSequence CreateTableIfNotExistImpl(string tableName, Action <bool> setResult) { CommonUtils.CheckStringParameter(tableName, false, "tableName", Protocol.Constants.TableServiceMaxStringPropertySizeInChars); TableServiceUtilities.CheckTableName(tableName, "tableName"); var doesTableExistTask = new InvokeTaskSequenceTask <bool>((set) => this.DoesTableExistImpl(tableName, set)); yield return(doesTableExistTask); if (doesTableExistTask.Result) { setResult(false); } else { var createTableTask = TaskImplHelper.GetRetryableAsyncTask <InvalidOperationException>((resultSetter) => this.CreateTableImpl(tableName, resultSetter), RetryPolicy); yield return(createTableTask); // wrap any exceptions try { if (createTableTask.Result == null) { setResult(true); } else { StorageClientException exception = Utilities.TranslateDataServiceClientException(createTableTask.Result) as StorageClientException; if (exception != null && exception.ErrorCode == StorageErrorCode.ResourceAlreadyExists && exception.ExtendedErrorInformation != null && exception.ExtendedErrorInformation.ErrorCode == TableErrorCodeStrings.TableAlreadyExists) { setResult(false); } else { throw createTableTask.Result; } } } catch (InvalidOperationException ex) { throw Utilities.TranslateDataServiceClientException(ex); } } }
/// <summary> /// Commits the blob by uploading any remaining data and the blocklist asynchronously. /// </summary> /// <returns>A sequence of events required for commit.</returns> private TaskSequence CommitImpl() { this.CheckWriteState(); if (this.blockBuffer != null && this.blockBuffer.Length != 0) { var task = new InvokeTaskSequenceTask(this.FlushInternal); yield return(task); var result = task.Result; // Materialize the errors Console.WriteLine(result); } // If all blocks are uploaded, commit if (this.UseBlocks) { this.SetBlobMD5(); var task = TaskImplHelper.GetRetryableAsyncTask( () => { // At the convenience layer we always upload uncommitted blocks List <PutBlockListItem> putBlockList = new List <PutBlockListItem>(); foreach (var id in this.blockList) { putBlockList.Add(new PutBlockListItem(id, BlockSearchMode.Uncommitted)); } return(this.Blob.ToBlockBlob.UploadBlockList(putBlockList, this.currentModifier)); }, this.currentModifier.RetryPolicy); yield return(task); var result = task.Result; Console.WriteLine(result); } // Now we can set the full size. this.Blob.Properties.Length = this.Length; // Clear the internal state this.Abort(); }
/// <summary> /// Deletes table if exists implementation. /// </summary> /// <param name="tableName">The table name.</param> /// <param name="setResult">The set result.</param> /// <returns>A sequence of tasks to do the operation.</returns> private TaskSequence DeleteTableIfExistImpl(string tableName, Action <bool> setResult) { var doesTableExistTask = TaskImplHelper.GetRetryableAsyncTask <bool>( (set) => this.DoesTableExistImpl(tableName, set), RetryPolicy); yield return(doesTableExistTask); if (!doesTableExistTask.Result) { setResult(false); } else { var deleteTableTask = TaskImplHelper.GetRetryableAsyncTask(() => this.DeleteTableImpl(tableName), RetryPolicy); yield return(deleteTableTask); var result = deleteTableTask.Result; setResult(true); } }
/// <summary> /// Perform a parallel upload of blocks for a blob from a given stream. /// </summary> /// <param name="uploadFunc">The upload func.</param> /// <returns>A <see cref="TaskSequence"/> that uploads the blob in parallel.</returns> /// <remarks> /// The operation is done as a series of alternate producer and consumer tasks. The producer tasks dispense out /// chunks of source stream as fixed size blocks. This is done in serial order on a thread using InvokeTaskSequence's /// serial execution. The consumer tasks upload each block in parallel on multiple thread. The producer thread waits /// for at least one consumer task to finish before adding more producer tasks. The producer thread quits when no /// more data can be read from the stream and no other pending consumer tasks. /// </remarks> internal TaskSequence ParallelExecute( Func <SmallBlockMemoryStream, string, string, BlobRequestOptions, Task <NullTaskReturn> > uploadFunc) { bool moreToUpload = true; List <IAsyncResult> asyncResults = new List <IAsyncResult>(); Random rand = new Random(); long blockIdSequenceNumber = (long)rand.Next() << 32; blockIdSequenceNumber += rand.Next(); do { int currentPendingTasks = asyncResults.Count; // Step 1 // Create producer tasks in a serial order as stream can only be read sequentially for (int i = currentPendingTasks; i < this.parellelism && moreToUpload; i++) { string blockId = null; string blockHash = null; SmallBlockMemoryStream blockAsStream = null; blockIdSequenceNumber++; InvokeTaskSequenceTask producerTask = new InvokeTaskSequenceTask(() => this.DispenseBlockStream( blockIdSequenceNumber, (stream, id, hashVal) => { blockAsStream = stream; blockId = id; blockHash = hashVal; })); yield return(producerTask); this.producerTasksCreated++; var scatch = producerTask.Result; Console.WriteLine(scatch); if (blockAsStream == null) { TraceHelper.WriteLine("No more upload tasks"); moreToUpload = false; } else { // Step 2 // Fire off consumer tasks that may finish on other threads; var task = uploadFunc(blockAsStream, blockId, blockHash, this.options); IAsyncResult asyncresult = task.ToAsyncResult(null, null); this.consumerTasksCreated++; asyncResults.Add(asyncresult); } } // Step 3 // Wait for 1 or more consumer tasks to finish inorder to bound set of parallel tasks if (asyncResults.Count > 0) { int waitTimeout = GetWaitTimeout(this.options); TraceHelper.WriteLine("Starting wait"); int waitResult = WaitHandle.WaitAny(asyncResults.Select(result => result.AsyncWaitHandle).ToArray(), waitTimeout); TraceHelper.WriteLine("Ending wait"); if (waitResult == WaitHandle.WaitTimeout) { throw TimeoutHelper.ThrowTimeoutError(this.options.Timeout.Value); } CompleteAsyncresult(asyncResults, waitResult); // Optimize away any other completed tasks for (int index = 0; index < asyncResults.Count; index++) { IAsyncResult result = asyncResults[index]; if (result.IsCompleted) { CompleteAsyncresult(asyncResults, index); index--; } } } }while (moreToUpload || asyncResults.Count != 0); TraceHelper.WriteLine( "Total producer tasks created {0}, consumer tasks created {1} ", this.producerTasksCreated, this.consumerTasksCreated); var commitTask = TaskImplHelper.GetRetryableAsyncTask(this.CommitBlob, this.options.RetryPolicy); yield return(commitTask); var commitTaskResult = commitTask.Result; Console.WriteLine(commitTaskResult); }
/// <summary> /// Performs the read operation. /// </summary> /// <param name="buffer">The buffer to read the data into.</param> /// <param name="offset">The zero-based byte offset in buffer at which to begin storing the data read from the current stream.</param> /// <param name="count">The maximum number of bytes to be read from the current stream.</param> /// <param name="setResult">The action to be done upon completion to return the number of bytes read.</param> /// <returns>A task sequence representing the operation.</returns> /// <remarks> /// If verification is on, retrieve the blocklist. /// If there are downloaded blocks, read from there. /// Otherwise, if there's an outstanding request, wait for completion and read from there. /// Otherwise perform a new request. /// </remarks> private TaskSequence ReadImpl(byte[] buffer, int offset, int count, Action <int> setResult) { if (this.Blob.Properties.BlobType == BlobType.Unspecified) { this.Blob.FetchAttributes(this.options); } var origCount = count; // If we don't have a blocklist and need it, get it. if (this.Blob.Properties.BlobType == BlobType.BlockBlob && this.IntegrityControlVerificationEnabled && this.blockList == null) { var blockListTask = TaskImplHelper.GetRetryableAsyncTask <IEnumerable <ListBlockItem> >( (result) => { return(this.Blob.ToBlockBlob.GetDownloadBlockList( BlockListingFilter.Committed, this.options, result)); }, this.options.RetryPolicy); yield return(blockListTask); this.blockList = blockListTask.Result.ToList(); // Verify we got a blocklist and it's in valid state. This will disable verification if it's invalid state this.VerifyBlocks(); } bool readComplete = false; // Clear any pending read-aheads if (this.readAheadResult != null && this.readAheadResult.IsCompleted) { this.readAheadResult = null; } // If we have any data, read existing data if (this.downloadedBlocksList.Any()) { readComplete = this.ReadBufferedData(buffer, ref offset, ref count); } // If we didn't complete our read and have outstanding readAhead, wait on it if (!readComplete && this.readAheadResult != null && !this.readAheadResult.IsCompleted) { this.readAheadResult.AsyncWaitHandle.WaitOne(); this.readAheadResult = null; // We now have more data, so we can do some read readComplete = this.ReadBufferedData(buffer, ref offset, ref count); } long gapStart, gapEnd; this.downloadedBlocksList.CalculateGapLength(this.position, count + this.readAheadSize, out gapStart, out gapEnd); // Figure out if we need to do a server request. There are two reasons: // * We have satisfied the read request, but the remaining data is low enough to warrant a ReadAhead // * We didn't satisfy any of the read request and therefore must do it now if (this.CalculateIfBufferTooLow(gapStart) || !readComplete) { long startReadAhead; long readAheadCount; this.CalculateReadAheadBounds(gapStart, gapEnd, count, out startReadAhead, out readAheadCount); var blocksLeft = startReadAhead != -1; var outsideBounds = this.LengthAvailable && (this.Position >= this.Length || startReadAhead >= this.Length); // If we didn't find any blocks, that means there's no data left. // If we have length, we can ensure we are within bounds. This will prevent extra round trips if (!blocksLeft || outsideBounds) { setResult(origCount - count); yield break; } // If we are doing an optional read-ahead, save the async result for future use if (readComplete) { // We should only have a single outstanding ReadAhed if (this.readAheadResult == null) { this.readAheadResult = TaskImplHelper.BeginImplWithRetry( () => { return(this.ReadAheadImpl(startReadAhead, readAheadCount)); }, this.options.RetryPolicy, (result) => { this.EndReadAhead(result); }, null); } } else { var task = TaskImplHelper.GetRetryableAsyncTask(() => this.ReadAheadImpl(startReadAhead, readAheadCount), this.options.RetryPolicy); yield return(task); var scratch = task.Result; // We now have data, so we read it readComplete = this.ReadBufferedData(buffer, ref offset, ref count); } } setResult(origCount - count); }