async Task CopyAsync(TableRequestOptions requestOptions, OperationContext operationContext) { // Query all the entities! IQueryStream <MTableEntity> oldTableStream = await oldTable.ExecuteQueryStreamedAsync( new TableQuery <MTableEntity>(), requestOptions, operationContext); MTableEntity oldEntity; string previousPartitionKey = null; while ((oldEntity = await oldTableStream.ReadRowAsync()) != null) { if (RowKeyIsInternal(oldEntity.RowKey)) { continue; } if (oldEntity.PartitionKey != previousPartitionKey) { previousPartitionKey = oldEntity.PartitionKey; await EnsurePartitionSwitchedAsync(oldEntity.PartitionKey, requestOptions, operationContext); if (IsBugEnabled(MTableOptionalBug.InsertBehindMigrator)) { // More reasonable formulation of this bug. If we're going // to allow CopyAsync while some clients are still writing // to the old table, give the developer credit for realizing // that the stream might be stale by the time the partition // is switched. oldTableStream = await oldTable.ExecuteQueryStreamedAsync( new TableQuery <MTableEntity> { FilterString = TableQuery.GenerateFilterCondition( TableConstants.PartitionKey, QueryComparisons.GreaterThanOrEqual, oldEntity.PartitionKey) }, requestOptions, operationContext); continue; } } // Can oldEntity be stale by the time we EnsurePartitionSwitchedAsync? // Currently no, because we don't start the copy until all clients have // entered PREFER_NEW state, meaning that all writes go to the new // table. When we implement the Kstart/Kdone optimization (or similar), // then clients will be able to write to the old table in parallel with // our scan and we'll need to rescan after EnsurePartitionSwitchedAsync. // Should we also stream from the new table and copy only the // entities not already copied? await TryCopyEntityToNewTableAsync(oldEntity, requestOptions, operationContext); } }
internal async Task RunQueryStreamedAsync(TableQuery <DynamicTableEntity> query) { int startRevision = await peekProxy.GetReferenceTableRevisionAsync(); FilterExpression filterExpr = ChainTableUtils.ParseFilterString(query.FilterString); Console.WriteLine("{0} starting streaming query: {1}", machineId, query); using (IQueryStream <DynamicTableEntity> stream = await migratingTable.ExecuteQueryStreamedAsync(query)) { PrimaryKey lastKey = ChainTableUtils.FirstValidPrimaryKey; for (;;) { PrimaryKey returnedContinuationKey = await stream.GetContinuationPrimaryKeyAsync(); PSharpRuntime.Assert(returnedContinuationKey == null || returnedContinuationKey.CompareTo(lastKey) >= 0, "{0}: query stream continuation key is {1}, expected >= {2}", machineId, returnedContinuationKey, lastKey); DynamicTableEntity row = await stream.ReadRowAsync(); // may be null, meaning end of stream // Must be after ReadRowAsync, otherwise additional rows could become valid // due to a mutation between GetValidStreamReadRows and ReadRowAsync and // we would falsely report a bug if ReadRowAsync returns one of those rows. List <DynamicTableEntity> validRows = await peekProxy.GetValidStreamReadRows(startRevision, filterExpr, lastKey); // Three cheers for automatic use of covariance in overload resolution! PSharpRuntime.Assert(validRows.Contains(row, BetterComparer.Instance), "{0} query stream returned {1}, which is not one of the valid rows: {2}", machineId, BetterComparer.ToString(row), BetterComparer.ToString(validRows)); Console.WriteLine("{0} query stream returned row {1}, which is valid", machineId, BetterComparer.ToString(row)); if (row == null) { // Any returnedContinuationKey (including null) is less or equal to a row of null. break; } else { PSharpRuntime.Assert(returnedContinuationKey != null && returnedContinuationKey.CompareTo(row.GetPrimaryKey()) <= 0, "{0}: query stream continuation key is {1}, expected <= {2}", machineId, returnedContinuationKey, row.GetPrimaryKey()); lastKey = ChainTableUtils.NextValidPrimaryKeyAfter(row.GetPrimaryKey()); } } } Console.WriteLine("{0} finished streaming query", machineId); }
internal async Task StartAsync() { using (await LockAsyncBuggable()) { configSubscription = outer.configService.Subscribe(new Subscriber(this), out currentConfig); if (currentConfig.state < TableClientState.USE_NEW_WITH_TOMBSTONES) { oldTableStream = await outer.oldTable.ExecuteQueryStreamedAsync(mtableQuery, requestOptions, operationContext); oldTableNext = await oldTableStream.ReadRowAsync(); } if (currentConfig.state > TableClientState.USE_OLD_HIDE_METADATA) { TableQuery <MTableEntity> newTableQuery = outer.IsBugEnabled(MTableOptionalBug.QueryStreamedFilterShadowing) ? mtableQuery // Yes, match everything (!). See newTableContinuationQuery in ApplyConfigurationAsync. : new TableQuery <MTableEntity>(); newTableStream = await outer.newTable.ExecuteQueryStreamedAsync(newTableQuery, requestOptions, operationContext); newTableNext = await newTableStream.ReadRowAsync(); } } }
async Task DoQueryStreamed() { int startRevision = await peekProxy.GetReferenceTableRevisionAsync(); // XXX: Test the filtering? var query = new TableQuery <DynamicTableEntity>(); using (IQueryStream <DynamicTableEntity> stream = await migratingTable.ExecuteQueryStreamedAsync(query)) { PrimaryKey continuationKey = await stream.GetContinuationPrimaryKeyAsync(); await peekProxy.ValidateQueryStreamGapAsync(startRevision, null, continuationKey); do { DynamicTableEntity row = await stream.ReadRowAsync(); PrimaryKey newContinuationKey = await stream.GetContinuationPrimaryKeyAsync(); if (row == null) { PSharpRuntime.Assert(newContinuationKey == null); await peekProxy.ValidateQueryStreamGapAsync(startRevision, continuationKey, null); } else { await peekProxy.ValidateQueryStreamGapAsync(startRevision, continuationKey, row.GetPrimaryKey()); await peekProxy.ValidateQueryStreamRowAsync(startRevision, row); await peekProxy.ValidateQueryStreamGapAsync(startRevision, ChainTableUtils.NextValidPrimaryKeyAfter(row.GetPrimaryKey()), newContinuationKey); } continuationKey = newContinuationKey; } while (continuationKey != null); } }
// XXX: This could loop a long time. Do we want to make it cancelable? public async Task <TElement> ReadRowAsync() { using (await LockAsyncBuggable()) { CheckDisposed(); for (;;) { // Figure out which side has the next primary key to return (or // both) and pull the row(s) with that key from one or both sides. int?cmp = DetermineNextSide(); if (cmp == null) { return(default(TElement)); } MTableEntity nextInOld = null, nextInNew = null; if (cmp <= 0) { nextInOld = oldTableNext; oldTableNext = await oldTableStream.ReadRowAsync(); } if (cmp >= 0) { nextInNew = newTableNext; newTableNext = await newTableStream.ReadRowAsync(); } MTableEntity ent = (nextInNew != null) ? nextInNew : nextInOld; // Compare to end of ExecuteQueryAtomicAsync. if (!RowKeyIsInternal(ent.RowKey) && origFilterExpr.Evaluate(ent) && !ent.deleted) { return(ent.Export <TElement>()); } // Otherwise keep going. } } }
public Task <TElement> ReadRowAsync() { return(plainProxy.ReadRowAsync()); }
async Task CleanupAsync(TableRequestOptions requestOptions, OperationContext operationContext) { // Clean up MTable-specific data from new table. // Future: Query only ones with properties we need to clean up. IQueryStream <MTableEntity> newTableStream = await newTable.ExecuteQueryStreamedAsync( new TableQuery <MTableEntity>(), requestOptions, operationContext); MTableEntity newEntity; while ((newEntity = await newTableStream.ReadRowAsync()) != null) { // XXX: Consider factoring out this "query and retry" pattern // into a separate method. Attempt: TableOperation cleanupOp = newEntity.deleted ? TableOperation.Delete(newEntity) : TableOperation.Replace(newEntity.Export <DynamicTableEntity>()); TableResult cleanupResult; try { cleanupResult = await newTable.ExecuteAsync(cleanupOp, requestOptions, operationContext); } catch (StorageException ex) { if (ex.GetHttpStatusCode() == HttpStatusCode.NotFound) { // Someone else deleted it concurrently. Nothing to do. await monitor.AnnotateLastBackendCallAsync(); continue; } else if (ex.GetHttpStatusCode() == HttpStatusCode.PreconditionFailed) { await monitor.AnnotateLastBackendCallAsync(); // Unfortunately we can't assume that anyone who concurrently modifies // the row while the table is in state USE_NEW_HIDE_METADATA will // clean it up, because of InsertOrMerge. (Consider redesign?) // Re-retrieve row. TableResult retrieveResult = await newTable.ExecuteAsync( TableOperation.Retrieve <MTableEntity>(newEntity.PartitionKey, newEntity.RowKey), requestOptions, operationContext); await monitor.AnnotateLastBackendCallAsync(); if ((HttpStatusCode)retrieveResult.HttpStatusCode == HttpStatusCode.NotFound) { continue; } else { newEntity = (MTableEntity)retrieveResult.Result; goto Attempt; } } else { throw ChainTableUtils.GenerateInternalException(ex); } } await monitor.AnnotateLastBackendCallAsync( spuriousETagChanges : cleanupResult.Etag == null?null : new List <SpuriousETagChange> { new SpuriousETagChange(newEntity.PartitionKey, newEntity.RowKey, cleanupResult.Etag) }); } // Delete everything from old table! No one should be modifying it concurrently. IQueryStream <MTableEntity> oldTableStream = await oldTable.ExecuteQueryStreamedAsync( new TableQuery <MTableEntity>(), requestOptions, operationContext); MTableEntity oldEntity; while ((oldEntity = await oldTableStream.ReadRowAsync()) != null) { await oldTable.ExecuteAsync(TableOperation.Delete(oldEntity), requestOptions, operationContext); await monitor.AnnotateLastBackendCallAsync(); } }
public Task <TElement> ReadRowAsync() { CallLogging.LogStart(proxyName, nameof(ReadRowAsync)); return(LogOutcomeAsync(() => original.ReadRowAsync(), (outcome) => CallLogging.LogEnd(proxyName, nameof(ReadRowAsync), outcome))); }
public Task <TElement> ReadRowAsync() { return(nonannotatableProxy.ReadRowAsync()); }
Task WalkTableInParallel(IChainTable2 table, TableRequestOptions requestOptions, OperationContext operationContext, Func <MTableEntity, Task <IQueryStream <MTableEntity> > > rowCallbackAsync) { return(Task.WhenAll(Enumerable.Range(0, 16).Select(hexNumber => Task.Run(async() => { var query = new TableQuery <MTableEntity>(); if (hexNumber == 0) { query.FilterString = TableQuery.GenerateFilterCondition( "PartitionKey", QueryComparisons.LessThanOrEqual, (hexNumber + 1).ToString("x")); } else if (hexNumber == 15) { query.FilterString = TableQuery.GenerateFilterCondition( "PartitionKey", QueryComparisons.GreaterThanOrEqual, hexNumber.ToString("x")); } else { query.FilterString = TableQuery.CombineFilters( TableQuery.GenerateFilterCondition( "PartitionKey", QueryComparisons.GreaterThanOrEqual, hexNumber.ToString("x")), TableOperators.And, TableQuery.GenerateFilterCondition( "PartitionKey", QueryComparisons.LessThanOrEqual, (hexNumber + 1).ToString("x"))); } try { IQueryStream <MTableEntity> tableStream = await table.ExecuteQueryStreamedAsync( query, requestOptions, operationContext); MTableEntity entity; while ((entity = await tableStream.ReadRowAsync()) != null) { IQueryStream <MTableEntity> newTableStream = await rowCallbackAsync(entity); if (newTableStream != null) { tableStream = newTableStream; } } } catch (Exception e) { while (e is AggregateException) { e = ((AggregateException)(e)).InnerException; } if (!(e is StorageException && ((StorageException)e).RequestInformation.HttpStatusCode == 404)) { throw; } } })))); }