MTableEntity ImportWithIfMatch(ITableEntity passedEntity, string ifMatch) { MTableEntity newEntity = ChainTableUtils.CopyEntity <MTableEntity>(passedEntity); newEntity.ETag = ifMatch; return(newEntity); }
private void Merge(DynamicTableEntity to, ITableEntity from) { foreach (KeyValuePair <string, EntityProperty> kvp in from.WriteEntity(null)) { to.Properties[kvp.Key] = ChainTableUtils.CopyProperty(kvp.Value); } }
async Task EnsureAffectedRowsMigratedAsync(TableBatchOperation batch, TableRequestOptions requestOptions, OperationContext operationContext) { string partitionKey = ChainTableUtils.GetBatchPartitionKey(batch); var query = GenerateQueryForAffectedRows(batch); IList <MTableEntity> oldRows = await oldTable.ExecuteQueryAtomicAsync(query, requestOptions, operationContext); await monitor.AnnotateLastBackendCallAsync(); IList <MTableEntity> newRows = await newTable.ExecuteQueryAtomicAsync(query, requestOptions, operationContext); await monitor.AnnotateLastBackendCallAsync(); Dictionary <string, MTableEntity> oldDict = oldRows.ToDictionary(ent => ent.RowKey); Dictionary <string, MTableEntity> newDict = newRows.ToDictionary(ent => ent.RowKey); // Migrate any affected rows not already migrated. foreach (TableOperation op in batch) { string targetedRowKey = op.GetEntity().RowKey; MTableEntity oldEntity; if (oldDict.TryGetValue(targetedRowKey, out oldEntity) && !newDict.ContainsKey(targetedRowKey)) { await TryCopyEntityToNewTableAsync(oldEntity, requestOptions, operationContext); } } }
// Marks the partition populated unless it is already switched. async Task TryMarkPartitionPopulatedAsync(string partitionKey, TableRequestOptions requestOptions, OperationContext operationContext) { var markPopulatedBatch = new TableBatchOperation(); markPopulatedBatch.Insert(new MTableEntity { PartitionKey = partitionKey, RowKey = ROW_KEY_PARTITION_META, partitionState = MTablePartitionState.POPULATED }); markPopulatedBatch.Insert(new DynamicTableEntity { PartitionKey = partitionKey, RowKey = ROW_KEY_PARTITION_POPULATED_ASSERTION }); try { await oldTable.ExecuteBatchAsync(markPopulatedBatch, requestOptions, operationContext); } // XXX: Optimization opportunity: if we swap the order of the // inserts, we can tell here if the partition is already switched. catch (StorageException ex) { if (ex.GetHttpStatusCode() != HttpStatusCode.Conflict) { throw ChainTableUtils.GenerateInternalException(ex); } } await monitor.AnnotateLastBackendCallAsync(); }
internal QueryStream(MigratingTable outer, TableQuery <TElement> query, TableRequestOptions requestOptions, OperationContext operationContext) { this.outer = outer; origFilterExpr = ChainTableUtils.ParseFilterString(query.FilterString); mtableQuery = ChainTableUtils.CopyQuery <TElement, MTableEntity>(query); this.requestOptions = requestOptions; this.operationContext = operationContext; }
public SortedDictionary <PrimaryKey, DynamicTableEntity> Dump() { var dump = new SortedDictionary <PrimaryKey, DynamicTableEntity>(); foreach (KeyValuePair <PrimaryKey, DynamicTableEntity> kvp in table) { dump.Add(kvp.Key, ChainTableUtils.CopyEntity <DynamicTableEntity>(kvp.Value)); } return(dump); }
internal async Task RunBatchAsync(TableBatchOperation batch) { TableBatchOperation batchCopy = ChainTableUtils.CopyBatch <DynamicTableEntity>(batch); TableCall originalCall = async table => await table.ExecuteBatchAsync(batch); MirrorTableCall referenceCall = async referenceTable => await referenceTable.ExecuteMirrorBatchAsync(batchCopy, successfulBatchResult); Console.WriteLine("{0} starting batch: {1}", machineId, BetterComparer.ToString(batch)); await RunCallAsync(originalCall, referenceCall); }
public Task <TElement> ReadRowAsync() { CheckDisposed(); if (enumerator == null) { return(Task.FromResult(default(TElement))); } TElement entity = ChainTableUtils.CopyEntity <TElement>(enumerator.Current.Value); MoveNext(); return(Task.FromResult(entity)); }
public override Task <IQueryStream <TElement> > ExecuteQueryStreamedAsync <TElement>(TableQuery <TElement> query, TableRequestOptions requestOptions = null, OperationContext operationContext = null) { FilterExpression filterExpr = ChainTableUtils.ParseFilterString(query.FilterString); if (query.SelectColumns != null) { throw new NotImplementedException("select"); } if (query.TakeCount != null) { throw new NotImplementedException("top"); } return(Task.FromResult((IQueryStream <TElement>) new NondeterministicQueryStream <TElement>(this, CurrentRevision, filterExpr))); }
public Task <TElement> ReadRowAsync() { if (continuationKey == null) { return(Task.FromResult(default(TElement))); } List <DynamicTableEntity> possibleRows = outer.GetValidStreamReadRows(startRevision, filterExpr, continuationKey); int choiceIndex = PSharpNondeterminism.Choice(possibleRows.Count); DynamicTableEntity choice = possibleRows[choiceIndex]; Console.WriteLine("NondeterministicQueryStream: possibleRows {0}, choiceIndex {1}", BetterComparer.ToString(possibleRows), choiceIndex); continuationKey = (choice == null) ? null : ChainTableUtils.NextValidPrimaryKeyAfter(choice.GetPrimaryKey()); return(Task.FromResult((choice == null) ? default(TElement) : ChainTableUtils.CopyEntity <TElement>(choice))); }
internal TElement Export <TElement>() where TElement : ITableEntity, new() { var ent2 = new TElement() { PartitionKey = PartitionKey, RowKey = RowKey, ETag = ETag, Timestamp = Timestamp, }; // This copy probably isn't needed in the common case, // but I'd rather not even have to think about it. ent2.ReadEntity(ChainTableUtils.CopyPropertyDict(userProperties), null); return(ent2); }
internal async Task RunQueryStreamedAsync(TableQuery <DynamicTableEntity> query) { int startRevision = await peekProxy.GetReferenceTableRevisionAsync(); FilterExpression filterExpr = ChainTableUtils.ParseFilterString(query.FilterString); Console.WriteLine("{0} starting streaming query: {1}", machineId, query); using (IQueryStream <DynamicTableEntity> stream = await migratingTable.ExecuteQueryStreamedAsync(query)) { PrimaryKey lastKey = ChainTableUtils.FirstValidPrimaryKey; for (;;) { PrimaryKey returnedContinuationKey = await stream.GetContinuationPrimaryKeyAsync(); PSharpRuntime.Assert(returnedContinuationKey == null || returnedContinuationKey.CompareTo(lastKey) >= 0, "{0}: query stream continuation key is {1}, expected >= {2}", machineId, returnedContinuationKey, lastKey); DynamicTableEntity row = await stream.ReadRowAsync(); // may be null, meaning end of stream // Must be after ReadRowAsync, otherwise additional rows could become valid // due to a mutation between GetValidStreamReadRows and ReadRowAsync and // we would falsely report a bug if ReadRowAsync returns one of those rows. List <DynamicTableEntity> validRows = await peekProxy.GetValidStreamReadRows(startRevision, filterExpr, lastKey); // Three cheers for automatic use of covariance in overload resolution! PSharpRuntime.Assert(validRows.Contains(row, BetterComparer.Instance), "{0} query stream returned {1}, which is not one of the valid rows: {2}", machineId, BetterComparer.ToString(row), BetterComparer.ToString(validRows)); Console.WriteLine("{0} query stream returned row {1}, which is valid", machineId, BetterComparer.ToString(row)); if (row == null) { // Any returnedContinuationKey (including null) is less or equal to a row of null. break; } else { PSharpRuntime.Assert(returnedContinuationKey != null && returnedContinuationKey.CompareTo(row.GetPrimaryKey()) <= 0, "{0}: query stream continuation key is {1}, expected <= {2}", machineId, returnedContinuationKey, row.GetPrimaryKey()); lastKey = ChainTableUtils.NextValidPrimaryKeyAfter(row.GetPrimaryKey()); } } } Console.WriteLine("{0} finished streaming query", machineId); }
async Task <IList <TableResult> > AttemptBatchOnOldTableAsync(TableBatchOperation batch, TableRequestOptions requestOptions, OperationContext operationContext) { string partitionKey = ChainTableUtils.GetBatchPartitionKey(batch); await TryMarkPartitionPopulatedAsync(partitionKey, requestOptions, operationContext); var oldBatch = new TableBatchOperation(); oldBatch.Merge(new DynamicTableEntity { PartitionKey = partitionKey, RowKey = ROW_KEY_PARTITION_POPULATED_ASSERTION, ETag = ChainTable2Constants.ETAG_ANY, }); // No AddRange? :( foreach (TableOperation op in batch) { oldBatch.Add(op); } IList <TableResult> oldResults; try { oldResults = await oldTable.ExecuteBatchAsync(oldBatch, requestOptions, operationContext); } catch (ChainTableBatchException ex) { if (ex.FailedOpIndex == 0) { // This must mean the partition is switched. await monitor.AnnotateLastBackendCallAsync(); return(null); } else { await monitor.AnnotateLastBackendCallAsync(wasLinearizationPoint : true); throw ChainTableUtils.GenerateBatchException(ex.GetHttpStatusCode(), ex.FailedOpIndex - 1); } } oldResults.RemoveAt(0); await monitor.AnnotateLastBackendCallAsync(wasLinearizationPoint : true, successfulBatchResult : oldResults); return(oldResults); }
public override Task <IList <TElement> > ExecuteQueryAtomicAsync <TElement>( TableQuery <TElement> query, TableRequestOptions requestOptions = null, OperationContext operationContext = null) { FilterExpression filterExpr = ChainTableUtils.ParseFilterString(query.FilterString); ChainTableUtils.GetSingleTargetedPartitionKey(filterExpr); // validation, ignore result if (query.SelectColumns != null) { throw new NotImplementedException("select"); } if (query.TakeCount != null) { throw new NotImplementedException("top"); } return(Task.FromResult( (IList <TElement>)(from kvp in table where filterExpr.Evaluate(kvp.Value) select ChainTableUtils.CopyEntity <TElement>(kvp.Value)).ToList())); }
public override Task <IQueryStream <TElement> > ExecuteQueryStreamedAsync <TElement>( TableQuery <TElement> query, TableRequestOptions requestOptions = null, OperationContext operationContext = null) { FilterExpression filterExpr = ChainTableUtils.ParseFilterString(query.FilterString); if (query.SelectColumns != null) { throw new NotImplementedException("select"); } if (query.TakeCount != null) { throw new NotImplementedException("top"); } // Easy deterministic implementation compliant with IChainTable2 // API: scan a snapshot. // XXX: At least as an option, this should choose // nondeterministically (using P#) from all behaviors compliant with // the API to verify that callers can handle it. return(Task.FromResult((IQueryStream <TElement>) new QueryStream <TElement>(filterExpr, table.GetEnumerator()))); }
public override Task <IQueryStream <TElement> > ExecuteQueryStreamedAsync <TElement>( TableQuery <TElement> query, TableRequestOptions requestOptions = null, OperationContext operationContext = null) { FilterExpression filterExpr = ChainTableUtils.ParseFilterString(query.FilterString); if (query.SelectColumns != null) { throw new NotImplementedException("select"); } if (query.TakeCount != null) { throw new NotImplementedException("top"); } // Easy deterministic implementation compliant with IChainTable2 // API: scan a snapshot. InMemoryTableWithHistory has the // nondeterministic implementation. // XXX: Nobody calls this any more. We could delete the code and // throw NotImplementedException. return(Task.FromResult((IQueryStream <TElement>) new QueryStream <TElement>(filterExpr, table.GetEnumerator()))); }
// NOTE: Mutates entity internal async Task TryCopyEntityToNewTableAsync(MTableEntity entity, TableRequestOptions requestOptions, OperationContext operationContext) { try { await newTable.ExecuteAsync(TableOperation.Insert(entity), requestOptions, operationContext); } catch (StorageException ex) { if (ex.GetHttpStatusCode() != HttpStatusCode.Conflict) { throw ChainTableUtils.GenerateInternalException(ex); } await monitor.AnnotateLastBackendCallAsync(); return; } await monitor.AnnotateLastBackendCallAsync( spuriousETagChanges : new List <SpuriousETagChange> { new SpuriousETagChange(entity.PartitionKey, entity.RowKey, entity.ETag) }); }
/* * FIXME: This will not work against real Azure table because a batch * can affect up to 100 rows but a filter string only allows 15 * comparisons. The only other thing we can really do in general is * query the whole partition, but that might be slow. We can try an * atomic query of the whole partition (maybe even with a timeout lower * than the default) and if that fails, fall back to breaking the query * into up to 8 partial queries. */ static TableQuery <MTableEntity> GenerateQueryForAffectedRows(TableBatchOperation batch) { string rowKeyFilterString = null; foreach (TableOperation op in batch) { string comparison = TableQuery.GenerateFilterCondition( TableConstants.RowKey, QueryComparisons.Equal, op.GetEntity().RowKey); rowKeyFilterString = (rowKeyFilterString == null) ? comparison : TableQuery.CombineFilters(rowKeyFilterString, TableOperators.Or, comparison); } string filterString = TableQuery.CombineFilters( TableQuery.GenerateFilterCondition( TableConstants.PartitionKey, QueryComparisons.Equal, ChainTableUtils.GetBatchPartitionKey(batch)), TableOperators.And, rowKeyFilterString); return(new TableQuery <MTableEntity> { FilterString = filterString }); }
async Task DoQueryStreamed() { int startRevision = await peekProxy.GetReferenceTableRevisionAsync(); // XXX: Test the filtering? var query = new TableQuery <DynamicTableEntity>(); using (IQueryStream <DynamicTableEntity> stream = await migratingTable.ExecuteQueryStreamedAsync(query)) { PrimaryKey continuationKey = await stream.GetContinuationPrimaryKeyAsync(); await peekProxy.ValidateQueryStreamGapAsync(startRevision, null, continuationKey); do { DynamicTableEntity row = await stream.ReadRowAsync(); PrimaryKey newContinuationKey = await stream.GetContinuationPrimaryKeyAsync(); if (row == null) { PSharpRuntime.Assert(newContinuationKey == null); await peekProxy.ValidateQueryStreamGapAsync(startRevision, continuationKey, null); } else { await peekProxy.ValidateQueryStreamGapAsync(startRevision, continuationKey, row.GetPrimaryKey()); await peekProxy.ValidateQueryStreamRowAsync(startRevision, row); await peekProxy.ValidateQueryStreamGapAsync(startRevision, ChainTableUtils.NextValidPrimaryKeyAfter(row.GetPrimaryKey()), newContinuationKey); } continuationKey = newContinuationKey; } while (continuationKey != null); } }
public override Task <IList <TableResult> > ExecuteMirrorBatchAsync( TableBatchOperation originalBatch, IList <TableResult> originalResponse, TableRequestOptions requestOptions = null, OperationContext operationContext = null) { ChainTableUtils.GetBatchPartitionKey(originalBatch); // For validation; result ignored // Copy the table. Entities are aliased to the original table, so don't mutate them. var tmpTable = new SortedDictionary <PrimaryKey, DynamicTableEntity>(table); int tmpNextEtag = nextEtag; var results = new List <TableResult>(); for (int i = 0; i < originalBatch.Count; i++) { TableOperation op = originalBatch[i]; TableOperationType opType = op.GetOperationType(); ITableEntity passedEntity = op.GetEntity(); PrimaryKey key = passedEntity.GetPrimaryKey(); DynamicTableEntity oldEntity = tmpTable.GetValueOrDefault(key); DynamicTableEntity newEntity = null; HttpStatusCode statusCode = HttpStatusCode.NoContent; if (opType == TableOperationType.Insert) { if (oldEntity != null) { throw ChainTableUtils.GenerateBatchException(HttpStatusCode.Conflict, i); } else { newEntity = ChainTableUtils.CopyEntity <DynamicTableEntity>(passedEntity); statusCode = HttpStatusCode.Created; } } else if (opType == TableOperationType.InsertOrReplace) { newEntity = ChainTableUtils.CopyEntity <DynamicTableEntity>(passedEntity); } else if (opType == TableOperationType.InsertOrMerge) { if (oldEntity == null) { newEntity = ChainTableUtils.CopyEntity <DynamicTableEntity>(passedEntity); } else { newEntity = ChainTableUtils.CopyEntity <DynamicTableEntity>(oldEntity); Merge(newEntity, passedEntity); } } else if (opType == TableOperationType.Delete && passedEntity.ETag == ChainTable2Constants.ETAG_DELETE_IF_EXISTS) { tmpTable.Remove(key); } else if (oldEntity == null) { throw ChainTableUtils.GenerateBatchException(HttpStatusCode.NotFound, i); } else if (string.IsNullOrEmpty(passedEntity.ETag)) { // Enforce this because real Azure table will. // XXX Ideally do this up front. throw new ArgumentException(string.Format("Operation {0} requires an explicit ETag.", i)); } else if (passedEntity.ETag != ChainTable2Constants.ETAG_ANY && oldEntity.ETag != passedEntity.ETag) { throw ChainTableUtils.GenerateBatchException(HttpStatusCode.PreconditionFailed, i); } else if (opType == TableOperationType.Delete) { tmpTable.Remove(key); } else if (opType == TableOperationType.Replace) { newEntity = ChainTableUtils.CopyEntity <DynamicTableEntity>(passedEntity); } else if (opType == TableOperationType.Merge) { newEntity = ChainTableUtils.CopyEntity <DynamicTableEntity>(oldEntity); Merge(newEntity, passedEntity); } else { // IChainTable2 does not allow Retrieve in a batch. throw new NotImplementedException(); } if (newEntity != null) { newEntity.ETag = (originalResponse != null) ? originalResponse[i].Etag : (tmpNextEtag++).ToString(); newEntity.Timestamp = DateTimeOffset.MinValue; // Arbitrary, deterministic tmpTable[key] = newEntity; } results.Add(new TableResult { Result = passedEntity, HttpStatusCode = (int)statusCode, Etag = (newEntity != null) ? newEntity.ETag : null, }); } // If we got here, commit. table = tmpTable; nextEtag = tmpNextEtag; for (int i = 0; i < originalBatch.Count; i++) { if (results[i].Etag != null) // not delete { originalBatch[i].GetEntity().ETag = results[i].Etag; } } return(Task.FromResult((IList <TableResult>)results)); }
internal async Task EnsurePartitionSwitchedAsync(string partitionKey, TableRequestOptions requestOptions, OperationContext operationContext) { var metaQuery = new TableQuery <MTableEntity> { FilterString = ChainTableUtils.GeneratePointRetrievalFilterCondition( new PrimaryKey(partitionKey, ROW_KEY_PARTITION_META)) }; Recheck: MTablePartitionState? state; if (IsBugEnabled(MTableOptionalBug.EnsurePartitionSwitchedFromPopulated)) { state = null; } else { state = (from r in (await oldTable.ExecuteQueryAtomicAsync(metaQuery, requestOptions, operationContext)) select r.partitionState).SingleOrDefault(); await monitor.AnnotateLastBackendCallAsync(); } switch (state) { case null: try { await oldTable.ExecuteAsync(TableOperation.Insert(new MTableEntity { PartitionKey = partitionKey, RowKey = ROW_KEY_PARTITION_META, partitionState = MTablePartitionState.SWITCHED }), requestOptions, operationContext); } catch (StorageException ex) { if (ex.GetHttpStatusCode() != HttpStatusCode.Conflict) { throw ChainTableUtils.GenerateInternalException(ex); } if (!IsBugEnabled(MTableOptionalBug.EnsurePartitionSwitchedFromPopulated)) { await monitor.AnnotateLastBackendCallAsync(); // We could now be in POPULATED or SWITCHED. // XXX: In production, what's more likely? Is it faster // to recheck first or just try the case below? goto Recheck; } } await monitor.AnnotateLastBackendCallAsync(); return; case MTablePartitionState.POPULATED: try { var batch = new TableBatchOperation(); batch.Replace(new MTableEntity { PartitionKey = partitionKey, RowKey = ROW_KEY_PARTITION_META, ETag = ChainTable2Constants.ETAG_ANY, partitionState = MTablePartitionState.SWITCHED }); batch.Delete(new MTableEntity { PartitionKey = partitionKey, RowKey = ROW_KEY_PARTITION_POPULATED_ASSERTION, ETag = ChainTable2Constants.ETAG_ANY, }); await oldTable.ExecuteBatchAsync(batch, requestOptions, operationContext); } catch (ChainTableBatchException ex) { // The only way this can fail (within the semantics) is // if someone else moved the partition to SWITCHED. if (!(ex.FailedOpIndex == 1 && ex.GetHttpStatusCode() == HttpStatusCode.NotFound)) { throw ChainTableUtils.GenerateInternalException(ex); } } await monitor.AnnotateLastBackendCallAsync(); return; case MTablePartitionState.SWITCHED: // Nothing to do return; } }
async Task DoRandomAtomicCalls() { for (int callNum = 0; callNum < MigrationModel.NUM_CALLS_PER_MACHINE; callNum++) { SortedDictionary <PrimaryKey, DynamicTableEntity> dump = await peekProxy.DumpReferenceTableAsync(); if (PSharpRuntime.Nondeterministic()) { // Query var query = new TableQuery <DynamicTableEntity>(); query.FilterString = ChainTableUtils.CombineFilters( TableQuery.GenerateFilterCondition( TableConstants.PartitionKey, QueryComparisons.Equal, MigrationModel.SINGLE_PARTITION_KEY), TableOperators.And, NondeterministicUserPropertyFilterString()); await RunQueryAtomicAsync(query); } else { // Batch write int batchSize = PSharpRuntime.Nondeterministic() ? 2 : 1; var batch = new TableBatchOperation(); var rowKeyChoices = new List <string> { "0", "1", "2", "3", "4", "5" }; for (int opNum = 0; opNum < batchSize; opNum++) { int opTypeNum = PSharpNondeterminism.Choice(7); int rowKeyI = PSharpNondeterminism.Choice(rowKeyChoices.Count); string rowKey = rowKeyChoices[rowKeyI]; rowKeyChoices.RemoveAt(rowKeyI); // Avoid duplicate in same batch var primaryKey = new PrimaryKey(MigrationModel.SINGLE_PARTITION_KEY, rowKey); string eTag = null; if (opTypeNum >= 1 && opTypeNum <= 3) { DynamicTableEntity existingEntity; int etagTypeNum = PSharpNondeterminism.Choice( dump.TryGetValue(primaryKey, out existingEntity) ? 3 : 2); switch (etagTypeNum) { case 0: eTag = ChainTable2Constants.ETAG_ANY; break; case 1: eTag = "wrong"; break; case 2: eTag = existingEntity.ETag; break; } } DynamicTableEntity entity = new DynamicTableEntity { PartitionKey = MigrationModel.SINGLE_PARTITION_KEY, RowKey = rowKey, ETag = eTag, Properties = new Dictionary <string, EntityProperty> { // Give us something to see on merge. Might help with tracing too! { string.Format("{0}_c{1}_o{2}", machineId.ToString(), callNum, opNum), new EntityProperty(true) }, // Property with 50%/50% distribution for use in filters. { "isHappy", new EntityProperty(PSharpRuntime.Nondeterministic()) } } }; switch (opTypeNum) { case 0: batch.Insert(entity); break; case 1: batch.Replace(entity); break; case 2: batch.Merge(entity); break; case 3: batch.Delete(entity); break; case 4: batch.InsertOrReplace(entity); break; case 5: batch.InsertOrMerge(entity); break; case 6: entity.ETag = ChainTable2Constants.ETAG_DELETE_IF_EXISTS; batch.Delete(entity); break; } } await RunBatchAsync(batch); } } }
async Task CleanupAsync(TableRequestOptions requestOptions, OperationContext operationContext) { // Clean up MTable-specific data from new table. // Future: Query only ones with properties we need to clean up. IQueryStream <MTableEntity> newTableStream = await newTable.ExecuteQueryStreamedAsync( new TableQuery <MTableEntity>(), requestOptions, operationContext); MTableEntity newEntity; while ((newEntity = await newTableStream.ReadRowAsync()) != null) { // XXX: Consider factoring out this "query and retry" pattern // into a separate method. Attempt: TableOperation cleanupOp = newEntity.deleted ? TableOperation.Delete(newEntity) : TableOperation.Replace(newEntity.Export <DynamicTableEntity>()); TableResult cleanupResult; try { cleanupResult = await newTable.ExecuteAsync(cleanupOp, requestOptions, operationContext); } catch (StorageException ex) { if (ex.GetHttpStatusCode() == HttpStatusCode.NotFound) { // Someone else deleted it concurrently. Nothing to do. await monitor.AnnotateLastBackendCallAsync(); continue; } else if (ex.GetHttpStatusCode() == HttpStatusCode.PreconditionFailed) { await monitor.AnnotateLastBackendCallAsync(); // Unfortunately we can't assume that anyone who concurrently modifies // the row while the table is in state USE_NEW_HIDE_METADATA will // clean it up, because of InsertOrMerge. (Consider redesign?) // Re-retrieve row. TableResult retrieveResult = await newTable.ExecuteAsync( TableOperation.Retrieve <MTableEntity>(newEntity.PartitionKey, newEntity.RowKey), requestOptions, operationContext); await monitor.AnnotateLastBackendCallAsync(); if ((HttpStatusCode)retrieveResult.HttpStatusCode == HttpStatusCode.NotFound) { continue; } else { newEntity = (MTableEntity)retrieveResult.Result; goto Attempt; } } else { throw ChainTableUtils.GenerateInternalException(ex); } } await monitor.AnnotateLastBackendCallAsync( spuriousETagChanges : cleanupResult.Etag == null?null : new List <SpuriousETagChange> { new SpuriousETagChange(newEntity.PartitionKey, newEntity.RowKey, cleanupResult.Etag) }); } // Delete everything from old table! No one should be modifying it concurrently. IQueryStream <MTableEntity> oldTableStream = await oldTable.ExecuteQueryStreamedAsync( new TableQuery <MTableEntity>(), requestOptions, operationContext); MTableEntity oldEntity; while ((oldEntity = await oldTableStream.ReadRowAsync()) != null) { await oldTable.ExecuteAsync(TableOperation.Delete(oldEntity), requestOptions, operationContext); await monitor.AnnotateLastBackendCallAsync(); } }
async Task ApplyConfigurationAsync(MTableConfiguration newConfig) { using (await LockAsyncBuggable()) { PrimaryKey continuationKey = InternalGetContinuationPrimaryKey(); // ExecuteQueryStreamedAsync validated that mtableQuery has no select or top. TableQuery <MTableEntity> newTableContinuationQuery = (continuationKey == null) ? null : new TableQuery <MTableEntity> { // As in ExecuteQueryAtomicAsync, we have to retrieve all // potential shadowing rows. // XXX: This pays even a bigger penalty for not keeping // conditions on the primary key. But if we were to simply // keep all such conditions, there's a potential to add // more and more continuation filter conditions due to // layering of IChainTable2s, which would lead to some extra // overhead and push us closer to the limit on number of // comparisons. Either try to parse for this or change // ExecuteQueryStreamedAsync to always take a continuation // primary key? FilterString = outer.IsBugEnabled(MTableOptionalBug.QueryStreamedFilterShadowing) ? ChainTableUtils.CombineFilters( ChainTableUtils.GenerateContinuationFilterCondition(continuationKey), TableOperators.And, mtableQuery.FilterString // Could be empty. ) : ChainTableUtils.GenerateContinuationFilterCondition(continuationKey), }; bool justStartedNewStream = false; // Actually, if the query started in state // USE_OLD_HIDE_METADATA, it is API-compliant to continue // returning data from the old table until the migrator starts // deleting it, at which point we switch to the new table. But // some callers may benefit from fresher data, even if it is // never guaranteed, so we go ahead and start the new stream. // XXX: Is this the right decision? if (newConfig.state > TableClientState.USE_OLD_HIDE_METADATA && currentConfig.state <= TableClientState.USE_OLD_HIDE_METADATA && newTableContinuationQuery != null) { newTableStream = await outer.newTable.ExecuteQueryStreamedAsync(newTableContinuationQuery, requestOptions, operationContext); newTableNext = await newTableStream.ReadRowAsync(); justStartedNewStream = true; } if (newConfig.state >= TableClientState.USE_NEW_HIDE_METADATA && currentConfig.state < TableClientState.USE_NEW_HIDE_METADATA) { oldTableStream.Dispose(); oldTableStream = null; oldTableNext = null; // Stop DetermineNextSide from trying to read the old stream. if (!outer.IsBugEnabled(MTableOptionalBug.QueryStreamedBackUpNewStream) && newTableContinuationQuery != null && !justStartedNewStream) { // The new stream could have gotten ahead of the old // stream if rows had not yet been migrated. This // was OK as long as we still planned to read those // rows from the old stream, but now we have to back // up the new stream to where the old stream was. newTableStream.Dispose(); newTableStream = await outer.newTable.ExecuteQueryStreamedAsync(newTableContinuationQuery, requestOptions, operationContext); newTableNext = await newTableStream.ReadRowAsync(); } } if (!outer.IsBugEnabled(MTableOptionalBug.QueryStreamedSaveNewConfig)) { currentConfig = newConfig; } } }
public override async Task <IList <TElement> > ExecuteQueryAtomicAsync <TElement>(TableQuery <TElement> query, TableRequestOptions requestOptions = null, OperationContext operationContext = null) { if (query.SelectColumns != null) { throw new NotImplementedException("select"); } if (query.TakeCount != null) { throw new NotImplementedException("top"); } FilterExpression origFilterExpr = ChainTableUtils.ParseFilterString(query.FilterString); string partitionKey = ChainTableUtils.GetSingleTargetedPartitionKey(origFilterExpr); TableQuery <MTableEntity> mtableQuery = ChainTableUtils.CopyQuery <TElement, MTableEntity>(query); MTableConfiguration config; using (configService.Subscribe(FixedSubscriber <MTableConfiguration> .Instance, out config)) { IEnumerable <MTableEntity> results; if (config.state <= TableClientState.USE_OLD_HIDE_METADATA) { results = await oldTable.ExecuteQueryAtomicAsync(mtableQuery, requestOptions, operationContext); await monitor.AnnotateLastBackendCallAsync(wasLinearizationPoint : true); } else if (config.state >= TableClientState.USE_NEW_WITH_TOMBSTONES) { results = await newTable.ExecuteQueryAtomicAsync(mtableQuery, requestOptions, operationContext); await monitor.AnnotateLastBackendCallAsync(wasLinearizationPoint : true); } else { // Modify the filter to make sure it matches the meta row. if (origFilterExpr is ComparisonExpression) { // filterExpr must be "PartitionKey eq A", which already matches the meta row; nothing to do. } else { // filterExpr must be "(PartitionKey eq A) and (B)". // Rewrite to "(PartitionKey eq A) and ((RowKey eq ROW_KEY_PARTITION_META) or (B))". var boe = (BooleanOperatorExpression)origFilterExpr; mtableQuery.FilterString = TableQuery.CombineFilters(boe.Left.ToFilterString(), TableOperators.And, TableQuery.CombineFilters( TableQuery.GenerateFilterCondition(TableConstants.RowKey, QueryComparisons.Equal, ROW_KEY_PARTITION_META), TableOperators.Or, boe.Right.ToFilterString())); } IList <MTableEntity> oldRows = await oldTable.ExecuteQueryAtomicAsync(mtableQuery, requestOptions, operationContext); MTablePartitionState?state = (from r in oldRows where r.RowKey == ROW_KEY_PARTITION_META select r.partitionState).SingleOrDefault(); IList <MTableEntity> newRows; if (state == MTablePartitionState.SWITCHED) { await monitor.AnnotateLastBackendCallAsync(); // If the filter string includes conditions on user-defined properties, // a row in the old table can be shadowed by a row in the new table // that doesn't satisfy those conditions. To make sure we retrieve all // potential shadowing rows, retrieve the entire partition. // XXX: At a minimum, we should try to keep conditions on the // primary key, but how clever do we want to get with query rewriting? if (!IsBugEnabled(MTableOptionalBug.QueryAtomicFilterShadowing)) { mtableQuery.FilterString = TableQuery.GenerateFilterCondition( TableConstants.PartitionKey, QueryComparisons.Equal, partitionKey); } newRows = await newTable.ExecuteQueryAtomicAsync(mtableQuery, requestOptions, operationContext); await monitor.AnnotateLastBackendCallAsync(wasLinearizationPoint : true); } else { await monitor.AnnotateLastBackendCallAsync(wasLinearizationPoint : true); newRows = new List <MTableEntity>(); } // Merge lists. Hopefully this is pretty clear. Walking the lists // in lockstep might be faster but is a lot more code. var merged = new SortedDictionary <string, MTableEntity>(StringComparer.Ordinal); foreach (MTableEntity ent in oldRows) { merged[ent.RowKey] = ent; } foreach (MTableEntity ent in newRows) { merged[ent.RowKey] = ent; } results = merged.Values; } return((from ent in results where !RowKeyIsInternal(ent.RowKey) && origFilterExpr.Evaluate(ent) && !ent.deleted select ent.Export <TElement>()).ToList()); } }
void TranslateOperationForNewTable( TableOperation op, MTableEntity existingEntity, bool leaveTombstones, ref TableOperation newOp, ref HttpStatusCode?errorCode) { ITableEntity passedEntity = op.GetEntity(); TableOperationType opType = op.GetOperationType(); switch (opType) { case TableOperationType.Insert: if (existingEntity == null) { newOp = TableOperation.Insert(ChainTableUtils.CopyEntity <MTableEntity>(passedEntity)); } else if (existingEntity.deleted) { newOp = TableOperation.Replace(ImportWithIfMatch(passedEntity, existingEntity.ETag)); } else { errorCode = HttpStatusCode.Conflict; } break; case TableOperationType.Replace: if ((errorCode = CheckExistingEntity(passedEntity, existingEntity)) == null) { newOp = TableOperation.Replace(ImportWithIfMatch(passedEntity, existingEntity.ETag)); } break; case TableOperationType.Merge: if ((errorCode = CheckExistingEntity(passedEntity, existingEntity)) == null) { newOp = TableOperation.Merge(ImportWithIfMatch(passedEntity, existingEntity.ETag)); } break; case TableOperationType.Delete: string buggablePartitionKey, buggableRowKey; if (IsBugEnabled(MTableOptionalBug.DeletePrimaryKey)) { buggablePartitionKey = buggableRowKey = null; } else { buggablePartitionKey = passedEntity.PartitionKey; buggableRowKey = passedEntity.RowKey; } if (leaveTombstones) { if (passedEntity.ETag == ChainTable2Constants.ETAG_DELETE_IF_EXISTS) { newOp = TableOperation.InsertOrReplace(new MTableEntity { PartitionKey = buggablePartitionKey, RowKey = buggableRowKey, deleted = true }); } else if ((errorCode = CheckExistingEntity(passedEntity, existingEntity)) == null) { newOp = TableOperation.Replace(new MTableEntity { PartitionKey = buggablePartitionKey, RowKey = buggableRowKey, deleted = true, ETag = existingEntity.ETag }); } } else { if (passedEntity.ETag == ChainTable2Constants.ETAG_DELETE_IF_EXISTS) { if (existingEntity != null) { newOp = TableOperation.Delete(new MTableEntity { PartitionKey = buggablePartitionKey, RowKey = buggableRowKey, // It's OK to delete the entity and return success whether or not // the entity is a tombstone by the time it is actually deleted. ETag = IsBugEnabled(MTableOptionalBug.DeleteNoLeaveTombstonesETag) ? null : ChainTable2Constants.ETAG_ANY }); } // Otherwise generate nothing. // FIXME: This is not linearizable! It can also generate empty batches. } else if ((errorCode = CheckExistingEntity(passedEntity, existingEntity)) == null) { // Another client in USE_NEW_WITH_TOMBSTONES could concurrently replace the // entity with a tombstone, in which case we need to return 404 to the caller, // hence this needs to be conditioned on the existing ETag. newOp = TableOperation.Delete(new MTableEntity { PartitionKey = buggablePartitionKey, RowKey = buggableRowKey, ETag = IsBugEnabled(MTableOptionalBug.DeleteNoLeaveTombstonesETag) ? null : existingEntity.ETag }); } } break; case TableOperationType.InsertOrReplace: newOp = TableOperation.InsertOrReplace(ChainTableUtils.CopyEntity <MTableEntity>(passedEntity)); break; case TableOperationType.InsertOrMerge: newOp = TableOperation.InsertOrMerge(ChainTableUtils.CopyEntity <MTableEntity>(passedEntity)); break; default: throw new NotImplementedException(); } }
async Task DoRandomAtomicCalls() { for (int callNum = 0; callNum < MigrationModel.NUM_CALLS_PER_MACHINE; callNum++) { TableCall originalCall; MirrorTableCall referenceCall; SortedDictionary <PrimaryKey, DynamicTableEntity> dump = await peekProxy.DumpReferenceTableAsync(); if (PSharpRuntime.Nondeterministic()) { // Query // XXX: Test the filtering? var query = new TableQuery <DynamicTableEntity>(); query.FilterString = TableQuery.GenerateFilterCondition( TableConstants.PartitionKey, QueryComparisons.Equal, MigrationModel.SINGLE_PARTITION_KEY); // async/await pair needed to upcast the return value to object. originalCall = async table => await table.ExecuteQueryAtomicAsync(query); referenceCall = async referenceTable => await referenceTable.ExecuteQueryAtomicAsync(query); Console.WriteLine("{0} starting query", machineId); } else { // Batch write int batchSize = PSharpRuntime.Nondeterministic() ? 2 : 1; var batch = new TableBatchOperation(); var rowKeyChoices = new List <string> { "0", "1", "2", "3", "4", "5" }; for (int opNum = 0; opNum < batchSize; opNum++) { int opTypeNum = PSharpNondeterminism.Choice(7); int rowKeyI = PSharpNondeterminism.Choice(rowKeyChoices.Count); string rowKey = rowKeyChoices[rowKeyI]; rowKeyChoices.RemoveAt(rowKeyI); // Avoid duplicate in same batch var primaryKey = new PrimaryKey(MigrationModel.SINGLE_PARTITION_KEY, rowKey); string eTag = null; if (opTypeNum >= 1 && opTypeNum <= 3) { DynamicTableEntity existingEntity; int etagTypeNum = PSharpNondeterminism.Choice( dump.TryGetValue(primaryKey, out existingEntity) ? 3 : 2); switch (etagTypeNum) { case 0: eTag = ChainTable2Constants.ETAG_ANY; break; case 1: eTag = "wrong"; break; case 2: eTag = existingEntity.ETag; break; } } DynamicTableEntity entity = new DynamicTableEntity { PartitionKey = MigrationModel.SINGLE_PARTITION_KEY, RowKey = rowKey, ETag = eTag, Properties = new Dictionary <string, EntityProperty> { // Give us something to see on merge. Might help with tracing too! { string.Format("{0}_c{1}_o{2}", machineId.ToString(), callNum, opNum), new EntityProperty(true) } } }; switch (opTypeNum) { case 0: batch.Insert(entity); break; case 1: batch.Replace(entity); break; case 2: batch.Merge(entity); break; case 3: batch.Delete(entity); break; case 4: batch.InsertOrReplace(entity); break; case 5: batch.InsertOrMerge(entity); break; case 6: entity.ETag = ChainTable2Constants.ETAG_DELETE_IF_EXISTS; batch.Delete(entity); break; } } TableBatchOperation batchCopy = ChainTableUtils.CopyBatch <DynamicTableEntity>(batch); originalCall = async table => await table.ExecuteBatchAsync(batch); referenceCall = async referenceTable => await referenceTable.ExecuteMirrorBatchAsync(batchCopy, successfulBatchResult); Console.WriteLine("{0} starting batch {1}", machineId, batch); } await RunCallAsync(originalCall, referenceCall); Console.WriteLine("{0} table call verified"); } }
async Task <IList <TableResult> > ExecuteBatchOnNewTableAsync(MTableConfiguration config, TableBatchOperation batch, TableRequestOptions requestOptions, OperationContext operationContext) { string partitionKey = ChainTableUtils.GetBatchPartitionKey(batch); await EnsurePartitionSwitchedAsync(partitionKey, requestOptions, operationContext); if (config.state <= TableClientState.PREFER_NEW) { await EnsureAffectedRowsMigratedAsync(batch, requestOptions, operationContext); } Attempt: // Batch on new table. var query = GenerateQueryForAffectedRows(batch); IList <MTableEntity> newRows = await newTable.ExecuteQueryAtomicAsync(query, requestOptions, operationContext); Dictionary <string, MTableEntity> newDict = newRows.ToDictionary(ent => ent.RowKey); // NOTE! At this point, the read has not yet been annotated. It is annotated below. var newBatch = new TableBatchOperation(); var inputToNewTableIndexMapping = new List <int?>(); for (int i = 0; i < batch.Count; i++) { TableOperation op = batch[i]; ITableEntity passedEntity = op.GetEntity(); MTableEntity existingEntity = newDict.GetValueOrDefault(passedEntity.RowKey); TableOperation newOp = null; HttpStatusCode?errorCode = null; TranslateOperationForNewTable( op, existingEntity, config.state <= TableClientState.USE_NEW_WITH_TOMBSTONES, ref newOp, ref errorCode); if (errorCode != null) { Debug.Assert(newOp == null); await monitor.AnnotateLastBackendCallAsync(wasLinearizationPoint : true); throw ChainTableUtils.GenerateBatchException(errorCode.Value, i); } if (newOp != null) { inputToNewTableIndexMapping.Add(newBatch.Count); newBatch.Add(newOp); } else { inputToNewTableIndexMapping.Add(null); } } await monitor.AnnotateLastBackendCallAsync(); IList <TableResult> newResults; try { newResults = await newTable.ExecuteBatchAsync(newBatch, requestOptions, operationContext); } catch (ChainTableBatchException) { // XXX: Try to distinguish expected concurrency exceptions from unexpected exceptions? await monitor.AnnotateLastBackendCallAsync(); goto Attempt; } // We made it! var results = new List <TableResult>(); for (int i = 0; i < batch.Count; i++) { ITableEntity passedEntity = batch[i].GetEntity(); int? newTableIndex = inputToNewTableIndexMapping[i]; string newETag = (IsBugEnabled(MTableOptionalBug.TombstoneOutputETag) ? newTableIndex != null : batch[i].GetOperationType() == TableOperationType.Delete) ? null : newResults[newTableIndex.Value].Etag; if (newETag != null) { passedEntity.ETag = newETag; } results.Add(new TableResult { HttpStatusCode = (int)( (batch[i].GetOperationType() == TableOperationType.Insert) ? HttpStatusCode.Created : HttpStatusCode.NoContent), Etag = newETag, Result = passedEntity, }); } await monitor.AnnotateLastBackendCallAsync(wasLinearizationPoint : true, successfulBatchResult : results); return(results); }
async Task CleanupAsync(TableRequestOptions requestOptions, OperationContext operationContext) { string previousPartitionKey = null; await WalkTableInParallel(newTable, requestOptions, operationContext, async (newEntity) => { if (newEntity.PartitionKey != previousPartitionKey) { if (previousPartitionKey != null) { Console.WriteLine("Cleaned Partition: {0}", previousPartitionKey); } previousPartitionKey = newEntity.PartitionKey; } // XXX: Consider factoring out this "query and retry" pattern // into a separate method. Attempt: TableOperation cleanupOp = newEntity.deleted ? TableOperation.Delete(newEntity) : TableOperation.Replace(newEntity.Export <DynamicTableEntity>()); TableResult cleanupResult; StorageException ex; try { cleanupResult = await newTable.ExecuteAsync(cleanupOp, requestOptions, operationContext); ex = null; } catch (StorageException exToHandle) { cleanupResult = null; ex = exToHandle; } if (ex != null) { if (ex.GetHttpStatusCode() == HttpStatusCode.NotFound) { // Someone else deleted it concurrently. Nothing to do. //await monitor.AnnotateLastBackendCallAsync(); return((IQueryStream <MTableEntity>)null); } else if (ex.GetHttpStatusCode() == HttpStatusCode.PreconditionFailed) { //await monitor.AnnotateLastBackendCallAsync(); // Unfortunately we can't assume that anyone who concurrently modifies // the row while the table is in state USE_NEW_HIDE_METADATA will // clean it up, because of InsertOrMerge. (Consider redesign?) // Re-retrieve row. TableResult retrieveResult = await newTable.ExecuteAsync( TableOperation.Retrieve <MTableEntity>(newEntity.PartitionKey, newEntity.RowKey), requestOptions, operationContext); //await monitor.AnnotateLastBackendCallAsync(); if ((HttpStatusCode)retrieveResult.HttpStatusCode == HttpStatusCode.NotFound) { return(null); } else { newEntity = (MTableEntity)retrieveResult.Result; goto Attempt; } } else { throw ChainTableUtils.GenerateInternalException(ex); } } else { await monitor.AnnotateLastBackendCallAsync( spuriousETagChanges: cleanupResult.Etag == null ? null : new List <SpuriousETagChange> { new SpuriousETagChange(newEntity.PartitionKey, newEntity.RowKey, cleanupResult.Etag) }); } return((IQueryStream <MTableEntity>)null); }); await WalkTableInParallel(oldTable, requestOptions, operationContext, async (oldEntity) => { await oldTable.ExecuteAsync(TableOperation.Delete(oldEntity), requestOptions, operationContext); await monitor.AnnotateLastBackendCallAsync(); return((IQueryStream <MTableEntity>)null); }); }