MTableEntity ImportWithIfMatch(ITableEntity passedEntity, string ifMatch)
{
MTableEntity newEntity = ChainTableUtils.CopyEntity <MTableEntity>(passedEntity);
newEntity.ETag = ifMatch;
return(newEntity);
}
private void Merge(DynamicTableEntity to, ITableEntity from)
{
foreach (KeyValuePair <string, EntityProperty> kvp in from.WriteEntity(null))
{
to.Properties[kvp.Key] = ChainTableUtils.CopyProperty(kvp.Value);
}
}
async Task EnsureAffectedRowsMigratedAsync(TableBatchOperation batch, TableRequestOptions requestOptions, OperationContext operationContext)
{
string partitionKey = ChainTableUtils.GetBatchPartitionKey(batch);
var query = GenerateQueryForAffectedRows(batch);
IList <MTableEntity> oldRows = await oldTable.ExecuteQueryAtomicAsync(query, requestOptions, operationContext);
await monitor.AnnotateLastBackendCallAsync();
IList <MTableEntity> newRows = await newTable.ExecuteQueryAtomicAsync(query, requestOptions, operationContext);
await monitor.AnnotateLastBackendCallAsync();
Dictionary <string, MTableEntity> oldDict = oldRows.ToDictionary(ent => ent.RowKey);
Dictionary <string, MTableEntity> newDict = newRows.ToDictionary(ent => ent.RowKey);
// Migrate any affected rows not already migrated.
foreach (TableOperation op in batch)
{
string targetedRowKey = op.GetEntity().RowKey;
MTableEntity oldEntity;
if (oldDict.TryGetValue(targetedRowKey, out oldEntity) && !newDict.ContainsKey(targetedRowKey))
{
await TryCopyEntityToNewTableAsync(oldEntity, requestOptions, operationContext);
}
}
}
// Marks the partition populated unless it is already switched.
async Task TryMarkPartitionPopulatedAsync(string partitionKey, TableRequestOptions requestOptions, OperationContext operationContext)
{
var markPopulatedBatch = new TableBatchOperation();
markPopulatedBatch.Insert(new MTableEntity
{
PartitionKey = partitionKey,
RowKey = ROW_KEY_PARTITION_META,
partitionState = MTablePartitionState.POPULATED
});
markPopulatedBatch.Insert(new DynamicTableEntity
{
PartitionKey = partitionKey,
RowKey = ROW_KEY_PARTITION_POPULATED_ASSERTION
});
try
{
await oldTable.ExecuteBatchAsync(markPopulatedBatch, requestOptions, operationContext);
}
// XXX: Optimization opportunity: if we swap the order of the
// inserts, we can tell here if the partition is already switched.
catch (StorageException ex) {
if (ex.GetHttpStatusCode() != HttpStatusCode.Conflict)
{
throw ChainTableUtils.GenerateInternalException(ex);
}
}
await monitor.AnnotateLastBackendCallAsync();
}
internal QueryStream(MigratingTable outer, TableQuery <TElement> query, TableRequestOptions requestOptions, OperationContext operationContext)
{
this.outer = outer;
origFilterExpr = ChainTableUtils.ParseFilterString(query.FilterString);
mtableQuery = ChainTableUtils.CopyQuery <TElement, MTableEntity>(query);
this.requestOptions = requestOptions;
this.operationContext = operationContext;
}
public SortedDictionary <PrimaryKey, DynamicTableEntity> Dump()
{
var dump = new SortedDictionary <PrimaryKey, DynamicTableEntity>();
foreach (KeyValuePair <PrimaryKey, DynamicTableEntity> kvp in table)
{
dump.Add(kvp.Key, ChainTableUtils.CopyEntity <DynamicTableEntity>(kvp.Value));
}
return(dump);
}
internal async Task RunBatchAsync(TableBatchOperation batch)
{
TableBatchOperation batchCopy = ChainTableUtils.CopyBatch <DynamicTableEntity>(batch);
TableCall originalCall = async table => await table.ExecuteBatchAsync(batch);
MirrorTableCall referenceCall = async referenceTable => await referenceTable.ExecuteMirrorBatchAsync(batchCopy, successfulBatchResult);
Console.WriteLine("{0} starting batch: {1}", machineId, BetterComparer.ToString(batch));
await RunCallAsync(originalCall, referenceCall);
}
public Task <TElement> ReadRowAsync()
{
CheckDisposed();
if (enumerator == null)
{
return(Task.FromResult(default(TElement)));
}
TElement entity = ChainTableUtils.CopyEntity <TElement>(enumerator.Current.Value);
MoveNext();
return(Task.FromResult(entity));
}
public override Task <IQueryStream <TElement> > ExecuteQueryStreamedAsync <TElement>(TableQuery <TElement> query, TableRequestOptions requestOptions = null, OperationContext operationContext = null)
{
FilterExpression filterExpr = ChainTableUtils.ParseFilterString(query.FilterString);
if (query.SelectColumns != null)
{
throw new NotImplementedException("select");
}
if (query.TakeCount != null)
{
throw new NotImplementedException("top");
}
return(Task.FromResult((IQueryStream <TElement>)
new NondeterministicQueryStream <TElement>(this, CurrentRevision, filterExpr)));
}
public Task <TElement> ReadRowAsync()
{
if (continuationKey == null)
{
return(Task.FromResult(default(TElement)));
}
List <DynamicTableEntity> possibleRows = outer.GetValidStreamReadRows(startRevision, filterExpr, continuationKey);
int choiceIndex = PSharpNondeterminism.Choice(possibleRows.Count);
DynamicTableEntity choice = possibleRows[choiceIndex];
Console.WriteLine("NondeterministicQueryStream: possibleRows {0}, choiceIndex {1}",
BetterComparer.ToString(possibleRows), choiceIndex);
continuationKey = (choice == null) ? null : ChainTableUtils.NextValidPrimaryKeyAfter(choice.GetPrimaryKey());
return(Task.FromResult((choice == null) ? default(TElement) : ChainTableUtils.CopyEntity <TElement>(choice)));
}
internal TElement Export <TElement>() where TElement : ITableEntity, new()
{
var ent2 = new TElement()
{
PartitionKey = PartitionKey,
RowKey = RowKey,
ETag = ETag,
Timestamp = Timestamp,
};
// This copy probably isn't needed in the common case,
// but I'd rather not even have to think about it.
ent2.ReadEntity(ChainTableUtils.CopyPropertyDict(userProperties), null);
return(ent2);
}
internal async Task RunQueryStreamedAsync(TableQuery <DynamicTableEntity> query)
{
int startRevision = await peekProxy.GetReferenceTableRevisionAsync();
FilterExpression filterExpr = ChainTableUtils.ParseFilterString(query.FilterString);
Console.WriteLine("{0} starting streaming query: {1}", machineId, query);
using (IQueryStream <DynamicTableEntity> stream = await migratingTable.ExecuteQueryStreamedAsync(query))
{
PrimaryKey lastKey = ChainTableUtils.FirstValidPrimaryKey;
for (;;)
{
PrimaryKey returnedContinuationKey = await stream.GetContinuationPrimaryKeyAsync();
PSharpRuntime.Assert(returnedContinuationKey == null || returnedContinuationKey.CompareTo(lastKey) >= 0,
"{0}: query stream continuation key is {1}, expected >= {2}",
machineId, returnedContinuationKey, lastKey);
DynamicTableEntity row = await stream.ReadRowAsync(); // may be null, meaning end of stream
// Must be after ReadRowAsync, otherwise additional rows could become valid
// due to a mutation between GetValidStreamReadRows and ReadRowAsync and
// we would falsely report a bug if ReadRowAsync returns one of those rows.
List <DynamicTableEntity> validRows = await peekProxy.GetValidStreamReadRows(startRevision, filterExpr, lastKey);
// Three cheers for automatic use of covariance in overload resolution!
PSharpRuntime.Assert(validRows.Contains(row, BetterComparer.Instance),
"{0} query stream returned {1}, which is not one of the valid rows: {2}",
machineId, BetterComparer.ToString(row), BetterComparer.ToString(validRows));
Console.WriteLine("{0} query stream returned row {1}, which is valid", machineId, BetterComparer.ToString(row));
if (row == null)
{
// Any returnedContinuationKey (including null) is less or equal to a row of null.
break;
}
else
{
PSharpRuntime.Assert(returnedContinuationKey != null && returnedContinuationKey.CompareTo(row.GetPrimaryKey()) <= 0,
"{0}: query stream continuation key is {1}, expected <= {2}",
machineId, returnedContinuationKey, row.GetPrimaryKey());
lastKey = ChainTableUtils.NextValidPrimaryKeyAfter(row.GetPrimaryKey());
}
}
}
Console.WriteLine("{0} finished streaming query", machineId);
}
async Task <IList <TableResult> > AttemptBatchOnOldTableAsync(TableBatchOperation batch, TableRequestOptions requestOptions, OperationContext operationContext)
{
string partitionKey = ChainTableUtils.GetBatchPartitionKey(batch);
await TryMarkPartitionPopulatedAsync(partitionKey, requestOptions, operationContext);
var oldBatch = new TableBatchOperation();
oldBatch.Merge(new DynamicTableEntity {
PartitionKey = partitionKey,
RowKey = ROW_KEY_PARTITION_POPULATED_ASSERTION,
ETag = ChainTable2Constants.ETAG_ANY,
});
// No AddRange? :(
foreach (TableOperation op in batch)
{
oldBatch.Add(op);
}
IList <TableResult> oldResults;
try
{
oldResults = await oldTable.ExecuteBatchAsync(oldBatch, requestOptions, operationContext);
}
catch (ChainTableBatchException ex)
{
if (ex.FailedOpIndex == 0)
{
// This must mean the partition is switched.
await monitor.AnnotateLastBackendCallAsync();
return(null);
}
else
{
await monitor.AnnotateLastBackendCallAsync(wasLinearizationPoint : true);
throw ChainTableUtils.GenerateBatchException(ex.GetHttpStatusCode(), ex.FailedOpIndex - 1);
}
}
oldResults.RemoveAt(0);
await monitor.AnnotateLastBackendCallAsync(wasLinearizationPoint : true, successfulBatchResult : oldResults);
return(oldResults);
}
public override Task <IList <TElement> > ExecuteQueryAtomicAsync <TElement>(
TableQuery <TElement> query, TableRequestOptions requestOptions = null, OperationContext operationContext = null)
{
FilterExpression filterExpr = ChainTableUtils.ParseFilterString(query.FilterString);
ChainTableUtils.GetSingleTargetedPartitionKey(filterExpr); // validation, ignore result
if (query.SelectColumns != null)
{
throw new NotImplementedException("select");
}
if (query.TakeCount != null)
{
throw new NotImplementedException("top");
}
return(Task.FromResult(
(IList <TElement>)(from kvp in table where filterExpr.Evaluate(kvp.Value)
select ChainTableUtils.CopyEntity <TElement>(kvp.Value)).ToList()));
}
public override Task <IQueryStream <TElement> > ExecuteQueryStreamedAsync <TElement>(
TableQuery <TElement> query, TableRequestOptions requestOptions = null, OperationContext operationContext = null)
{
FilterExpression filterExpr = ChainTableUtils.ParseFilterString(query.FilterString);
if (query.SelectColumns != null)
{
throw new NotImplementedException("select");
}
if (query.TakeCount != null)
{
throw new NotImplementedException("top");
}
// Easy deterministic implementation compliant with IChainTable2
// API: scan a snapshot.
// XXX: At least as an option, this should choose
// nondeterministically (using P#) from all behaviors compliant with
// the API to verify that callers can handle it.
return(Task.FromResult((IQueryStream <TElement>) new QueryStream <TElement>(filterExpr, table.GetEnumerator())));
}
public override Task <IQueryStream <TElement> > ExecuteQueryStreamedAsync <TElement>(
TableQuery <TElement> query, TableRequestOptions requestOptions = null, OperationContext operationContext = null)
{
FilterExpression filterExpr = ChainTableUtils.ParseFilterString(query.FilterString);
if (query.SelectColumns != null)
{
throw new NotImplementedException("select");
}
if (query.TakeCount != null)
{
throw new NotImplementedException("top");
}
// Easy deterministic implementation compliant with IChainTable2
// API: scan a snapshot. InMemoryTableWithHistory has the
// nondeterministic implementation.
// XXX: Nobody calls this any more. We could delete the code and
// throw NotImplementedException.
return(Task.FromResult((IQueryStream <TElement>) new QueryStream <TElement>(filterExpr, table.GetEnumerator())));
}
// NOTE: Mutates entity
internal async Task TryCopyEntityToNewTableAsync(MTableEntity entity, TableRequestOptions requestOptions, OperationContext operationContext)
{
try
{
await newTable.ExecuteAsync(TableOperation.Insert(entity), requestOptions, operationContext);
}
catch (StorageException ex)
{
if (ex.GetHttpStatusCode() != HttpStatusCode.Conflict)
{
throw ChainTableUtils.GenerateInternalException(ex);
}
await monitor.AnnotateLastBackendCallAsync();
return;
}
await monitor.AnnotateLastBackendCallAsync(
spuriousETagChanges : new List <SpuriousETagChange> {
new SpuriousETagChange(entity.PartitionKey, entity.RowKey, entity.ETag)
});
}
/*
* FIXME: This will not work against real Azure table because a batch
* can affect up to 100 rows but a filter string only allows 15
* comparisons. The only other thing we can really do in general is
* query the whole partition, but that might be slow. We can try an
* atomic query of the whole partition (maybe even with a timeout lower
* than the default) and if that fails, fall back to breaking the query
* into up to 8 partial queries.
*/
static TableQuery <MTableEntity> GenerateQueryForAffectedRows(TableBatchOperation batch)
{
string rowKeyFilterString = null;
foreach (TableOperation op in batch)
{
string comparison = TableQuery.GenerateFilterCondition(
TableConstants.RowKey, QueryComparisons.Equal, op.GetEntity().RowKey);
rowKeyFilterString = (rowKeyFilterString == null) ? comparison
: TableQuery.CombineFilters(rowKeyFilterString, TableOperators.Or, comparison);
}
string filterString = TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition(
TableConstants.PartitionKey, QueryComparisons.Equal, ChainTableUtils.GetBatchPartitionKey(batch)),
TableOperators.And,
rowKeyFilterString);
return(new TableQuery <MTableEntity> {
FilterString = filterString
});
}
async Task DoQueryStreamed()
{
int startRevision = await peekProxy.GetReferenceTableRevisionAsync();
// XXX: Test the filtering?
var query = new TableQuery <DynamicTableEntity>();
using (IQueryStream <DynamicTableEntity> stream = await migratingTable.ExecuteQueryStreamedAsync(query))
{
PrimaryKey continuationKey = await stream.GetContinuationPrimaryKeyAsync();
await peekProxy.ValidateQueryStreamGapAsync(startRevision, null, continuationKey);
do
{
DynamicTableEntity row = await stream.ReadRowAsync();
PrimaryKey newContinuationKey = await stream.GetContinuationPrimaryKeyAsync();
if (row == null)
{
PSharpRuntime.Assert(newContinuationKey == null);
await peekProxy.ValidateQueryStreamGapAsync(startRevision, continuationKey, null);
}
else
{
await peekProxy.ValidateQueryStreamGapAsync(startRevision, continuationKey, row.GetPrimaryKey());
await peekProxy.ValidateQueryStreamRowAsync(startRevision, row);
await peekProxy.ValidateQueryStreamGapAsync(startRevision,
ChainTableUtils.NextValidPrimaryKeyAfter(row.GetPrimaryKey()), newContinuationKey);
}
continuationKey = newContinuationKey;
} while (continuationKey != null);
}
}
public override Task <IList <TableResult> > ExecuteMirrorBatchAsync(
TableBatchOperation originalBatch, IList <TableResult> originalResponse,
TableRequestOptions requestOptions = null, OperationContext operationContext = null)
{
ChainTableUtils.GetBatchPartitionKey(originalBatch); // For validation; result ignored
// Copy the table. Entities are aliased to the original table, so don't mutate them.
var tmpTable = new SortedDictionary <PrimaryKey, DynamicTableEntity>(table);
int tmpNextEtag = nextEtag;
var results = new List <TableResult>();
for (int i = 0; i < originalBatch.Count; i++)
{
TableOperation op = originalBatch[i];
TableOperationType opType = op.GetOperationType();
ITableEntity passedEntity = op.GetEntity();
PrimaryKey key = passedEntity.GetPrimaryKey();
DynamicTableEntity oldEntity = tmpTable.GetValueOrDefault(key);
DynamicTableEntity newEntity = null;
HttpStatusCode statusCode = HttpStatusCode.NoContent;
if (opType == TableOperationType.Insert)
{
if (oldEntity != null)
{
throw ChainTableUtils.GenerateBatchException(HttpStatusCode.Conflict, i);
}
else
{
newEntity = ChainTableUtils.CopyEntity <DynamicTableEntity>(passedEntity);
statusCode = HttpStatusCode.Created;
}
}
else if (opType == TableOperationType.InsertOrReplace)
{
newEntity = ChainTableUtils.CopyEntity <DynamicTableEntity>(passedEntity);
}
else if (opType == TableOperationType.InsertOrMerge)
{
if (oldEntity == null)
{
newEntity = ChainTableUtils.CopyEntity <DynamicTableEntity>(passedEntity);
}
else
{
newEntity = ChainTableUtils.CopyEntity <DynamicTableEntity>(oldEntity);
Merge(newEntity, passedEntity);
}
}
else if (opType == TableOperationType.Delete &&
passedEntity.ETag == ChainTable2Constants.ETAG_DELETE_IF_EXISTS)
{
tmpTable.Remove(key);
}
else if (oldEntity == null)
{
throw ChainTableUtils.GenerateBatchException(HttpStatusCode.NotFound, i);
}
else if (string.IsNullOrEmpty(passedEntity.ETag))
{
// Enforce this because real Azure table will.
// XXX Ideally do this up front.
throw new ArgumentException(string.Format("Operation {0} requires an explicit ETag.", i));
}
else if (passedEntity.ETag != ChainTable2Constants.ETAG_ANY &&
oldEntity.ETag != passedEntity.ETag)
{
throw ChainTableUtils.GenerateBatchException(HttpStatusCode.PreconditionFailed, i);
}
else if (opType == TableOperationType.Delete)
{
tmpTable.Remove(key);
}
else if (opType == TableOperationType.Replace)
{
newEntity = ChainTableUtils.CopyEntity <DynamicTableEntity>(passedEntity);
}
else if (opType == TableOperationType.Merge)
{
newEntity = ChainTableUtils.CopyEntity <DynamicTableEntity>(oldEntity);
Merge(newEntity, passedEntity);
}
else
{
// IChainTable2 does not allow Retrieve in a batch.
throw new NotImplementedException();
}
if (newEntity != null)
{
newEntity.ETag = (originalResponse != null) ? originalResponse[i].Etag : (tmpNextEtag++).ToString();
newEntity.Timestamp = DateTimeOffset.MinValue; // Arbitrary, deterministic
tmpTable[key] = newEntity;
}
results.Add(new TableResult {
Result = passedEntity,
HttpStatusCode = (int)statusCode,
Etag = (newEntity != null) ? newEntity.ETag : null,
});
}
// If we got here, commit.
table = tmpTable;
nextEtag = tmpNextEtag;
for (int i = 0; i < originalBatch.Count; i++)
{
if (results[i].Etag != null) // not delete
{
originalBatch[i].GetEntity().ETag = results[i].Etag;
}
}
return(Task.FromResult((IList <TableResult>)results));
}
internal async Task EnsurePartitionSwitchedAsync(string partitionKey, TableRequestOptions requestOptions, OperationContext operationContext)
{
var metaQuery = new TableQuery <MTableEntity>
{
FilterString = ChainTableUtils.GeneratePointRetrievalFilterCondition(
new PrimaryKey(partitionKey, ROW_KEY_PARTITION_META))
};
Recheck:
MTablePartitionState? state;
if (IsBugEnabled(MTableOptionalBug.EnsurePartitionSwitchedFromPopulated))
{
state = null;
}
else
{
state =
(from r in (await oldTable.ExecuteQueryAtomicAsync(metaQuery, requestOptions, operationContext))
select r.partitionState).SingleOrDefault();
await monitor.AnnotateLastBackendCallAsync();
}
switch (state)
{
case null:
try
{
await oldTable.ExecuteAsync(TableOperation.Insert(new MTableEntity
{
PartitionKey = partitionKey,
RowKey = ROW_KEY_PARTITION_META,
partitionState = MTablePartitionState.SWITCHED
}), requestOptions, operationContext);
}
catch (StorageException ex)
{
if (ex.GetHttpStatusCode() != HttpStatusCode.Conflict)
{
throw ChainTableUtils.GenerateInternalException(ex);
}
if (!IsBugEnabled(MTableOptionalBug.EnsurePartitionSwitchedFromPopulated))
{
await monitor.AnnotateLastBackendCallAsync();
// We could now be in POPULATED or SWITCHED.
// XXX: In production, what's more likely? Is it faster
// to recheck first or just try the case below?
goto Recheck;
}
}
await monitor.AnnotateLastBackendCallAsync();
return;
case MTablePartitionState.POPULATED:
try
{
var batch = new TableBatchOperation();
batch.Replace(new MTableEntity
{
PartitionKey = partitionKey,
RowKey = ROW_KEY_PARTITION_META,
ETag = ChainTable2Constants.ETAG_ANY,
partitionState = MTablePartitionState.SWITCHED
});
batch.Delete(new MTableEntity
{
PartitionKey = partitionKey,
RowKey = ROW_KEY_PARTITION_POPULATED_ASSERTION,
ETag = ChainTable2Constants.ETAG_ANY,
});
await oldTable.ExecuteBatchAsync(batch, requestOptions, operationContext);
}
catch (ChainTableBatchException ex)
{
// The only way this can fail (within the semantics) is
// if someone else moved the partition to SWITCHED.
if (!(ex.FailedOpIndex == 1 && ex.GetHttpStatusCode() == HttpStatusCode.NotFound))
{
throw ChainTableUtils.GenerateInternalException(ex);
}
}
await monitor.AnnotateLastBackendCallAsync();
return;
case MTablePartitionState.SWITCHED:
// Nothing to do
return;
}
}
async Task DoRandomAtomicCalls()
{
for (int callNum = 0; callNum < MigrationModel.NUM_CALLS_PER_MACHINE; callNum++)
{
SortedDictionary <PrimaryKey, DynamicTableEntity> dump = await peekProxy.DumpReferenceTableAsync();
if (PSharpRuntime.Nondeterministic())
{
// Query
var query = new TableQuery <DynamicTableEntity>();
query.FilterString = ChainTableUtils.CombineFilters(
TableQuery.GenerateFilterCondition(
TableConstants.PartitionKey, QueryComparisons.Equal, MigrationModel.SINGLE_PARTITION_KEY),
TableOperators.And,
NondeterministicUserPropertyFilterString());
await RunQueryAtomicAsync(query);
}
else
{
// Batch write
int batchSize = PSharpRuntime.Nondeterministic() ? 2 : 1;
var batch = new TableBatchOperation();
var rowKeyChoices = new List <string> {
"0", "1", "2", "3", "4", "5"
};
for (int opNum = 0; opNum < batchSize; opNum++)
{
int opTypeNum = PSharpNondeterminism.Choice(7);
int rowKeyI = PSharpNondeterminism.Choice(rowKeyChoices.Count);
string rowKey = rowKeyChoices[rowKeyI];
rowKeyChoices.RemoveAt(rowKeyI); // Avoid duplicate in same batch
var primaryKey = new PrimaryKey(MigrationModel.SINGLE_PARTITION_KEY, rowKey);
string eTag = null;
if (opTypeNum >= 1 && opTypeNum <= 3)
{
DynamicTableEntity existingEntity;
int etagTypeNum = PSharpNondeterminism.Choice(
dump.TryGetValue(primaryKey, out existingEntity) ? 3 : 2);
switch (etagTypeNum)
{
case 0: eTag = ChainTable2Constants.ETAG_ANY; break;
case 1: eTag = "wrong"; break;
case 2: eTag = existingEntity.ETag; break;
}
}
DynamicTableEntity entity = new DynamicTableEntity
{
PartitionKey = MigrationModel.SINGLE_PARTITION_KEY,
RowKey = rowKey,
ETag = eTag,
Properties = new Dictionary <string, EntityProperty> {
// Give us something to see on merge. Might help with tracing too!
{ string.Format("{0}_c{1}_o{2}", machineId.ToString(), callNum, opNum),
new EntityProperty(true) },
// Property with 50%/50% distribution for use in filters.
{ "isHappy", new EntityProperty(PSharpRuntime.Nondeterministic()) }
}
};
switch (opTypeNum)
{
case 0: batch.Insert(entity); break;
case 1: batch.Replace(entity); break;
case 2: batch.Merge(entity); break;
case 3: batch.Delete(entity); break;
case 4: batch.InsertOrReplace(entity); break;
case 5: batch.InsertOrMerge(entity); break;
case 6:
entity.ETag = ChainTable2Constants.ETAG_DELETE_IF_EXISTS;
batch.Delete(entity); break;
}
}
await RunBatchAsync(batch);
}
}
}
async Task CleanupAsync(TableRequestOptions requestOptions, OperationContext operationContext)
{
// Clean up MTable-specific data from new table.
// Future: Query only ones with properties we need to clean up.
IQueryStream <MTableEntity> newTableStream = await newTable.ExecuteQueryStreamedAsync(
new TableQuery <MTableEntity>(), requestOptions, operationContext);
MTableEntity newEntity;
while ((newEntity = await newTableStream.ReadRowAsync()) != null)
{
// XXX: Consider factoring out this "query and retry" pattern
// into a separate method.
Attempt:
TableOperation cleanupOp = newEntity.deleted ? TableOperation.Delete(newEntity)
: TableOperation.Replace(newEntity.Export <DynamicTableEntity>());
TableResult cleanupResult;
try
{
cleanupResult = await newTable.ExecuteAsync(cleanupOp, requestOptions, operationContext);
}
catch (StorageException ex)
{
if (ex.GetHttpStatusCode() == HttpStatusCode.NotFound)
{
// Someone else deleted it concurrently. Nothing to do.
await monitor.AnnotateLastBackendCallAsync();
continue;
}
else if (ex.GetHttpStatusCode() == HttpStatusCode.PreconditionFailed)
{
await monitor.AnnotateLastBackendCallAsync();
// Unfortunately we can't assume that anyone who concurrently modifies
// the row while the table is in state USE_NEW_HIDE_METADATA will
// clean it up, because of InsertOrMerge. (Consider redesign?)
// Re-retrieve row.
TableResult retrieveResult = await newTable.ExecuteAsync(
TableOperation.Retrieve <MTableEntity>(newEntity.PartitionKey, newEntity.RowKey),
requestOptions, operationContext);
await monitor.AnnotateLastBackendCallAsync();
if ((HttpStatusCode)retrieveResult.HttpStatusCode == HttpStatusCode.NotFound)
{
continue;
}
else
{
newEntity = (MTableEntity)retrieveResult.Result;
goto Attempt;
}
}
else
{
throw ChainTableUtils.GenerateInternalException(ex);
}
}
await monitor.AnnotateLastBackendCallAsync(
spuriousETagChanges :
cleanupResult.Etag == null?null : new List <SpuriousETagChange> {
new SpuriousETagChange(newEntity.PartitionKey, newEntity.RowKey, cleanupResult.Etag)
});
}
// Delete everything from old table! No one should be modifying it concurrently.
IQueryStream <MTableEntity> oldTableStream = await oldTable.ExecuteQueryStreamedAsync(
new TableQuery <MTableEntity>(), requestOptions, operationContext);
MTableEntity oldEntity;
while ((oldEntity = await oldTableStream.ReadRowAsync()) != null)
{
await oldTable.ExecuteAsync(TableOperation.Delete(oldEntity), requestOptions, operationContext);
await monitor.AnnotateLastBackendCallAsync();
}
}
async Task ApplyConfigurationAsync(MTableConfiguration newConfig)
{
using (await LockAsyncBuggable())
{
PrimaryKey continuationKey = InternalGetContinuationPrimaryKey();
// ExecuteQueryStreamedAsync validated that mtableQuery has no select or top.
TableQuery <MTableEntity> newTableContinuationQuery =
(continuationKey == null) ? null : new TableQuery <MTableEntity>
{
// As in ExecuteQueryAtomicAsync, we have to retrieve all
// potential shadowing rows.
// XXX: This pays even a bigger penalty for not keeping
// conditions on the primary key. But if we were to simply
// keep all such conditions, there's a potential to add
// more and more continuation filter conditions due to
// layering of IChainTable2s, which would lead to some extra
// overhead and push us closer to the limit on number of
// comparisons. Either try to parse for this or change
// ExecuteQueryStreamedAsync to always take a continuation
// primary key?
FilterString =
outer.IsBugEnabled(MTableOptionalBug.QueryStreamedFilterShadowing)
? ChainTableUtils.CombineFilters(
ChainTableUtils.GenerateContinuationFilterCondition(continuationKey),
TableOperators.And,
mtableQuery.FilterString // Could be empty.
)
: ChainTableUtils.GenerateContinuationFilterCondition(continuationKey),
};
bool justStartedNewStream = false;
// Actually, if the query started in state
// USE_OLD_HIDE_METADATA, it is API-compliant to continue
// returning data from the old table until the migrator starts
// deleting it, at which point we switch to the new table. But
// some callers may benefit from fresher data, even if it is
// never guaranteed, so we go ahead and start the new stream.
// XXX: Is this the right decision?
if (newConfig.state > TableClientState.USE_OLD_HIDE_METADATA &&
currentConfig.state <= TableClientState.USE_OLD_HIDE_METADATA &&
newTableContinuationQuery != null)
{
newTableStream = await outer.newTable.ExecuteQueryStreamedAsync(newTableContinuationQuery, requestOptions, operationContext);
newTableNext = await newTableStream.ReadRowAsync();
justStartedNewStream = true;
}
if (newConfig.state >= TableClientState.USE_NEW_HIDE_METADATA &&
currentConfig.state < TableClientState.USE_NEW_HIDE_METADATA)
{
oldTableStream.Dispose();
oldTableStream = null;
oldTableNext = null; // Stop DetermineNextSide from trying to read the old stream.
if (!outer.IsBugEnabled(MTableOptionalBug.QueryStreamedBackUpNewStream) &&
newTableContinuationQuery != null && !justStartedNewStream)
{
// The new stream could have gotten ahead of the old
// stream if rows had not yet been migrated. This
// was OK as long as we still planned to read those
// rows from the old stream, but now we have to back
// up the new stream to where the old stream was.
newTableStream.Dispose();
newTableStream = await outer.newTable.ExecuteQueryStreamedAsync(newTableContinuationQuery, requestOptions, operationContext);
newTableNext = await newTableStream.ReadRowAsync();
}
}
if (!outer.IsBugEnabled(MTableOptionalBug.QueryStreamedSaveNewConfig))
{
currentConfig = newConfig;
}
}
}
public override async Task <IList <TElement> > ExecuteQueryAtomicAsync <TElement>(TableQuery <TElement> query, TableRequestOptions requestOptions = null, OperationContext operationContext = null)
{
if (query.SelectColumns != null)
{
throw new NotImplementedException("select");
}
if (query.TakeCount != null)
{
throw new NotImplementedException("top");
}
FilterExpression origFilterExpr = ChainTableUtils.ParseFilterString(query.FilterString);
string partitionKey = ChainTableUtils.GetSingleTargetedPartitionKey(origFilterExpr);
TableQuery <MTableEntity> mtableQuery = ChainTableUtils.CopyQuery <TElement, MTableEntity>(query);
MTableConfiguration config;
using (configService.Subscribe(FixedSubscriber <MTableConfiguration> .Instance, out config))
{
IEnumerable <MTableEntity> results;
if (config.state <= TableClientState.USE_OLD_HIDE_METADATA)
{
results = await oldTable.ExecuteQueryAtomicAsync(mtableQuery, requestOptions, operationContext);
await monitor.AnnotateLastBackendCallAsync(wasLinearizationPoint : true);
}
else if (config.state >= TableClientState.USE_NEW_WITH_TOMBSTONES)
{
results = await newTable.ExecuteQueryAtomicAsync(mtableQuery, requestOptions, operationContext);
await monitor.AnnotateLastBackendCallAsync(wasLinearizationPoint : true);
}
else
{
// Modify the filter to make sure it matches the meta row.
if (origFilterExpr is ComparisonExpression)
{
// filterExpr must be "PartitionKey eq A", which already matches the meta row; nothing to do.
}
else
{
// filterExpr must be "(PartitionKey eq A) and (B)".
// Rewrite to "(PartitionKey eq A) and ((RowKey eq ROW_KEY_PARTITION_META) or (B))".
var boe = (BooleanOperatorExpression)origFilterExpr;
mtableQuery.FilterString =
TableQuery.CombineFilters(boe.Left.ToFilterString(), TableOperators.And,
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition(TableConstants.RowKey, QueryComparisons.Equal, ROW_KEY_PARTITION_META),
TableOperators.Or, boe.Right.ToFilterString()));
}
IList <MTableEntity> oldRows = await oldTable.ExecuteQueryAtomicAsync(mtableQuery, requestOptions, operationContext);
MTablePartitionState?state =
(from r in oldRows where r.RowKey == ROW_KEY_PARTITION_META select r.partitionState).SingleOrDefault();
IList <MTableEntity> newRows;
if (state == MTablePartitionState.SWITCHED)
{
await monitor.AnnotateLastBackendCallAsync();
// If the filter string includes conditions on user-defined properties,
// a row in the old table can be shadowed by a row in the new table
// that doesn't satisfy those conditions. To make sure we retrieve all
// potential shadowing rows, retrieve the entire partition.
// XXX: At a minimum, we should try to keep conditions on the
// primary key, but how clever do we want to get with query rewriting?
if (!IsBugEnabled(MTableOptionalBug.QueryAtomicFilterShadowing))
{
mtableQuery.FilterString = TableQuery.GenerateFilterCondition(
TableConstants.PartitionKey, QueryComparisons.Equal, partitionKey);
}
newRows = await newTable.ExecuteQueryAtomicAsync(mtableQuery, requestOptions, operationContext);
await monitor.AnnotateLastBackendCallAsync(wasLinearizationPoint : true);
}
else
{
await monitor.AnnotateLastBackendCallAsync(wasLinearizationPoint : true);
newRows = new List <MTableEntity>();
}
// Merge lists. Hopefully this is pretty clear. Walking the lists
// in lockstep might be faster but is a lot more code.
var merged = new SortedDictionary <string, MTableEntity>(StringComparer.Ordinal);
foreach (MTableEntity ent in oldRows)
{
merged[ent.RowKey] = ent;
}
foreach (MTableEntity ent in newRows)
{
merged[ent.RowKey] = ent;
}
results = merged.Values;
}
return((from ent in results where !RowKeyIsInternal(ent.RowKey) &&
origFilterExpr.Evaluate(ent) && !ent.deleted
select ent.Export <TElement>()).ToList());
}
}
void TranslateOperationForNewTable(
TableOperation op, MTableEntity existingEntity, bool leaveTombstones,
ref TableOperation newOp, ref HttpStatusCode?errorCode)
{
ITableEntity passedEntity = op.GetEntity();
TableOperationType opType = op.GetOperationType();
switch (opType)
{
case TableOperationType.Insert:
if (existingEntity == null)
{
newOp = TableOperation.Insert(ChainTableUtils.CopyEntity <MTableEntity>(passedEntity));
}
else if (existingEntity.deleted)
{
newOp = TableOperation.Replace(ImportWithIfMatch(passedEntity, existingEntity.ETag));
}
else
{
errorCode = HttpStatusCode.Conflict;
}
break;
case TableOperationType.Replace:
if ((errorCode = CheckExistingEntity(passedEntity, existingEntity)) == null)
{
newOp = TableOperation.Replace(ImportWithIfMatch(passedEntity, existingEntity.ETag));
}
break;
case TableOperationType.Merge:
if ((errorCode = CheckExistingEntity(passedEntity, existingEntity)) == null)
{
newOp = TableOperation.Merge(ImportWithIfMatch(passedEntity, existingEntity.ETag));
}
break;
case TableOperationType.Delete:
string buggablePartitionKey, buggableRowKey;
if (IsBugEnabled(MTableOptionalBug.DeletePrimaryKey))
{
buggablePartitionKey = buggableRowKey = null;
}
else
{
buggablePartitionKey = passedEntity.PartitionKey;
buggableRowKey = passedEntity.RowKey;
}
if (leaveTombstones)
{
if (passedEntity.ETag == ChainTable2Constants.ETAG_DELETE_IF_EXISTS)
{
newOp = TableOperation.InsertOrReplace(new MTableEntity {
PartitionKey = buggablePartitionKey, RowKey = buggableRowKey, deleted = true
});
}
else if ((errorCode = CheckExistingEntity(passedEntity, existingEntity)) == null)
{
newOp = TableOperation.Replace(new MTableEntity {
PartitionKey = buggablePartitionKey, RowKey = buggableRowKey,
deleted = true, ETag = existingEntity.ETag
});
}
}
else
{
if (passedEntity.ETag == ChainTable2Constants.ETAG_DELETE_IF_EXISTS)
{
if (existingEntity != null)
{
newOp = TableOperation.Delete(new MTableEntity {
PartitionKey = buggablePartitionKey, RowKey = buggableRowKey,
// It's OK to delete the entity and return success whether or not
// the entity is a tombstone by the time it is actually deleted.
ETag = IsBugEnabled(MTableOptionalBug.DeleteNoLeaveTombstonesETag) ? null : ChainTable2Constants.ETAG_ANY
});
}
// Otherwise generate nothing.
// FIXME: This is not linearizable! It can also generate empty batches.
}
else if ((errorCode = CheckExistingEntity(passedEntity, existingEntity)) == null)
{
// Another client in USE_NEW_WITH_TOMBSTONES could concurrently replace the
// entity with a tombstone, in which case we need to return 404 to the caller,
// hence this needs to be conditioned on the existing ETag.
newOp = TableOperation.Delete(new MTableEntity {
PartitionKey = buggablePartitionKey, RowKey = buggableRowKey,
ETag = IsBugEnabled(MTableOptionalBug.DeleteNoLeaveTombstonesETag) ? null : existingEntity.ETag
});
}
}
break;
case TableOperationType.InsertOrReplace:
newOp = TableOperation.InsertOrReplace(ChainTableUtils.CopyEntity <MTableEntity>(passedEntity));
break;
case TableOperationType.InsertOrMerge:
newOp = TableOperation.InsertOrMerge(ChainTableUtils.CopyEntity <MTableEntity>(passedEntity));
break;
default:
throw new NotImplementedException();
}
}
async Task DoRandomAtomicCalls()
{
for (int callNum = 0; callNum < MigrationModel.NUM_CALLS_PER_MACHINE; callNum++)
{
TableCall originalCall;
MirrorTableCall referenceCall;
SortedDictionary <PrimaryKey, DynamicTableEntity> dump = await peekProxy.DumpReferenceTableAsync();
if (PSharpRuntime.Nondeterministic())
{
// Query
// XXX: Test the filtering?
var query = new TableQuery <DynamicTableEntity>();
query.FilterString = TableQuery.GenerateFilterCondition(
TableConstants.PartitionKey, QueryComparisons.Equal, MigrationModel.SINGLE_PARTITION_KEY);
// async/await pair needed to upcast the return value to object.
originalCall = async table => await table.ExecuteQueryAtomicAsync(query);
referenceCall = async referenceTable => await referenceTable.ExecuteQueryAtomicAsync(query);
Console.WriteLine("{0} starting query", machineId);
}
else
{
// Batch write
int batchSize = PSharpRuntime.Nondeterministic() ? 2 : 1;
var batch = new TableBatchOperation();
var rowKeyChoices = new List <string> {
"0", "1", "2", "3", "4", "5"
};
for (int opNum = 0; opNum < batchSize; opNum++)
{
int opTypeNum = PSharpNondeterminism.Choice(7);
int rowKeyI = PSharpNondeterminism.Choice(rowKeyChoices.Count);
string rowKey = rowKeyChoices[rowKeyI];
rowKeyChoices.RemoveAt(rowKeyI); // Avoid duplicate in same batch
var primaryKey = new PrimaryKey(MigrationModel.SINGLE_PARTITION_KEY, rowKey);
string eTag = null;
if (opTypeNum >= 1 && opTypeNum <= 3)
{
DynamicTableEntity existingEntity;
int etagTypeNum = PSharpNondeterminism.Choice(
dump.TryGetValue(primaryKey, out existingEntity) ? 3 : 2);
switch (etagTypeNum)
{
case 0: eTag = ChainTable2Constants.ETAG_ANY; break;
case 1: eTag = "wrong"; break;
case 2: eTag = existingEntity.ETag; break;
}
}
DynamicTableEntity entity = new DynamicTableEntity
{
PartitionKey = MigrationModel.SINGLE_PARTITION_KEY,
RowKey = rowKey,
ETag = eTag,
Properties = new Dictionary <string, EntityProperty> {
// Give us something to see on merge. Might help with tracing too!
{ string.Format("{0}_c{1}_o{2}", machineId.ToString(), callNum, opNum),
new EntityProperty(true) }
}
};
switch (opTypeNum)
{
case 0: batch.Insert(entity); break;
case 1: batch.Replace(entity); break;
case 2: batch.Merge(entity); break;
case 3: batch.Delete(entity); break;
case 4: batch.InsertOrReplace(entity); break;
case 5: batch.InsertOrMerge(entity); break;
case 6:
entity.ETag = ChainTable2Constants.ETAG_DELETE_IF_EXISTS;
batch.Delete(entity); break;
}
}
TableBatchOperation batchCopy = ChainTableUtils.CopyBatch <DynamicTableEntity>(batch);
originalCall = async table => await table.ExecuteBatchAsync(batch);
referenceCall = async referenceTable => await referenceTable.ExecuteMirrorBatchAsync(batchCopy, successfulBatchResult);
Console.WriteLine("{0} starting batch {1}", machineId, batch);
}
await RunCallAsync(originalCall, referenceCall);
Console.WriteLine("{0} table call verified");
}
}
async Task <IList <TableResult> > ExecuteBatchOnNewTableAsync(MTableConfiguration config,
TableBatchOperation batch, TableRequestOptions requestOptions, OperationContext operationContext)
{
string partitionKey = ChainTableUtils.GetBatchPartitionKey(batch);
await EnsurePartitionSwitchedAsync(partitionKey, requestOptions, operationContext);
if (config.state <= TableClientState.PREFER_NEW)
{
await EnsureAffectedRowsMigratedAsync(batch, requestOptions, operationContext);
}
Attempt:
// Batch on new table.
var query = GenerateQueryForAffectedRows(batch);
IList <MTableEntity> newRows = await newTable.ExecuteQueryAtomicAsync(query, requestOptions, operationContext);
Dictionary <string, MTableEntity> newDict = newRows.ToDictionary(ent => ent.RowKey);
// NOTE! At this point, the read has not yet been annotated. It is annotated below.
var newBatch = new TableBatchOperation();
var inputToNewTableIndexMapping = new List <int?>();
for (int i = 0; i < batch.Count; i++)
{
TableOperation op = batch[i];
ITableEntity passedEntity = op.GetEntity();
MTableEntity existingEntity = newDict.GetValueOrDefault(passedEntity.RowKey);
TableOperation newOp = null;
HttpStatusCode?errorCode = null;
TranslateOperationForNewTable(
op, existingEntity, config.state <= TableClientState.USE_NEW_WITH_TOMBSTONES,
ref newOp, ref errorCode);
if (errorCode != null)
{
Debug.Assert(newOp == null);
await monitor.AnnotateLastBackendCallAsync(wasLinearizationPoint : true);
throw ChainTableUtils.GenerateBatchException(errorCode.Value, i);
}
if (newOp != null)
{
inputToNewTableIndexMapping.Add(newBatch.Count);
newBatch.Add(newOp);
}
else
{
inputToNewTableIndexMapping.Add(null);
}
}
await monitor.AnnotateLastBackendCallAsync();
IList <TableResult> newResults;
try
{
newResults = await newTable.ExecuteBatchAsync(newBatch, requestOptions, operationContext);
}
catch (ChainTableBatchException)
{
// XXX: Try to distinguish expected concurrency exceptions from unexpected exceptions?
await monitor.AnnotateLastBackendCallAsync();
goto Attempt;
}
// We made it!
var results = new List <TableResult>();
for (int i = 0; i < batch.Count; i++)
{
ITableEntity passedEntity = batch[i].GetEntity();
int? newTableIndex = inputToNewTableIndexMapping[i];
string newETag =
(IsBugEnabled(MTableOptionalBug.TombstoneOutputETag)
? newTableIndex != null
: batch[i].GetOperationType() == TableOperationType.Delete)
? null : newResults[newTableIndex.Value].Etag;
if (newETag != null)
{
passedEntity.ETag = newETag;
}
results.Add(new TableResult
{
HttpStatusCode = (int)(
(batch[i].GetOperationType() == TableOperationType.Insert) ? HttpStatusCode.Created : HttpStatusCode.NoContent),
Etag = newETag,
Result = passedEntity,
});
}
await monitor.AnnotateLastBackendCallAsync(wasLinearizationPoint : true, successfulBatchResult : results);
return(results);
}
async Task CleanupAsync(TableRequestOptions requestOptions, OperationContext operationContext)
{
string previousPartitionKey = null;
await WalkTableInParallel(newTable, requestOptions, operationContext, async (newEntity) =>
{
if (newEntity.PartitionKey != previousPartitionKey)
{
if (previousPartitionKey != null)
{
Console.WriteLine("Cleaned Partition: {0}", previousPartitionKey);
}
previousPartitionKey = newEntity.PartitionKey;
}
// XXX: Consider factoring out this "query and retry" pattern
// into a separate method.
Attempt:
TableOperation cleanupOp = newEntity.deleted ? TableOperation.Delete(newEntity)
: TableOperation.Replace(newEntity.Export <DynamicTableEntity>());
TableResult cleanupResult;
StorageException ex;
try
{
cleanupResult = await newTable.ExecuteAsync(cleanupOp, requestOptions, operationContext);
ex = null;
}
catch (StorageException exToHandle)
{
cleanupResult = null;
ex = exToHandle;
}
if (ex != null)
{
if (ex.GetHttpStatusCode() == HttpStatusCode.NotFound)
{
// Someone else deleted it concurrently. Nothing to do.
//await monitor.AnnotateLastBackendCallAsync();
return((IQueryStream <MTableEntity>)null);
}
else if (ex.GetHttpStatusCode() == HttpStatusCode.PreconditionFailed)
{
//await monitor.AnnotateLastBackendCallAsync();
// Unfortunately we can't assume that anyone who concurrently modifies
// the row while the table is in state USE_NEW_HIDE_METADATA will
// clean it up, because of InsertOrMerge. (Consider redesign?)
// Re-retrieve row.
TableResult retrieveResult = await newTable.ExecuteAsync(
TableOperation.Retrieve <MTableEntity>(newEntity.PartitionKey, newEntity.RowKey),
requestOptions, operationContext);
//await monitor.AnnotateLastBackendCallAsync();
if ((HttpStatusCode)retrieveResult.HttpStatusCode == HttpStatusCode.NotFound)
{
return(null);
}
else
{
newEntity = (MTableEntity)retrieveResult.Result;
goto Attempt;
}
}
else
{
throw ChainTableUtils.GenerateInternalException(ex);
}
}
else
{
await monitor.AnnotateLastBackendCallAsync(
spuriousETagChanges:
cleanupResult.Etag == null ? null : new List <SpuriousETagChange>
{
new SpuriousETagChange(newEntity.PartitionKey, newEntity.RowKey, cleanupResult.Etag)
});
}
return((IQueryStream <MTableEntity>)null);
});
await WalkTableInParallel(oldTable, requestOptions, operationContext, async (oldEntity) =>
{
await oldTable.ExecuteAsync(TableOperation.Delete(oldEntity), requestOptions, operationContext);
await monitor.AnnotateLastBackendCallAsync();
return((IQueryStream <MTableEntity>)null);
});
}
