public override Task <IQueryStream <TElement> > ExecuteQueryStreamedAsync <TElement>(TableQuery <TElement> query, TableRequestOptions requestOptions = null, OperationContext operationContext = null)
{
CallLogging.LogStart(proxyName, nameof(ExecuteQueryStreamedAsync), query, requestOptions, operationContext);
return(LogOutcomeAsync <IQueryStream <TElement> >(async() => {
IQueryStream <TElement> origStream = await original.ExecuteQueryStreamedAsync(query, requestOptions, operationContext);
return new QueryStreamLoggingProxy <TElement>(origStream, string.Format("{0} QueryStream {1}", proxyName, queryStreamCount++));
},
(outcome) => CallLogging.LogEnd(proxyName, nameof(ExecuteQueryStreamedAsync), outcome, query, requestOptions, operationContext)));
}
async Task CopyAsync(TableRequestOptions requestOptions, OperationContext operationContext)
{
// Query all the entities!
IQueryStream <MTableEntity> oldTableStream = await oldTable.ExecuteQueryStreamedAsync(
new TableQuery <MTableEntity>(), requestOptions, operationContext);
MTableEntity oldEntity;
string previousPartitionKey = null;
while ((oldEntity = await oldTableStream.ReadRowAsync()) != null)
{
if (RowKeyIsInternal(oldEntity.RowKey))
{
continue;
}
if (oldEntity.PartitionKey != previousPartitionKey)
{
previousPartitionKey = oldEntity.PartitionKey;
await EnsurePartitionSwitchedAsync(oldEntity.PartitionKey, requestOptions, operationContext);
if (IsBugEnabled(MTableOptionalBug.InsertBehindMigrator))
{
// More reasonable formulation of this bug. If we're going
// to allow CopyAsync while some clients are still writing
// to the old table, give the developer credit for realizing
// that the stream might be stale by the time the partition
// is switched.
oldTableStream = await oldTable.ExecuteQueryStreamedAsync(
new TableQuery <MTableEntity> {
FilterString = TableQuery.GenerateFilterCondition(
TableConstants.PartitionKey,
QueryComparisons.GreaterThanOrEqual,
oldEntity.PartitionKey)
}, requestOptions, operationContext);
continue;
}
}
// Can oldEntity be stale by the time we EnsurePartitionSwitchedAsync?
// Currently no, because we don't start the copy until all clients have
// entered PREFER_NEW state, meaning that all writes go to the new
// table. When we implement the Kstart/Kdone optimization (or similar),
// then clients will be able to write to the old table in parallel with
// our scan and we'll need to rescan after EnsurePartitionSwitchedAsync.
// Should we also stream from the new table and copy only the
// entities not already copied?
await TryCopyEntityToNewTableAsync(oldEntity, requestOptions, operationContext);
}
}
public override async Task <IQueryStream <TElement> > ExecuteQueryStreamedAsync <TElement>(TableQuery <TElement> query, TableRequestOptions requestOptions = null, OperationContext operationContext = null)
{
// Philosophically, maybe this proxy-making belongs on the host
// side, but that would require a second custom wrapper because we
// still need the custom wrapper on the caller side to do the
// different event types.
IQueryStream <TElement> remoteStream = await nonannotatableCallProxy.ExecuteQueryStreamedAsync(
query, requestOptions, operationContext);
return(new QueryStreamPSharpProxy <TElement>(
PSharpRealProxy.MakeTransparentProxy(callerMachineId, hostMachineId, remoteStream,
null, () => new TableNonannotatableCallEvent())));
}
public override async Task <IQueryStream <TElement> > ExecuteQueryStreamedAsync <TElement>(TableQuery <TElement> query, TableRequestOptions requestOptions = null, OperationContext operationContext = null)
{
//Trace.TraceInformation("{0} calling {1}.ExecuteQueryStreamedAsync({2})", callerMachineId, debugName, BetterComparer.ToString(query));
// Philosophically, maybe this proxy-making belongs on the host
// side, but that would require a second custom wrapper because we
// still need the custom wrapper on the caller side to do the
// different event types.
IQueryStream <TElement> remoteStream = await plainEventProxy.ExecuteQueryStreamedAsync(
query, requestOptions, operationContext);
return(new QueryStreamPSharpProxy <TElement>(
PSharpRealProxy.MakeTransparentProxy(callerMachineId, hostMachineId, remoteStream,
string.Format("<{0} QueryStream>", debugName), () => new GenericDispatchableEvent())));
}
internal async Task RunQueryStreamedAsync(TableQuery <DynamicTableEntity> query)
{
int startRevision = await peekProxy.GetReferenceTableRevisionAsync();
FilterExpression filterExpr = ChainTableUtils.ParseFilterString(query.FilterString);
Console.WriteLine("{0} starting streaming query: {1}", machineId, query);
using (IQueryStream <DynamicTableEntity> stream = await migratingTable.ExecuteQueryStreamedAsync(query))
{
PrimaryKey lastKey = ChainTableUtils.FirstValidPrimaryKey;
for (;;)
{
PrimaryKey returnedContinuationKey = await stream.GetContinuationPrimaryKeyAsync();
PSharpRuntime.Assert(returnedContinuationKey == null || returnedContinuationKey.CompareTo(lastKey) >= 0,
"{0}: query stream continuation key is {1}, expected >= {2}",
machineId, returnedContinuationKey, lastKey);
DynamicTableEntity row = await stream.ReadRowAsync(); // may be null, meaning end of stream
// Must be after ReadRowAsync, otherwise additional rows could become valid
// due to a mutation between GetValidStreamReadRows and ReadRowAsync and
// we would falsely report a bug if ReadRowAsync returns one of those rows.
List <DynamicTableEntity> validRows = await peekProxy.GetValidStreamReadRows(startRevision, filterExpr, lastKey);
// Three cheers for automatic use of covariance in overload resolution!
PSharpRuntime.Assert(validRows.Contains(row, BetterComparer.Instance),
"{0} query stream returned {1}, which is not one of the valid rows: {2}",
machineId, BetterComparer.ToString(row), BetterComparer.ToString(validRows));
Console.WriteLine("{0} query stream returned row {1}, which is valid", machineId, BetterComparer.ToString(row));
if (row == null)
{
// Any returnedContinuationKey (including null) is less or equal to a row of null.
break;
}
else
{
PSharpRuntime.Assert(returnedContinuationKey != null && returnedContinuationKey.CompareTo(row.GetPrimaryKey()) <= 0,
"{0}: query stream continuation key is {1}, expected <= {2}",
machineId, returnedContinuationKey, row.GetPrimaryKey());
lastKey = ChainTableUtils.NextValidPrimaryKeyAfter(row.GetPrimaryKey());
}
}
}
Console.WriteLine("{0} finished streaming query", machineId);
}
internal async Task StartAsync()
{
using (await LockAsyncBuggable())
{
configSubscription = outer.configService.Subscribe(new Subscriber(this), out currentConfig);
if (currentConfig.state < TableClientState.USE_NEW_WITH_TOMBSTONES)
{
oldTableStream = await outer.oldTable.ExecuteQueryStreamedAsync(mtableQuery, requestOptions, operationContext);
oldTableNext = await oldTableStream.ReadRowAsync();
}
if (currentConfig.state > TableClientState.USE_OLD_HIDE_METADATA)
{
TableQuery <MTableEntity> newTableQuery =
outer.IsBugEnabled(MTableOptionalBug.QueryStreamedFilterShadowing) ? mtableQuery
// Yes, match everything (!). See newTableContinuationQuery in ApplyConfigurationAsync.
: new TableQuery <MTableEntity>();
newTableStream = await outer.newTable.ExecuteQueryStreamedAsync(newTableQuery, requestOptions, operationContext);
newTableNext = await newTableStream.ReadRowAsync();
}
}
}
async Task DoQueryStreamed()
{
int startRevision = await peekProxy.GetReferenceTableRevisionAsync();
// XXX: Test the filtering?
var query = new TableQuery <DynamicTableEntity>();
using (IQueryStream <DynamicTableEntity> stream = await migratingTable.ExecuteQueryStreamedAsync(query))
{
PrimaryKey continuationKey = await stream.GetContinuationPrimaryKeyAsync();
await peekProxy.ValidateQueryStreamGapAsync(startRevision, null, continuationKey);
do
{
DynamicTableEntity row = await stream.ReadRowAsync();
PrimaryKey newContinuationKey = await stream.GetContinuationPrimaryKeyAsync();
if (row == null)
{
PSharpRuntime.Assert(newContinuationKey == null);
await peekProxy.ValidateQueryStreamGapAsync(startRevision, continuationKey, null);
}
else
{
await peekProxy.ValidateQueryStreamGapAsync(startRevision, continuationKey, row.GetPrimaryKey());
await peekProxy.ValidateQueryStreamRowAsync(startRevision, row);
await peekProxy.ValidateQueryStreamGapAsync(startRevision,
ChainTableUtils.NextValidPrimaryKeyAfter(row.GetPrimaryKey()), newContinuationKey);
}
continuationKey = newContinuationKey;
} while (continuationKey != null);
}
}
internal QueryStreamPSharpProxy(IQueryStream <TElement> plainProxy)
{
this.plainProxy = plainProxy;
}
async Task CleanupAsync(TableRequestOptions requestOptions, OperationContext operationContext)
{
// Clean up MTable-specific data from new table.
// Future: Query only ones with properties we need to clean up.
IQueryStream <MTableEntity> newTableStream = await newTable.ExecuteQueryStreamedAsync(
new TableQuery <MTableEntity>(), requestOptions, operationContext);
MTableEntity newEntity;
while ((newEntity = await newTableStream.ReadRowAsync()) != null)
{
// XXX: Consider factoring out this "query and retry" pattern
// into a separate method.
Attempt:
TableOperation cleanupOp = newEntity.deleted ? TableOperation.Delete(newEntity)
: TableOperation.Replace(newEntity.Export <DynamicTableEntity>());
TableResult cleanupResult;
try
{
cleanupResult = await newTable.ExecuteAsync(cleanupOp, requestOptions, operationContext);
}
catch (StorageException ex)
{
if (ex.GetHttpStatusCode() == HttpStatusCode.NotFound)
{
// Someone else deleted it concurrently. Nothing to do.
await monitor.AnnotateLastBackendCallAsync();
continue;
}
else if (ex.GetHttpStatusCode() == HttpStatusCode.PreconditionFailed)
{
await monitor.AnnotateLastBackendCallAsync();
// Unfortunately we can't assume that anyone who concurrently modifies
// the row while the table is in state USE_NEW_HIDE_METADATA will
// clean it up, because of InsertOrMerge. (Consider redesign?)
// Re-retrieve row.
TableResult retrieveResult = await newTable.ExecuteAsync(
TableOperation.Retrieve <MTableEntity>(newEntity.PartitionKey, newEntity.RowKey),
requestOptions, operationContext);
await monitor.AnnotateLastBackendCallAsync();
if ((HttpStatusCode)retrieveResult.HttpStatusCode == HttpStatusCode.NotFound)
{
continue;
}
else
{
newEntity = (MTableEntity)retrieveResult.Result;
goto Attempt;
}
}
else
{
throw ChainTableUtils.GenerateInternalException(ex);
}
}
await monitor.AnnotateLastBackendCallAsync(
spuriousETagChanges :
cleanupResult.Etag == null?null : new List <SpuriousETagChange> {
new SpuriousETagChange(newEntity.PartitionKey, newEntity.RowKey, cleanupResult.Etag)
});
}
// Delete everything from old table! No one should be modifying it concurrently.
IQueryStream <MTableEntity> oldTableStream = await oldTable.ExecuteQueryStreamedAsync(
new TableQuery <MTableEntity>(), requestOptions, operationContext);
MTableEntity oldEntity;
while ((oldEntity = await oldTableStream.ReadRowAsync()) != null)
{
await oldTable.ExecuteAsync(TableOperation.Delete(oldEntity), requestOptions, operationContext);
await monitor.AnnotateLastBackendCallAsync();
}
}
async Task ApplyConfigurationAsync(MTableConfiguration newConfig)
{
using (await LockAsyncBuggable())
{
PrimaryKey continuationKey = InternalGetContinuationPrimaryKey();
// ExecuteQueryStreamedAsync validated that mtableQuery has no select or top.
TableQuery <MTableEntity> newTableContinuationQuery =
(continuationKey == null) ? null : new TableQuery <MTableEntity>
{
// As in ExecuteQueryAtomicAsync, we have to retrieve all
// potential shadowing rows.
// XXX: This pays even a bigger penalty for not keeping
// conditions on the primary key. But if we were to simply
// keep all such conditions, there's a potential to add
// more and more continuation filter conditions due to
// layering of IChainTable2s, which would lead to some extra
// overhead and push us closer to the limit on number of
// comparisons. Either try to parse for this or change
// ExecuteQueryStreamedAsync to always take a continuation
// primary key?
FilterString =
outer.IsBugEnabled(MTableOptionalBug.QueryStreamedFilterShadowing)
? ChainTableUtils.CombineFilters(
ChainTableUtils.GenerateContinuationFilterCondition(continuationKey),
TableOperators.And,
mtableQuery.FilterString // Could be empty.
)
: ChainTableUtils.GenerateContinuationFilterCondition(continuationKey),
};
bool justStartedNewStream = false;
// Actually, if the query started in state
// USE_OLD_HIDE_METADATA, it is API-compliant to continue
// returning data from the old table until the migrator starts
// deleting it, at which point we switch to the new table. But
// some callers may benefit from fresher data, even if it is
// never guaranteed, so we go ahead and start the new stream.
// XXX: Is this the right decision?
if (newConfig.state > TableClientState.USE_OLD_HIDE_METADATA &&
currentConfig.state <= TableClientState.USE_OLD_HIDE_METADATA &&
newTableContinuationQuery != null)
{
newTableStream = await outer.newTable.ExecuteQueryStreamedAsync(newTableContinuationQuery, requestOptions, operationContext);
newTableNext = await newTableStream.ReadRowAsync();
justStartedNewStream = true;
}
if (newConfig.state >= TableClientState.USE_NEW_HIDE_METADATA &&
currentConfig.state < TableClientState.USE_NEW_HIDE_METADATA)
{
oldTableStream.Dispose();
oldTableStream = null;
oldTableNext = null; // Stop DetermineNextSide from trying to read the old stream.
if (!outer.IsBugEnabled(MTableOptionalBug.QueryStreamedBackUpNewStream) &&
newTableContinuationQuery != null && !justStartedNewStream)
{
// The new stream could have gotten ahead of the old
// stream if rows had not yet been migrated. This
// was OK as long as we still planned to read those
// rows from the old stream, but now we have to back
// up the new stream to where the old stream was.
newTableStream.Dispose();
newTableStream = await outer.newTable.ExecuteQueryStreamedAsync(newTableContinuationQuery, requestOptions, operationContext);
newTableNext = await newTableStream.ReadRowAsync();
}
}
if (!outer.IsBugEnabled(MTableOptionalBug.QueryStreamedSaveNewConfig))
{
currentConfig = newConfig;
}
}
}
internal QueryStreamLoggingProxy(IQueryStream <TElement> original, string proxyName)
{
this.original = original;
this.proxyName = proxyName;
}
internal QueryStreamPSharpProxy(IQueryStream <TElement> nonannotatableProxy)
{
this.nonannotatableProxy = nonannotatableProxy;
}
Task WalkTableInParallel(IChainTable2 table, TableRequestOptions requestOptions, OperationContext operationContext,
Func <MTableEntity, Task <IQueryStream <MTableEntity> > > rowCallbackAsync)
{
return(Task.WhenAll(Enumerable.Range(0, 16).Select(hexNumber => Task.Run(async() =>
{
var query = new TableQuery <MTableEntity>();
if (hexNumber == 0)
{
query.FilterString = TableQuery.GenerateFilterCondition(
"PartitionKey",
QueryComparisons.LessThanOrEqual,
(hexNumber + 1).ToString("x"));
}
else if (hexNumber == 15)
{
query.FilterString = TableQuery.GenerateFilterCondition(
"PartitionKey",
QueryComparisons.GreaterThanOrEqual,
hexNumber.ToString("x"));
}
else
{
query.FilterString = TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition(
"PartitionKey",
QueryComparisons.GreaterThanOrEqual,
hexNumber.ToString("x")),
TableOperators.And,
TableQuery.GenerateFilterCondition(
"PartitionKey",
QueryComparisons.LessThanOrEqual,
(hexNumber + 1).ToString("x")));
}
try
{
IQueryStream <MTableEntity> tableStream = await table.ExecuteQueryStreamedAsync(
query, requestOptions, operationContext);
MTableEntity entity;
while ((entity = await tableStream.ReadRowAsync()) != null)
{
IQueryStream <MTableEntity> newTableStream = await rowCallbackAsync(entity);
if (newTableStream != null)
{
tableStream = newTableStream;
}
}
}
catch (Exception e)
{
while (e is AggregateException)
{
e = ((AggregateException)(e)).InnerException;
}
if (!(e is StorageException && ((StorageException)e).RequestInformation.HttpStatusCode == 404))
{
throw;
}
}
}))));
}