// perform a query
public override async Task QueryAsync(PartitionQueryEvent queryEvent, EffectTracker effectTracker)
{
try
{
var instanceQuery = queryEvent.InstanceQuery;
#if FASTER_SUPPORTS_PSF
IAsyncEnumerable <OrchestrationState> queryPSFsAsync(ClientSession <Key, Value, EffectTracker, TrackedObject, PartitionReadEvent, Functions> session)
{
// Issue the PSF query. Note that pending operations will be completed before this returns.
var querySpec = new List <(IPSF, IEnumerable <PSFKey>)>();
if (instanceQuery.HasRuntimeStatus)
{
querySpec.Add((this.RuntimeStatusPsf, instanceQuery.RuntimeStatus.Select(s => new PSFKey(s))));
}
if (instanceQuery.CreatedTimeFrom.HasValue || instanceQuery.CreatedTimeTo.HasValue)
{
IEnumerable <PSFKey> enumerateDateBinKeys()
{
var to = instanceQuery.CreatedTimeTo ?? DateTime.UtcNow;
var from = instanceQuery.CreatedTimeFrom ?? to.AddDays(-7); // TODO Some default so we don't have to iterate from the first possible date
for (var dt = from; dt <= to; dt += PSFKey.DateBinInterval)
{
yield return(new PSFKey(dt));
}
}
querySpec.Add((this.CreatedTimePsf, enumerateDateBinKeys()));
}
if (!string.IsNullOrWhiteSpace(instanceQuery.InstanceIdPrefix))
{
querySpec.Add((this.InstanceIdPrefixPsf, new[] { new PSFKey(instanceQuery.InstanceIdPrefix) }));
}
var querySettings = new PSFQuerySettings
{
// This is a match-all-PSFs enumeration so do not continue after any PSF has hit EOS
OnStreamEnded = (unusedPsf, unusedIndex) => false
};
OrchestrationState getOrchestrationState(ref Value v)
{
if (v.Val is byte[] serialized)
{
var result = ((InstanceState)Serializer.DeserializeTrackedObject(serialized))?.OrchestrationState;
if (result != null && !instanceQuery.FetchInput)
{
result.Input = null;
}
return(result);
}
else
{
var state = ((InstanceState)((TrackedObject)v))?.OrchestrationState;
var result = state?.ClearFieldsImmutably(instanceQuery.FetchInput, true);
return(result);
}
}
return(session.QueryPSFAsync(querySpec, matches => matches.All(b => b), querySettings)
.Select(providerData => getOrchestrationState(ref providerData.GetValue()))
.Where(orchestrationState => orchestrationState != null));
}
#else
IAsyncEnumerable <OrchestrationState> queryPSFsAsync(ClientSession <Key, Value, EffectTracker, TrackedObject, object, IFunctions <Key, Value, EffectTracker, TrackedObject, object> > session)
=> this.ScanOrchestrationStates(effectTracker, queryEvent);
#endif
// create an individual session for this query so the main session can be used
// while the query is progressing.
using (var session = this.CreateASession())
{
var orchestrationStates = (this.partition.Settings.UsePSFQueries && instanceQuery.IsSet)
? queryPSFsAsync(session)
: this.ScanOrchestrationStates(effectTracker, queryEvent);
await effectTracker.ProcessQueryResultAsync(queryEvent, orchestrationStates);
}
}
catch (Exception exception)
when(this.terminationToken.IsCancellationRequested && !Utils.IsFatal(exception))
{
throw new OperationCanceledException("Partition was terminated.", exception, this.terminationToken);
}
}
IAsyncEnumerable <OrchestrationState> ScanOrchestrationStates(
EffectTracker effectTracker,
PartitionQueryEvent queryEvent)
{
var instanceQuery = queryEvent.InstanceQuery;
string queryId = queryEvent.EventIdString;
this.partition.EventDetailTracer?.TraceEventProcessingDetail($"starting query {queryId}");
// we use a separate thread to iterate, since Faster can iterate synchronously only at the moment
// and we don't want it to block thread pool worker threads
var channel = Channel.CreateBounded <OrchestrationState>(500);
var scanThread = new Thread(RunScan)
{
Name = $"QueryScan-{queryId}"
};
scanThread.Start();
return(channel.Reader.ReadAllAsync());
void RunScan()
{
using var _ = EventTraceContext.MakeContext(0, queryId);
// get the unique set of keys appearing in the log and emit them
using var iter1 = this.fht.Iterate();
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
long scanned = 0;
long deserialized = 0;
long matched = 0;
long lastReport;
void ReportProgress()
{
this.partition.EventDetailTracer?.TraceEventProcessingDetail(
$"query {queryId} scan position={iter1.CurrentAddress} elapsed={stopwatch.Elapsed.TotalSeconds:F2}s scanned={scanned} deserialized={deserialized} matched={matched}");
lastReport = stopwatch.ElapsedMilliseconds;
}
ReportProgress();
while (iter1.GetNext(out RecordInfo recordInfo) && !recordInfo.Tombstone)
{
if (stopwatch.ElapsedMilliseconds - lastReport > 5000)
{
ReportProgress();
}
TrackedObjectKey key = iter1.GetKey().Val;
if (key.ObjectType == TrackedObjectKey.TrackedObjectType.Instance)
{
scanned++;
//this.partition.EventDetailTracer?.TraceEventProcessingDetail($"found instance {key.InstanceId}");
if (string.IsNullOrEmpty(instanceQuery?.InstanceIdPrefix) ||
key.InstanceId.StartsWith(instanceQuery.InstanceIdPrefix))
{
//this.partition.EventDetailTracer?.TraceEventProcessingDetail($"reading instance {key.InstanceId}");
object val = iter1.GetValue().Val;
//this.partition.EventDetailTracer?.TraceEventProcessingDetail($"read instance {key.InstanceId}, is {(val == null ? "null" : val.GetType().Name)}");
InstanceState instanceState;
if (val is byte[] bytes)
{
instanceState = (InstanceState)Serializer.DeserializeTrackedObject(bytes);
deserialized++;
}
else
{
instanceState = (InstanceState)val;
}
// reading the orchestrationState may race with updating the orchestration state
// but it is benign because the OrchestrationState object is immutable
var orchestrationState = instanceState?.OrchestrationState;
if (orchestrationState != null &&
instanceQuery.Matches(orchestrationState))
{
matched++;
this.partition.EventDetailTracer?.TraceEventProcessingDetail($"match instance {key.InstanceId}");
var value = orchestrationState.ClearFieldsImmutably(!instanceQuery.FetchInput, false);
var task = channel.Writer.WriteAsync(value);
if (!task.IsCompleted)
{
task.AsTask().Wait();
}
}
}
}
}
ReportProgress();
channel.Writer.Complete();
this.partition.EventDetailTracer?.TraceEventProcessingDetail($"finished query {queryId}");
}
}
// perform a query public abstract Task QueryAsync(PartitionQueryEvent queryEvent, EffectTracker effectTracker);
