private void GetRevisionsQueryString(DocumentDatabase database, DocumentsOperationContext context, out IncludeRevisionsCommand includeRevisions)
{
includeRevisions = null;
var rif = new RevisionIncludeField();
var revisionsByChangeVectors = GetStringValuesQueryString("revisions", required: false);
var revisionByDateTimeBefore = GetStringValuesQueryString("revisionsBefore", required: false);
if (revisionsByChangeVectors.Count == 0 && revisionByDateTimeBefore.Count == 0)
{
return;
}
if (DateTime.TryParseExact(revisionByDateTimeBefore.ToString(), DefaultFormat.DateTimeFormatsToRead, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var dateTime))
{
rif.RevisionsBeforeDateTime = dateTime.ToUniversalTime();
}
foreach (var changeVector in revisionsByChangeVectors)
{
rif.RevisionsChangeVectorsPaths.Add(changeVector);
}
includeRevisions = new IncludeRevisionsCommand(database, context, rif);
}
public override void AddRevisionIncludes(IncludeRevisionsCommand command)
{
if (command.RevisionsChangeVectorResults is null)
{
return;
}
throw new NotSupportedException();
}
public GraphQueryResultRetriever(GraphQuery graphQuery, DocumentDatabase database,
IndexQueryServerSide query,
QueryTimingsScope queryTimings,
DocumentsStorage documentsStorage,
DocumentsOperationContext context,
FieldsToFetch fieldsToFetch,
IncludeDocumentsCommand includeDocumentsCommand,
IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand,
IncludeRevisionsCommand includeRevisionsCommand)
: base(database, query, queryTimings, fieldsToFetch, documentsStorage, context, false, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand)
{
_graphQuery = graphQuery;
_context = context;
}
public CollectionQueryEnumerable(DocumentDatabase database, DocumentsStorage documents, FieldsToFetch fieldsToFetch, string collection,
IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext context, IncludeDocumentsCommand includeDocumentsCommand,
IncludeRevisionsCommand includeRevisionsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, Reference <int> totalResults,
Reference <int> scannedResults, Reference <long> skippedResults, CancellationToken token)
{
_database = database;
_documents = documents;
_fieldsToFetch = fieldsToFetch;
_collection = collection;
_isAllDocsCollection = collection == Constants.Documents.Collections.AllDocumentsCollection;
_query = query;
_queryTimings = queryTimings;
_context = context;
_includeDocumentsCommand = includeDocumentsCommand;
_includeRevisionsCommand = includeRevisionsCommand;
_includeCompareExchangeValuesCommand = includeCompareExchangeValuesCommand;
_totalResults = totalResults;
_token = token;
_scannedResults = scannedResults;
_skippedResults = skippedResults;
}
protected QueryResultRetrieverBase(
DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, FieldsToFetch fieldsToFetch, DocumentsStorage documentsStorage,
JsonOperationContext context, bool reduceResults, IncludeDocumentsCommand includeDocumentsCommand,
IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand)
{
_database = database;
_query = query;
_context = context;
_includeDocumentsCommand = includeDocumentsCommand;
_includeRevisionsCommand = includeRevisionsCommand;
_includeCompareExchangeValuesCommand = includeCompareExchangeValuesCommand;
ValidateFieldsToFetch(fieldsToFetch);
FieldsToFetch = fieldsToFetch;
DocumentsStorage = documentsStorage;
RetrieverScope = queryTimings?.For(nameof(QueryTimingsScope.Names.Retriever), start: false);
DocumentFields = query?.DocumentFields ?? DocumentFields.All;
_blittableTraverser = reduceResults ? BlittableJsonTraverser.FlatMapReduceResults : BlittableJsonTraverser.Default;
}
public Enumerator(DocumentDatabase database, DocumentsStorage documents, FieldsToFetch fieldsToFetch, string collection, bool isAllDocsCollection,
IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext context, IncludeDocumentsCommand includeDocumentsCommand,
IncludeRevisionsCommand includeRevisionsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, Reference <int> totalResults,
Reference <int> scannedResults, string startAfterId, Reference <long> alreadySeenIdsCount, DocumentFields fields, Reference <long> skippedResults, CancellationToken token)
{
_documents = documents;
_fieldsToFetch = fieldsToFetch;
_collection = collection;
_isAllDocsCollection = isAllDocsCollection;
_query = query;
_queryTimings = queryTimings;
_context = context;
_totalResults = totalResults;
_scannedResults = scannedResults;
_totalResults.Value = 0;
_startAfterId = startAfterId;
_alreadySeenIdsCount = alreadySeenIdsCount;
_fields = fields;
_skippedResults = skippedResults;
_token = token;
if (_fieldsToFetch.IsDistinct)
{
_alreadySeenProjections = new HashSet <ulong>();
}
_resultsRetriever = new MapQueryResultRetriever(database, query, queryTimings, documents, context, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand);
(_ids, _startsWith) = ExtractIdsFromQuery(query, context);
if (_query.Metadata.FilterScript != null)
{
var key = new FilterKey(_query.Metadata);
_releaseFilterScriptRunner = database.Scripts.GetScriptRunner(key, readOnly: true, patchRun: out _filterScriptRun);
}
}
public CountersQueryResultRetriever(DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsStorage documentsStorage, JsonOperationContext context, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand)
: base(database, query, queryTimings, documentsStorage, context, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand)
{
}
public override void AddRevisionIncludes(IncludeRevisionsCommand command)
{
_revisionsChangeVectorResults = command.RevisionsChangeVectorResults;
_revisionsDateTimeBeforeResults = command.IdByRevisionsByDateTimeResults;
}
public override IQueryResultRetriever GetQueryResultRetriever(IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext documentsContext, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand)
{
throw new NotSupportedException($"Index {Name} is in-memory implementation of a faulty index", _e);
}
public override IQueryResultRetriever GetQueryResultRetriever(IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext documentsContext, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand)
{
return(new TimeSeriesQueryResultRetriever(DocumentDatabase, query, queryTimings, DocumentDatabase.DocumentsStorage, documentsContext, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand));
}
protected StoredValueQueryResultRetriever(string storedValueFieldName, DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsStorage documentsStorage, JsonOperationContext context, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand)
: base(database, query, queryTimings, fieldsToFetch, documentsStorage, context, true, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand)
{
if (storedValueFieldName == null)
{
throw new ArgumentNullException(nameof(storedValueFieldName));
}
_storedValueFieldName = storedValueFieldName;
_context = context;
}
public MapReduceQueryResultRetriever(DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsStorage documentsStorage, JsonOperationContext context, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand)
: base(Constants.Documents.Indexing.Fields.ReduceKeyValueFieldName, database, query, queryTimings, documentsStorage, context, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand)
{
}
public override IQueryResultRetriever GetQueryResultRetriever(IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext documentsContext, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand)
{
throw new System.NotImplementedException();
}
public abstract void AddRevisionIncludes(IncludeRevisionsCommand command);
private async Task <TResult> ExecuteQuery <TResult>(TResult final, IndexQueryServerSide query, QueryOperationContext queryContext, long?existingResultEtag, OperationCancelToken token) where TResult : QueryResultServerSide <Document>
{
try
{
Database.ServerStore.FeatureGuardian.Assert(Feature.GraphApi);
using (QueryRunner.MarkQueryAsRunning(Constants.Documents.Indexing.DummyGraphIndexName, query, token))
using (var timingScope = new QueryTimingsScope())
{
var qr = await GetQueryResults(query, queryContext, existingResultEtag, token);
if (qr.NotModified)
{
final.NotModified = true;
return(final);
}
var q = query.Metadata.Query;
//TODO: handle order by, load, clauses
IncludeDocumentsCommand idc = null;
IncludeRevisionsCommand irc = null;
IncludeCompareExchangeValuesCommand icevc = null;
if (q.Select == null && q.SelectFunctionBody.FunctionText == null)
{
await HandleResultsWithoutSelectAsync(queryContext.Documents, qr.Matches, final, token.Token);
}
else if (q.Select != null)
{
//TODO : investigate fields to fetch
var fieldsToFetch = new FieldsToFetch(query, null);
idc = new IncludeDocumentsCommand(Database.DocumentsStorage, queryContext.Documents, query.Metadata.Includes, fieldsToFetch.IsProjection);
icevc = IncludeCompareExchangeValuesCommand.ExternalScope(queryContext, query.Metadata.CompareExchangeValueIncludes);
irc = new IncludeRevisionsCommand(database: Database, context: queryContext.Documents, query.Metadata.RevisionIncludes);
var resultRetriever = new GraphQueryResultRetriever(
q.GraphQuery,
Database,
query,
timingScope,
Database.DocumentsStorage,
queryContext.Documents,
fieldsToFetch,
idc,
icevc,
irc);
HashSet <ulong> alreadySeenProjections = null;
if (q.IsDistinct)
{
alreadySeenProjections = new HashSet <ulong>();
}
foreach (var match in qr.Matches)
{
if (match.Empty)
{
continue;
}
var result = resultRetriever.ProjectFromMatch(match, queryContext.Documents, token.Token);
// ReSharper disable once PossibleNullReferenceException
if (q.IsDistinct && alreadySeenProjections.Add(result.DataHash) == false)
{
continue;
}
await final.AddResultAsync(result, token.Token);
}
}
if (idc == null)
{
idc = new IncludeDocumentsCommand(Database.DocumentsStorage, queryContext.Documents, query.Metadata.Includes, isProjection: false);
}
if (query.Metadata.Includes?.Length > 0)
{
foreach (var result in final.Results)
{
idc.Gather(result);
}
}
idc.Fill(final.Includes);
final.TotalResults = final.Results.Count;
final.LongTotalResults = final.Results.Count;
if (query.Limit != null || query.Offset != null)
{
final.CappedMaxResults = Math.Min(
query.Limit ?? int.MaxValue,
final.TotalResults - (query.Offset ?? 0)
);
}
final.IsStale = qr.QueryPlan.IsStale;
final.ResultEtag = qr.QueryPlan.ResultEtag;
return(final);
}
}
catch (OperationCanceledException oce)
{
throw new OperationCanceledException($"Database:{Database} Query:{query.Metadata.Query} has been cancelled ", oce);
}
}
private async ValueTask ExecuteCollectionQueryAsync(QueryResultServerSide <Document> resultToFill, IndexQueryServerSide query, string collection, QueryOperationContext context, bool pulseReadingTransaction, CancellationToken token)
{
using (var queryScope = query.Timings?.For(nameof(QueryTimingsScope.Names.Query)))
{
QueryTimingsScope gatherScope = null;
QueryTimingsScope fillScope = null;
if (queryScope != null && query.Metadata.Includes?.Length > 0)
{
var includesScope = queryScope.For(nameof(QueryTimingsScope.Names.Includes), start: false);
gatherScope = includesScope.For(nameof(QueryTimingsScope.Names.Gather), start: false);
fillScope = includesScope.For(nameof(QueryTimingsScope.Names.Fill), start: false);
}
// we optimize for empty queries without sorting options, appending CollectionIndexPrefix to be able to distinguish index for collection vs. physical index
resultToFill.IsStale = false;
resultToFill.LastQueryTime = DateTime.MinValue;
resultToFill.IndexTimestamp = DateTime.MinValue;
resultToFill.IncludedPaths = query.Metadata.Includes;
var fieldsToFetch = new FieldsToFetch(query, null);
var includeDocumentsCommand = new IncludeDocumentsCommand(Database.DocumentsStorage, context.Documents, query.Metadata.Includes, fieldsToFetch.IsProjection);
var includeRevisionsCommand = new IncludeRevisionsCommand(Database, context.Documents, query.Metadata.RevisionIncludes);
var includeCompareExchangeValuesCommand = IncludeCompareExchangeValuesCommand.ExternalScope(context, query.Metadata.CompareExchangeValueIncludes);
var totalResults = new Reference <int>();
var skippedResults = new Reference <long>();
var scannedResults = new Reference <int>();
IEnumerator <Document> enumerator;
if (pulseReadingTransaction == false)
{
var documents = new CollectionQueryEnumerable(Database, Database.DocumentsStorage, fieldsToFetch, collection, query, queryScope, context.Documents,
includeDocumentsCommand, includeRevisionsCommand: includeRevisionsCommand,
includeCompareExchangeValuesCommand: includeCompareExchangeValuesCommand, totalResults: totalResults, scannedResults, skippedResults, token);
enumerator = documents.GetEnumerator();
}
else
{
enumerator = new PulsedTransactionEnumerator <Document, CollectionQueryResultsIterationState>(context.Documents,
state =>
{
query.Start = state.Start;
query.PageSize = state.Take;
var documents = new CollectionQueryEnumerable(Database, Database.DocumentsStorage, fieldsToFetch, collection, query, queryScope,
context.Documents, includeDocumentsCommand, includeRevisionsCommand, includeCompareExchangeValuesCommand, totalResults, scannedResults,
skippedResults, token);
return(documents);
},
new CollectionQueryResultsIterationState(context.Documents, Database.Configuration.Databases.PulseReadTransactionLimit)
{
Start = query.Start,
Take = query.PageSize
});
}
IncludeCountersCommand includeCountersCommand = null;
IncludeTimeSeriesCommand includeTimeSeriesCommand = null;
if (query.Metadata.RevisionIncludes != null)
{
includeRevisionsCommand = new IncludeRevisionsCommand(
Database,
context.Documents,
query.Metadata.RevisionIncludes);
}
if (query.Metadata.CounterIncludes != null)
{
includeCountersCommand = new IncludeCountersCommand(
Database,
context.Documents,
query.Metadata.CounterIncludes.Counters);
}
if (query.Metadata.TimeSeriesIncludes != null)
{
includeTimeSeriesCommand = new IncludeTimeSeriesCommand(
context.Documents,
query.Metadata.TimeSeriesIncludes.TimeSeries);
}
try
{
using (enumerator)
{
while (enumerator.MoveNext())
{
var document = enumerator.Current;
token.ThrowIfCancellationRequested();
await resultToFill.AddResultAsync(document, token);
using (gatherScope?.Start())
{
includeDocumentsCommand.Gather(document);
includeCompareExchangeValuesCommand?.Gather(document);
}
includeCountersCommand?.Fill(document);
includeTimeSeriesCommand?.Fill(document);
includeRevisionsCommand?.Fill(document);
}
}
}
catch (Exception e)
{
if (resultToFill.SupportsExceptionHandling == false)
{
throw;
}
await resultToFill.HandleExceptionAsync(e, token);
}
using (fillScope?.Start())
{
includeDocumentsCommand.Fill(resultToFill.Includes);
includeCompareExchangeValuesCommand.Materialize();
}
if (includeCompareExchangeValuesCommand != null)
{
resultToFill.AddCompareExchangeValueIncludes(includeCompareExchangeValuesCommand);
}
if (includeCountersCommand != null)
{
resultToFill.AddCounterIncludes(includeCountersCommand);
}
if (includeTimeSeriesCommand != null)
{
resultToFill.AddTimeSeriesIncludes(includeTimeSeriesCommand);
}
if (includeRevisionsCommand != null)
{
resultToFill.AddRevisionIncludes(includeRevisionsCommand);
}
resultToFill.RegisterTimeSeriesFields(query, fieldsToFetch);
resultToFill.TotalResults = (totalResults.Value == 0 && resultToFill.Results.Count != 0) ? -1 : totalResults.Value;
resultToFill.LongTotalResults = resultToFill.TotalResults;
resultToFill.SkippedResults = Convert.ToInt32(skippedResults.Value);
resultToFill.ScannedResults = scannedResults.Value;
if (query.Offset != null || query.Limit != null)
{
if (resultToFill.TotalResults == -1)
{
resultToFill.CappedMaxResults = query.Limit ?? -1;
}
else
{
resultToFill.CappedMaxResults = Math.Min(
query.Limit ?? int.MaxValue,
resultToFill.TotalResults - (query.Offset ?? 0)
);
}
}
}
}
