protected QueryResultRetrieverBase(DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, FieldsToFetch fieldsToFetch, DocumentsStorage documentsStorage, JsonOperationContext context, bool reduceResults, IncludeDocumentsCommand includeDocumentsCommand)
{
_database = database;
_query = query;
_context = context;
_includeDocumentsCommand = includeDocumentsCommand;
DocumentsStorage = documentsStorage;
RetrieverScope = queryTimings?.For(nameof(QueryTimingsScope.Names.Retriever), start: false);
FieldsToFetch = fieldsToFetch;
_blittableTraverser = reduceResults ? BlittableJsonTraverser.FlatMapReduceResults : BlittableJsonTraverser.Default;
}
public QueryFilter(Index index, IndexQueryServerSide query, DocumentsOperationContext documentsContext, Reference <int> skippedResults,
Reference <int> scannedDocuments, IQueryResultRetriever retriever, QueryTimingsScope queryTimings)
{
_query = query;
_documentsContext = documentsContext;
_skippedResults = skippedResults;
_scannedDocuments = scannedDocuments;
_retriever = retriever;
_queryTimings = queryTimings;
var key = new CollectionQueryEnumerable.FilterKey(query.Metadata);
_filterSingleRun = index.DocumentDatabase.Scripts.GetScriptRunner(key, readOnly: true, patchRun: out _filterScriptRun);
}
public CollectionQueryEnumerable(DocumentDatabase database, DocumentsStorage documents, FieldsToFetch fieldsToFetch, string collection,
IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext context, IncludeDocumentsCommand includeDocumentsCommand, Reference <int> totalResults)
{
_database = database;
_documents = documents;
_fieldsToFetch = fieldsToFetch;
_collection = collection;
_isAllDocsCollection = collection == Constants.Documents.Collections.AllDocumentsCollection;
_query = query;
_queryTimings = queryTimings;
_context = context;
_includeDocumentsCommand = includeDocumentsCommand;
_totalResults = totalResults;
}
public GraphQueryResultRetriever(GraphQuery graphQuery, DocumentDatabase database,
IndexQueryServerSide query,
QueryTimingsScope queryTimings,
DocumentsStorage documentsStorage,
DocumentsOperationContext context,
FieldsToFetch fieldsToFetch,
IncludeDocumentsCommand includeDocumentsCommand,
IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand,
IncludeRevisionsCommand includeRevisionsCommand)
: base(database, query, queryTimings, fieldsToFetch, documentsStorage, context, false, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand)
{
_graphQuery = graphQuery;
_context = context;
}
protected object GetFunctionValue(FieldsToFetch.FieldToFetch fieldToFetch, string documentId, object[] args)
{
using (_functionScope = _functionScope?.Start() ?? _projectionScope?.For(nameof(QueryTimingsScope.Names.JavaScript)))
{
args[args.Length - 1] = _query.QueryParameters;
var value = InvokeFunction(
fieldToFetch.QueryField.Name,
_query.Metadata.Query,
documentId,
args,
_functionScope);
return(value);
}
}
public override Document Get(Lucene.Net.Documents.Document input, ScoreDoc scoreDoc, IState state)
{
if (FieldsToFetch.IsProjection)
{
return(GetProjection(input, scoreDoc, null, state));
}
using (_storageScope = _storageScope?.Start() ?? RetrieverScope?.For(nameof(QueryTimingsScope.Names.Storage)))
{
var doc = DirectGet(input, null, DocumentFields.All, state);
FinishDocumentSetup(doc, scoreDoc);
return(doc);
}
}
public override Document Get(Lucene.Net.Documents.Document input, float score, IState state)
{
if (FieldsToFetch.IsProjection)
{
return(GetProjection(input, score, null, state));
}
using (_storageScope = _storageScope?.Start() ?? RetrieverScope?.For(nameof(QueryTimingsScope.Names.Storage)))
{
var doc = DirectGet(input, null, DocumentFields.All, state);
if (doc != null)
{
doc.IndexScore = score;
}
return(doc);
}
}
public CollectionQueryEnumerable(DocumentDatabase database, DocumentsStorage documents, FieldsToFetch fieldsToFetch, string collection,
IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext context, IncludeDocumentsCommand includeDocumentsCommand,
IncludeRevisionsCommand includeRevisionsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, Reference <int> totalResults,
Reference <int> scannedResults, Reference <long> skippedResults, CancellationToken token)
{
_database = database;
_documents = documents;
_fieldsToFetch = fieldsToFetch;
_collection = collection;
_isAllDocsCollection = collection == Constants.Documents.Collections.AllDocumentsCollection;
_query = query;
_queryTimings = queryTimings;
_context = context;
_includeDocumentsCommand = includeDocumentsCommand;
_includeRevisionsCommand = includeRevisionsCommand;
_includeCompareExchangeValuesCommand = includeCompareExchangeValuesCommand;
_totalResults = totalResults;
_token = token;
_scannedResults = scannedResults;
_skippedResults = skippedResults;
}
protected (Document Document, List <Document> List) GetProjection(Lucene.Net.Documents.Document input, Lucene.Net.Search.ScoreDoc scoreDoc, string lowerId, IState state, CancellationToken token)
{
using (_projectionScope = _projectionScope?.Start() ?? RetrieverScope?.For(nameof(QueryTimingsScope.Names.Projection)))
{
Document doc = null;
if (FieldsToFetch.AnyExtractableFromIndex == false)
{
using (_projectionStorageScope = _projectionStorageScope?.Start() ?? _projectionScope?.For(nameof(QueryTimingsScope.Names.Storage)))
doc = DirectGet(input, lowerId, DocumentFields.All, state);
if (doc == null)
{
if (FieldsToFetch.Projection.MustExtractFromDocument)
{
if (FieldsToFetch.Projection.MustExtractOrThrow)
{
FieldsToFetch.Projection.ThrowCouldNotExtractProjectionOnDocumentBecauseDocumentDoesNotExistException(lowerId);
}
}
return(default);
public Enumerator(DocumentDatabase database, DocumentsStorage documents, FieldsToFetch fieldsToFetch, string collection, bool isAllDocsCollection,
IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext context, IncludeDocumentsCommand includeDocumentsCommand, Reference <int> totalResults)
{
_documents = documents;
_fieldsToFetch = fieldsToFetch;
_collection = collection;
_isAllDocsCollection = isAllDocsCollection;
_query = query;
_context = context;
_totalResults = totalResults;
_totalResults.Value = 0;
if (_fieldsToFetch.IsDistinct)
{
_alreadySeenProjections = new HashSet <ulong>();
}
_resultsRetriever = new MapQueryResultRetriever(database, query, queryTimings, documents, context, fieldsToFetch, includeDocumentsCommand);
(_ids, _startsWith) = ExtractIdsFromQuery(query, context);
}
public override (Document Document, List <Document> List) Get(Lucene.Net.Documents.Document input, Lucene.Net.Search.ScoreDoc scoreDoc, IState state, CancellationToken token)
{
using (RetrieverScope?.Start())
{
if (TryGetKey(input, state, out string id) == false)
{
throw new InvalidOperationException($"Could not extract '{Constants.Documents.Indexing.Fields.DocumentIdFieldName}' from index.");
}
if (FieldsToFetch.IsProjection)
{
return(GetProjection(input, scoreDoc, id, state, token));
}
using (_storageScope = _storageScope?.Start() ?? RetrieverScope?.For(nameof(QueryTimingsScope.Names.Storage)))
{
var doc = DirectGet(null, id, DocumentFields, state);
FinishDocumentSetup(doc, scoreDoc);
return(doc, null);
}
}
}
public Enumerator(DocumentDatabase database, DocumentsStorage documents, FieldsToFetch fieldsToFetch, string collection, bool isAllDocsCollection,
IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext context, IncludeDocumentsCommand includeDocumentsCommand,
IncludeRevisionsCommand includeRevisionsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, Reference <int> totalResults,
Reference <int> scannedResults, string startAfterId, Reference <long> alreadySeenIdsCount, DocumentFields fields, Reference <long> skippedResults, CancellationToken token)
{
_documents = documents;
_fieldsToFetch = fieldsToFetch;
_collection = collection;
_isAllDocsCollection = isAllDocsCollection;
_query = query;
_queryTimings = queryTimings;
_context = context;
_totalResults = totalResults;
_scannedResults = scannedResults;
_totalResults.Value = 0;
_startAfterId = startAfterId;
_alreadySeenIdsCount = alreadySeenIdsCount;
_fields = fields;
_skippedResults = skippedResults;
_token = token;
if (_fieldsToFetch.IsDistinct)
{
_alreadySeenProjections = new HashSet <ulong>();
}
_resultsRetriever = new MapQueryResultRetriever(database, query, queryTimings, documents, context, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand);
(_ids, _startsWith) = ExtractIdsFromQuery(query, context);
if (_query.Metadata.FilterScript != null)
{
var key = new FilterKey(_query.Metadata);
_releaseFilterScriptRunner = database.Scripts.GetScriptRunner(key, readOnly: true, patchRun: out _filterScriptRun);
}
}
public CountersQueryResultRetriever(DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsStorage documentsStorage, JsonOperationContext context, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand)
: base(database, query, queryTimings, documentsStorage, context, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand)
{
}
public IEnumerable <(Document Result, Dictionary <string, Dictionary <string, string[]> > Highlightings, ExplanationResult Explanation)> Query(IndexQueryServerSide query, QueryTimingsScope queryTimings, FieldsToFetch fieldsToFetch, Reference <int> totalResults, Reference <int> skippedResults, IQueryResultRetriever retriever, DocumentsOperationContext documentsContext, Func <string, SpatialField> getSpatialField, CancellationToken token)
{
ExplanationOptions explanationOptions = null;
var pageSize = query.PageSize;
var isDistinctCount = pageSize == 0 && query.Metadata.IsDistinct;
if (isDistinctCount)
{
pageSize = int.MaxValue;
}
pageSize = GetPageSize(_searcher, pageSize);
var docsToGet = pageSize;
var position = query.Start;
if (query.Metadata.IsOptimizedSortOnly && _index.Definition.HasDynamicFields == false)
{
foreach (var result in QuerySortOnly(query, retriever, position, pageSize, totalResults, token))
{
yield return(result);
}
yield break;
}
QueryTimingsScope luceneScope = null;
QueryTimingsScope highlightingScope = null;
QueryTimingsScope explanationsScope = null;
if (queryTimings != null)
{
luceneScope = queryTimings.For(nameof(QueryTimingsScope.Names.Lucene), start: false);
highlightingScope = query.Metadata.HasHighlightings
? queryTimings.For(nameof(QueryTimingsScope.Names.Highlightings), start: false)
: null;
explanationsScope = query.Metadata.HasExplanations
? queryTimings.For(nameof(QueryTimingsScope.Names.Explanations), start: false)
: null;
}
var returnedResults = 0;
var luceneQuery = GetLuceneQuery(documentsContext, query.Metadata, query.QueryParameters, _analyzer, _queryBuilderFactories);
var sort = GetSort(query, _index, getSpatialField, documentsContext);
using (var scope = new IndexQueryingScope(_indexType, query, fieldsToFetch, _searcher, retriever, _state))
{
if (query.Metadata.HasHighlightings)
{
using (highlightingScope?.For(nameof(QueryTimingsScope.Names.Setup)))
SetupHighlighter(query, luceneQuery, documentsContext);
}
while (true)
{
token.ThrowIfCancellationRequested();
TopDocs search;
using (luceneScope?.Start())
search = ExecuteQuery(luceneQuery, query.Start, docsToGet, sort);
totalResults.Value = search.TotalHits;
scope.RecordAlreadyPagedItemsInPreviousPage(search);
for (; position < search.ScoreDocs.Length && pageSize > 0; position++)
{
token.ThrowIfCancellationRequested();
var scoreDoc = search.ScoreDocs[position];
global::Lucene.Net.Documents.Document document;
using (luceneScope?.Start())
document = _searcher.Doc(scoreDoc.Doc, _state);
if (retriever.TryGetKey(document, _state, out string key) && scope.WillProbablyIncludeInResults(key) == false)
{
skippedResults.Value++;
continue;
}
var result = retriever.Get(document, scoreDoc.Score, _state);
if (scope.TryIncludeInResults(result) == false)
{
skippedResults.Value++;
continue;
}
returnedResults++;
if (isDistinctCount == false)
{
Dictionary <string, Dictionary <string, string[]> > highlightings = null;
if (query.Metadata.HasHighlightings)
{
using (highlightingScope?.Start())
highlightings = GetHighlighterResults(query, _searcher, scoreDoc, result, document, documentsContext);
}
ExplanationResult explanation = null;
if (query.Metadata.HasExplanations)
{
using (explanationsScope?.Start())
{
if (explanationOptions == null)
{
explanationOptions = query.Metadata.Explanation.GetOptions(documentsContext, query.QueryParameters);
}
explanation = GetQueryExplanations(explanationOptions, luceneQuery, _searcher, scoreDoc, result, document);
}
}
yield return(result, highlightings, explanation);
}
if (returnedResults == pageSize)
{
yield break;
}
}
if (search.TotalHits == search.ScoreDocs.Length)
{
break;
}
if (returnedResults >= pageSize)
{
break;
}
Debug.Assert(_maxNumberOfOutputsPerDocument > 0);
docsToGet += GetPageSize(_searcher, (long)(pageSize - returnedResults) * _maxNumberOfOutputsPerDocument);
}
if (isDistinctCount)
{
totalResults.Value = returnedResults;
}
}
}
public override IQueryResultRetriever GetQueryResultRetriever(IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext documentsContext, FieldsToFetch fieldsToFetch,
IncludeDocumentsCommand includeDocumentsCommand)
{
throw new NotImplementedException();
}
protected QueryFilter GetQueryFilter(Index index, IndexQueryServerSide query, DocumentsOperationContext documentsContext, Reference <int> skippedResults, Reference <int> scannedDocuments, IQueryResultRetriever retriever, QueryTimingsScope queryTimings)
{
if (query.Metadata.FilterScript is null)
{
return(null);
}
return(new QueryFilter(index, query, documentsContext, skippedResults, scannedDocuments, retriever, queryTimings));
}
private void ExecuteCollectionQuery(QueryResultServerSide <Document> resultToFill, IndexQueryServerSide query, string collection, DocumentsOperationContext context, bool pulseReadingTransaction, CancellationToken cancellationToken)
{
using (var queryScope = query.Timings?.For(nameof(QueryTimingsScope.Names.Query)))
{
QueryTimingsScope gatherScope = null;
QueryTimingsScope fillScope = null;
if (queryScope != null && query.Metadata.Includes?.Length > 0)
{
var includesScope = queryScope.For(nameof(QueryTimingsScope.Names.Includes), start: false);
gatherScope = includesScope.For(nameof(QueryTimingsScope.Names.Gather), start: false);
fillScope = includesScope.For(nameof(QueryTimingsScope.Names.Fill), start: false);
}
// we optimize for empty queries without sorting options, appending CollectionIndexPrefix to be able to distinguish index for collection vs. physical index
resultToFill.IsStale = false;
resultToFill.LastQueryTime = DateTime.MinValue;
resultToFill.IndexTimestamp = DateTime.MinValue;
resultToFill.IncludedPaths = query.Metadata.Includes;
var fieldsToFetch = new FieldsToFetch(query, null);
var includeDocumentsCommand = new IncludeDocumentsCommand(Database.DocumentsStorage, context, query.Metadata.Includes, fieldsToFetch.IsProjection);
var totalResults = new Reference <int>();
IEnumerator <Document> enumerator;
if (pulseReadingTransaction == false)
{
var documents = new CollectionQueryEnumerable(Database, Database.DocumentsStorage, fieldsToFetch, collection, query, queryScope, context, includeDocumentsCommand, totalResults);
enumerator = documents.GetEnumerator();
}
else
{
enumerator = new PulsedTransactionEnumerator <Document, CollectionQueryResultsIterationState>(context,
state =>
{
query.Start = state.Start;
query.PageSize = state.Take;
var documents = new CollectionQueryEnumerable(Database, Database.DocumentsStorage, fieldsToFetch, collection, query, queryScope, context,
includeDocumentsCommand, totalResults);
return(documents);
},
new CollectionQueryResultsIterationState(context, Database.Configuration.Databases.PulseReadTransactionLimit)
{
Start = query.Start,
Take = query.PageSize
});
}
IncludeCountersCommand includeCountersCommand = null;
if (query.Metadata.CounterIncludes != null)
{
includeCountersCommand = new IncludeCountersCommand(
Database,
context,
query.Metadata.CounterIncludes.Counters);
}
try
{
using (enumerator)
{
while (enumerator.MoveNext())
{
var document = enumerator.Current;
cancellationToken.ThrowIfCancellationRequested();
resultToFill.AddResult(document);
using (gatherScope?.Start())
includeDocumentsCommand.Gather(document);
includeCountersCommand?.Fill(document);
}
}
}
catch (Exception e)
{
if (resultToFill.SupportsExceptionHandling == false)
{
throw;
}
resultToFill.HandleException(e);
}
using (fillScope?.Start())
includeDocumentsCommand.Fill(resultToFill.Includes);
if (includeCountersCommand != null)
{
resultToFill.AddCounterIncludes(includeCountersCommand);
}
resultToFill.TotalResults = (totalResults.Value == 0 && resultToFill.Results.Count != 0) ? -1 : totalResults.Value;
if (query.Offset != null || query.Limit != null)
{
if (resultToFill.TotalResults == -1)
{
resultToFill.CappedMaxResults = query.Limit ?? -1;
}
else
{
resultToFill.CappedMaxResults = Math.Min(
query.Limit ?? int.MaxValue,
resultToFill.TotalResults - (query.Offset ?? 0)
);
}
}
}
}
private object InvokeFunction(string methodName, Query query, string documentId, object[] args, QueryTimingsScope timings)
{
if (query.DeclaredFunctions != null &&
query.DeclaredFunctions.TryGetValue(methodName, out var func) &&
func.Type == DeclaredFunction.FunctionType.TimeSeries)
{
_timeSeriesRetriever ??= new TimeSeriesRetriever(_includeDocumentsCommand.Context, _query.QueryParameters, _loadedDocuments, _query.IsFromStudio);
return(_timeSeriesRetriever.InvokeTimeSeriesFunction(func, documentId, args, addProjectionToResult: FieldsToFetch.SingleBodyOrMethodWithNoAlias));
}
var key = new QueryKey(query.DeclaredFunctions);
using (_database.Scripts.GetScriptRunner(key, readOnly: true, patchRun: out var run))
using (var result = run.Run(_context, _context as DocumentsOperationContext, methodName, args, timings))
{
_includeDocumentsCommand?.AddRange(run.Includes, documentId);
_includeCompareExchangeValuesCommand?.AddRange(run.CompareExchangeValueIncludes);
if (result.IsNull)
{
return(null);
}
return(run.Translate(result, _context, QueryResultModifier.Instance));
}
}
private void ExecuteCollectionQuery(QueryResultServerSide <Document> resultToFill, IndexQueryServerSide query, string collection, DocumentsOperationContext context, CancellationToken cancellationToken)
{
using (var queryScope = query.Timings?.For(nameof(QueryTimingsScope.Names.Query)))
{
QueryTimingsScope gatherScope = null;
QueryTimingsScope fillScope = null;
if (queryScope != null && query.Metadata.Includes?.Length > 0)
{
var includesScope = queryScope.For(nameof(QueryTimingsScope.Names.Includes), start: false);
gatherScope = includesScope.For(nameof(QueryTimingsScope.Names.Gather), start: false);
fillScope = includesScope.For(nameof(QueryTimingsScope.Names.Fill), start: false);
}
var isAllDocsCollection = collection == Constants.Documents.Collections.AllDocumentsCollection;
// we optimize for empty queries without sorting options, appending CollectionIndexPrefix to be able to distinguish index for collection vs. physical index
resultToFill.IndexName = isAllDocsCollection ? "AllDocs" : CollectionIndexPrefix + collection;
resultToFill.IsStale = false;
resultToFill.LastQueryTime = DateTime.MinValue;
resultToFill.IndexTimestamp = DateTime.MinValue;
resultToFill.IncludedPaths = query.Metadata.Includes;
var fieldsToFetch = new FieldsToFetch(query, null);
var includeDocumentsCommand = new IncludeDocumentsCommand(Database.DocumentsStorage, context, query.Metadata.Includes, fieldsToFetch.IsProjection);
var totalResults = new Reference <int>();
var documents = new CollectionQueryEnumerable(Database, Database.DocumentsStorage, fieldsToFetch, collection, query, queryScope, context, includeDocumentsCommand, totalResults);
IncludeCountersCommand includeCountersCommand = null;
if (query.Metadata.HasCounters)
{
includeCountersCommand = new IncludeCountersCommand(
Database,
context,
query.Metadata.CounterIncludes.Counters);
}
try
{
foreach (var document in documents)
{
cancellationToken.ThrowIfCancellationRequested();
resultToFill.AddResult(document);
using (gatherScope?.Start())
includeDocumentsCommand.Gather(document);
includeCountersCommand?.Fill(document);
}
}
catch (Exception e)
{
if (resultToFill.SupportsExceptionHandling == false)
{
throw;
}
resultToFill.HandleException(e);
}
using (fillScope?.Start())
includeDocumentsCommand.Fill(resultToFill.Includes);
if (includeCountersCommand != null)
{
resultToFill.AddCounterIncludes(includeCountersCommand);
}
resultToFill.TotalResults = (totalResults.Value == 0 && resultToFill.Results.Count != 0) ? -1 : totalResults.Value;
}
}
protected StoredValueQueryResultRetriever(string storedValueFieldName, DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsStorage documentsStorage, JsonOperationContext context, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand)
: base(database, query, queryTimings, fieldsToFetch, documentsStorage, context, true, includeDocumentsCommand, includeCompareExchangeValuesCommand)
{
if (storedValueFieldName == null)
{
throw new ArgumentNullException(nameof(storedValueFieldName));
}
_storedValueFieldName = storedValueFieldName;
_context = context;
}
public MapReduceQueryResultRetriever(DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsStorage documentsStorage, JsonOperationContext context, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand)
: base(Constants.Documents.Indexing.Fields.ReduceKeyValueFieldName, database, query, queryTimings, documentsStorage, context, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand)
{
}
private async Task <TResult> ExecuteQuery <TResult>(TResult final, IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token) where TResult : QueryResultServerSide <Document>
{
try
{
if (Database.ServerStore.Configuration.Core.FeaturesAvailability == FeaturesAvailability.Stable)
{
FeaturesAvailabilityException.Throw("Graph Queries");
}
using (QueryRunner.MarkQueryAsRunning(Constants.Documents.Indexing.DummyGraphIndexName, query, token))
using (var timingScope = new QueryTimingsScope())
{
var qr = await GetQueryResults(query, documentsContext, existingResultEtag, token);
if (qr.NotModified)
{
final.NotModified = true;
return(final);
}
var q = query.Metadata.Query;
//TODO: handle order by, load, clauses
IncludeDocumentsCommand idc = null;
if (q.Select == null && q.SelectFunctionBody.FunctionText == null)
{
HandleResultsWithoutSelect(documentsContext, qr.Matches, final);
}
else if (q.Select != null)
{
//TODO : investigate fields to fetch
var fieldsToFetch = new FieldsToFetch(query, null);
idc = new IncludeDocumentsCommand(Database.DocumentsStorage, documentsContext, query.Metadata.Includes, fieldsToFetch.IsProjection);
var resultRetriever = new GraphQueryResultRetriever(
q.GraphQuery,
Database,
query,
timingScope,
Database.DocumentsStorage,
documentsContext,
fieldsToFetch,
idc);
HashSet <ulong> alreadySeenProjections = null;
if (q.IsDistinct)
{
alreadySeenProjections = new HashSet <ulong>();
}
foreach (var match in qr.Matches)
{
if (match.Empty)
{
continue;
}
var result = resultRetriever.ProjectFromMatch(match, documentsContext);
// ReSharper disable once PossibleNullReferenceException
if (q.IsDistinct && alreadySeenProjections.Add(result.DataHash) == false)
{
continue;
}
final.AddResult(result);
}
}
if (idc == null)
{
idc = new IncludeDocumentsCommand(Database.DocumentsStorage, documentsContext, query.Metadata.Includes, isProjection: false);
}
if (query.Metadata.Includes?.Length > 0)
{
foreach (var result in final.Results)
{
idc.Gather(result);
}
}
idc.Fill(final.Includes);
final.TotalResults = final.Results.Count;
if (query.Limit != null || query.Offset != null)
{
final.CappedMaxResults = Math.Min(
query.Limit ?? int.MaxValue,
final.TotalResults - (query.Offset ?? 0)
);
}
final.IsStale = qr.QueryPlan.IsStale;
final.ResultEtag = qr.QueryPlan.ResultEtag;
return(final);
}
}
catch (OperationCanceledException oce)
{
throw new OperationCanceledException($"Database:{Database} Query:{query.Metadata.Query} has been cancelled ", oce);
}
}
protected bool TryGetValue(FieldsToFetch.FieldToFetch fieldToFetch, Document document, Lucene.Net.Documents.Document luceneDoc, IState state, Dictionary <string, IndexField> indexFields, bool?anyDynamicIndexFields, out string key, out object value)
{
key = fieldToFetch.ProjectedName ?? fieldToFetch.Name.Value;
if (fieldToFetch.QueryField == null)
{
return(TryGetFieldValueFromDocument(document, fieldToFetch, out value));
}
if (fieldToFetch.QueryField.Function != null)
{
var args = new object[fieldToFetch.QueryField.FunctionArgs.Length + 1];
for (int i = 0; i < fieldToFetch.FunctionArgs.Length; i++)
{
TryGetValue(fieldToFetch.FunctionArgs[i], document, luceneDoc, state, indexFields, anyDynamicIndexFields, out _, out args[i]);
if (ReferenceEquals(args[i], document))
{
args[i] = Tuple.Create(document, luceneDoc, state, indexFields, anyDynamicIndexFields);
}
}
value = GetFunctionValue(fieldToFetch, document.Id, args);
return(true);
}
if (fieldToFetch.QueryField.IsCounter)
{
string name;
string id = document.Id;
if (fieldToFetch.QueryField.IsParameter)
{
if (_query.QueryParameters == null)
{
throw new InvalidQueryException("The query is parametrized but the actual values of parameters were not provided", _query.Query, null);
}
if (_query.QueryParameters.TryGetMember(fieldToFetch.QueryField.Name, out var nameObject) == false)
{
throw new InvalidQueryException($"Value of parameter '{fieldToFetch.QueryField.Name}' was not provided", _query.Query, _query.QueryParameters);
}
name = nameObject.ToString();
key = fieldToFetch.QueryField.Alias ?? name;
}
else
{
name = fieldToFetch.Name.Value;
}
if (fieldToFetch.QueryField.SourceAlias != null &&
BlittableJsonTraverser.Default.TryRead(document.Data, fieldToFetch.QueryField.SourceAlias, out var sourceId, out _))
{
id = sourceId.ToString();
}
if (fieldToFetch.QueryField.FunctionArgs != null)
{
value = GetCounterRaw(id, name);
}
else
{
value = GetCounter(id, name);
}
return(true);
}
if (fieldToFetch.QueryField.ValueTokenType != null)
{
var val = fieldToFetch.QueryField.Value;
if (fieldToFetch.QueryField.ValueTokenType.Value == ValueTokenType.Parameter)
{
if (_query == null)
{
value = null;
return(false); // only happens for debug endpoints and more like this
}
_query.QueryParameters.TryGet((string)val, out val);
}
value = val;
return(true);
}
if (fieldToFetch.QueryField.HasSourceAlias == false)
{
return(TryGetFieldValueFromDocument(document, fieldToFetch, out value));
}
if (_loadedDocumentIds == null)
{
_loadedDocumentIds = new HashSet <string>();
_loadedDocuments = new Dictionary <string, Document>();
_loadedDocumentsByAliasName = new Dictionary <string, Document>();
}
_loadedDocumentIds.Clear();
//_loadedDocuments.Clear(); - explicitly not clearing this, we want to cache this for the duration of the query
_loadedDocuments[document.Id ?? string.Empty] = document;
if (fieldToFetch.QueryField.SourceAlias != null)
{
if (fieldToFetch.QueryField.IsQuoted)
{
_loadedDocumentIds.Add(fieldToFetch.QueryField.SourceAlias);
}
else if (fieldToFetch.QueryField.IsParameter)
{
if (_query.QueryParameters == null)
{
throw new InvalidQueryException("The query is parametrized but the actual values of parameters were not provided", _query.Query, (BlittableJsonReaderObject)null);
}
if (_query.QueryParameters.TryGetMember(fieldToFetch.QueryField.SourceAlias, out var id) == false)
{
throw new InvalidQueryException($"Value of parameter '{fieldToFetch.QueryField.SourceAlias}' was not provided", _query.Query, _query.QueryParameters);
}
_loadedDocumentIds.Add(id.ToString());
}
else if (fieldToFetch.QueryField.LoadFromAlias != null)
{
if (_loadedDocumentsByAliasName.TryGetValue(fieldToFetch.QueryField.LoadFromAlias, out var loadedDoc))
{
IncludeUtil.GetDocIdFromInclude(loadedDoc.Data, fieldToFetch.QueryField.SourceAlias, _loadedDocumentIds, _database.IdentityPartsSeparator);
}
}
else if (fieldToFetch.CanExtractFromIndex)
{
if (luceneDoc != null)
{
var field = luceneDoc.GetField(fieldToFetch.QueryField.SourceAlias);
if (field != null)
{
var fieldValue = ConvertType(_context, field, GetFieldType(field.Name, luceneDoc), state);
_loadedDocumentIds.Add(fieldValue.ToString());
}
}
}
else
{
IncludeUtil.GetDocIdFromInclude(document.Data, fieldToFetch.QueryField.SourceAlias, _loadedDocumentIds, _database.IdentityPartsSeparator);
}
}
else
{
_loadedDocumentIds.Add(document.Id ?? string.Empty); // null source alias is the root doc
_loadedDocumentsByAliasName.Clear();
}
if (_loadedDocumentIds.Count == 0)
{
if (fieldToFetch.QueryField.SourceIsArray)
{
value = new List <object>();
return(true);
}
value = null;
return(false);
}
var buffer = new List <object>();
foreach (var docId in _loadedDocumentIds)
{
if (docId == null)
{
continue;
}
if (_loadedDocuments.TryGetValue(docId, out var doc) == false)
{
using (_loadScope = _loadScope?.Start() ?? _projectionScope?.For(nameof(QueryTimingsScope.Names.Load)))
_loadedDocuments[docId] = doc = LoadDocument(docId);
}
if (doc == null)
{
continue;
}
if (fieldToFetch.QueryField.Alias != null)
{
_loadedDocumentsByAliasName[fieldToFetch.QueryField.Alias] = doc;
}
if (string.IsNullOrEmpty(fieldToFetch.Name.Value)) // we need the whole document here
{
buffer.Add(doc);
continue;
}
if (TryGetFieldValueFromDocument(doc, fieldToFetch, out var val))
{
if (val is string == false && val is LazyStringValue == false && val is System.Collections.IEnumerable items)
{
// we flatten arrays in projections
foreach (var item in items)
{
buffer.Add(item);
}
fieldToFetch.QueryField.SourceIsArray = true;
}
else
{
buffer.Add(val);
}
}
}
if (fieldToFetch.QueryField.SourceIsArray)
{
value = buffer;
return(true);
}
if (buffer.Count > 0)
{
if (buffer.Count > 1)
{
ThrowOnlyArrayFieldCanHaveMultipleValues(fieldToFetch);
}
value = buffer[0];
return(true);
}
value = null;
return(false);
}
public override IQueryResultRetriever GetQueryResultRetriever(IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext documentsContext, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand)
{
return(new TimeSeriesQueryResultRetriever(DocumentDatabase, query, queryTimings, DocumentDatabase.DocumentsStorage, documentsContext, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand));
}
public IEnumerable <QueryResult> Query(IndexQueryServerSide query, QueryTimingsScope queryTimings, FieldsToFetch fieldsToFetch, Reference <int> totalResults, Reference <int> skippedResults, Reference <int> scannedDocuments, IQueryResultRetriever retriever, DocumentsOperationContext documentsContext, Func <string, SpatialField> getSpatialField, CancellationToken token)
{
ExplanationOptions explanationOptions = null;
var pageSize = query.PageSize;
var isDistinctCount = pageSize == 0 && query.Metadata.IsDistinct;
if (isDistinctCount)
{
pageSize = int.MaxValue;
}
pageSize = GetPageSize(_searcher, pageSize);
var docsToGet = pageSize;
var position = query.Start;
QueryTimingsScope luceneScope = null;
QueryTimingsScope highlightingScope = null;
QueryTimingsScope explanationsScope = null;
if (queryTimings != null)
{
luceneScope = queryTimings.For(nameof(QueryTimingsScope.Names.Lucene), start: false);
highlightingScope = query.Metadata.HasHighlightings
? queryTimings.For(nameof(QueryTimingsScope.Names.Highlightings), start: false)
: null;
explanationsScope = query.Metadata.HasExplanations
? queryTimings.For(nameof(QueryTimingsScope.Names.Explanations), start: false)
: null;
}
var returnedResults = 0;
var luceneQuery = GetLuceneQuery(documentsContext, query.Metadata, query.QueryParameters, _analyzer, _queryBuilderFactories);
using (var queryFilter = GetQueryFilter(_index, query, documentsContext, skippedResults, scannedDocuments, retriever, queryTimings))
using (GetSort(query, _index, getSpatialField, documentsContext, out var sort))
using (var scope = new IndexQueryingScope(_indexType, query, fieldsToFetch, _searcher, retriever, _state))
{
if (query.Metadata.HasHighlightings)
{
using (highlightingScope?.For(nameof(QueryTimingsScope.Names.Setup)))
SetupHighlighter(query, luceneQuery, documentsContext);
}
while (true)
{
token.ThrowIfCancellationRequested();
TopDocs search;
using (luceneScope?.Start())
search = ExecuteQuery(luceneQuery, query.Start, docsToGet, sort);
totalResults.Value = search.TotalHits;
scope.RecordAlreadyPagedItemsInPreviousPage(search, token);
for (; position < search.ScoreDocs.Length && pageSize > 0; position++)
{
token.ThrowIfCancellationRequested();
var scoreDoc = search.ScoreDocs[position];
global::Lucene.Net.Documents.Document document;
using (luceneScope?.Start())
document = _searcher.Doc(scoreDoc.Doc, _state);
if (retriever.TryGetKey(document, _state, out string key) && scope.WillProbablyIncludeInResults(key) == false)
{
skippedResults.Value++;
continue;
}
var filterResult = queryFilter?.Apply(document, key, _state);
if (filterResult is not null and not FilterResult.Accepted)
{
if (filterResult is FilterResult.Skipped)
{
continue;
}
if (filterResult is FilterResult.LimitReached)
{
break;
}
}
bool markedAsSkipped = false;
var r = retriever.Get(document, scoreDoc, _state, token);
if (r.Document != null)
{
var qr = CreateQueryResult(r.Document);
if (qr.Result == null)
{
continue;
}
yield return(qr);
}
else if (r.List != null)
{
int numberOfProjectedResults = 0;
foreach (Document item in r.List)
{
var qr = CreateQueryResult(item);
if (qr.Result == null)
{
continue;
}
yield return(qr);
numberOfProjectedResults++;
}
if (numberOfProjectedResults > 1)
{
totalResults.Value += numberOfProjectedResults - 1;
}
}
else
{
skippedResults.Value++;
}
QueryResult CreateQueryResult(Document d)
{
if (scope.TryIncludeInResults(d) == false)
{
d?.Dispose();
if (markedAsSkipped == false)
{
skippedResults.Value++;
markedAsSkipped = true;
}
return(default);
protected Document GetProjection(Lucene.Net.Documents.Document input, Lucene.Net.Search.ScoreDoc scoreDoc, string lowerId, IState state)
{
using (_projectionScope = _projectionScope?.Start() ?? RetrieverScope?.For(nameof(QueryTimingsScope.Names.Projection)))
{
Document doc = null;
if (FieldsToFetch.AnyExtractableFromIndex == false)
{
using (_projectionStorageScope = _projectionStorageScope?.Start() ?? _projectionScope?.For(nameof(QueryTimingsScope.Names.Storage)))
doc = DirectGet(input, lowerId, DocumentFields.All, state);
if (doc == null)
{
return(null);
}
return(GetProjectionFromDocumentInternal(doc, input, scoreDoc, FieldsToFetch, _context, state));
}
var documentLoaded = false;
var result = new DynamicJsonValue();
Dictionary <string, FieldsToFetch.FieldToFetch> fields = null;
if (FieldsToFetch.ExtractAllFromIndex)
{
fields = input.GetFields()
.Where(x => x.Name != Constants.Documents.Indexing.Fields.DocumentIdFieldName &&
x.Name != Constants.Documents.Indexing.Fields.SourceDocumentIdFieldName &&
x.Name != Constants.Documents.Indexing.Fields.ReduceKeyHashFieldName &&
x.Name != Constants.Documents.Indexing.Fields.ReduceKeyValueFieldName &&
x.Name != Constants.Documents.Indexing.Fields.ValueFieldName &&
FieldUtil.GetRangeTypeFromFieldName(x.Name) == RangeType.None)
.Distinct(UniqueFieldNames.Instance)
.ToDictionary(x => x.Name, x => new FieldsToFetch.FieldToFetch(x.Name, null, null, x.IsStored, isDocumentId: false, isTimeSeries: false));
}
if (fields == null)
{
fields = FieldsToFetch.Fields;
}
else if (FieldsToFetch.Fields != null && FieldsToFetch.Fields.Count > 0)
{
foreach (var kvp in FieldsToFetch.Fields)
{
if (fields.ContainsKey(kvp.Key))
{
continue;
}
fields[kvp.Key] = kvp.Value;
}
}
foreach (var fieldToFetch in fields.Values)
{
if (TryExtractValueFromIndex(fieldToFetch, input, result, state))
{
continue;
}
if (documentLoaded == false)
{
using (_projectionStorageScope = _projectionStorageScope?.Start() ?? _projectionScope?.For(nameof(QueryTimingsScope.Names.Storage)))
doc = DirectGet(input, lowerId, DocumentFields.All, state);
documentLoaded = true;
}
if (doc == null)
{
continue;
}
if (TryGetValue(fieldToFetch, doc, input, state, FieldsToFetch.IndexFields, FieldsToFetch.AnyDynamicIndexFields, out var key, out var fieldVal))
{
if (FieldsToFetch.SingleBodyOrMethodWithNoAlias)
{
if (fieldVal is BlittableJsonReaderObject nested)
{
doc.Data = nested;
}
else if (fieldVal is Document d)
{
doc = d;
}
else
{
ThrowInvalidQueryBodyResponse(fieldVal);
}
FinishDocumentSetup(doc, scoreDoc);
return(doc);
}
if (fieldVal is List <object> list)
{
fieldVal = new DynamicJsonArray(list);
}
if (fieldVal is Document d2)
{
fieldVal = d2.Data;
}
result[key] = fieldVal;
}
}
if (doc == null)
{
doc = new Document
{
Id = _context.GetLazyString(lowerId)
};
}
return(ReturnProjection(result, doc, scoreDoc, _context));
}
}
public override IQueryResultRetriever GetQueryResultRetriever(IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext documentsContext, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand)
{
throw new NotSupportedException($"Index {Name} is in-memory implementation of a faulty index", _e);
}
public override IQueryResultRetriever GetQueryResultRetriever(IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext documentsContext, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand)
{
return(new MapQueryResultRetriever(DocumentDatabase, query, queryTimings, DocumentDatabase.DocumentsStorage, documentsContext, fieldsToFetch, includeDocumentsCommand));
}
public MapQueryResultRetriever(DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsStorage documentsStorage, DocumentsOperationContext context, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand)
: base(database, query, queryTimings, fieldsToFetch, documentsStorage, context, false, includeDocumentsCommand)
{
_context = context;
}
