public Enumerator(DocumentDatabase database, DocumentsStorage documents, FieldsToFetch fieldsToFetch, string collection, bool isAllDocsCollection,
IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext context, IncludeDocumentsCommand includeDocumentsCommand,
IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, Reference <int> totalResults, long?queryOperationInternalStart,
DocumentFields fields)
{
_documents = documents;
_fieldsToFetch = fieldsToFetch;
_collection = collection;
_isAllDocsCollection = isAllDocsCollection;
_query = query;
_context = context;
_totalResults = totalResults;
_totalResults.Value = 0;
_queryOperationInternalStart = queryOperationInternalStart;
_fields = fields;
if (_fieldsToFetch.IsDistinct)
{
_alreadySeenProjections = new HashSet <ulong>();
}
_resultsRetriever = new MapQueryResultRetriever(database, query, queryTimings, documents, context, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand);
(_ids, _startsWith) = ExtractIdsFromQuery(query, context);
}
public override void AddCompareExchangeValueIncludes(IncludeCompareExchangeValuesCommand command)
{
if (command.Results == null)
{
return;
}
throw new NotSupportedException();
}
public GraphQueryResultRetriever(GraphQuery graphQuery, DocumentDatabase database,
IndexQueryServerSide query,
QueryTimingsScope queryTimings,
DocumentsStorage documentsStorage,
DocumentsOperationContext context,
FieldsToFetch fieldsToFetch,
IncludeDocumentsCommand includeDocumentsCommand,
IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand)
: base(database, query, queryTimings, fieldsToFetch, documentsStorage, context, false, includeDocumentsCommand, includeCompareExchangeValuesCommand)
{
_graphQuery = graphQuery;
_context = context;
}
private void GetCompareExchangeValueQueryString(DocumentDatabase database, out IncludeCompareExchangeValuesCommand includeCompareExchangeValues)
{
includeCompareExchangeValues = null;
var compareExchangeValues = GetStringValuesQueryString("cmpxchg", required: false);
if (compareExchangeValues.Count == 0)
{
return;
}
includeCompareExchangeValues = IncludeCompareExchangeValuesCommand.InternalScope(database, compareExchangeValues);
}
public CollectionQueryEnumerable(DocumentDatabase database, DocumentsStorage documents, FieldsToFetch fieldsToFetch, string collection,
IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext context, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, Reference <int> totalResults)
{
_database = database;
_documents = documents;
_fieldsToFetch = fieldsToFetch;
_collection = collection;
_isAllDocsCollection = collection == Constants.Documents.Collections.AllDocumentsCollection;
_query = query;
_queryTimings = queryTimings;
_context = context;
_includeDocumentsCommand = includeDocumentsCommand;
_includeCompareExchangeValuesCommand = includeCompareExchangeValuesCommand;
_totalResults = totalResults;
}
protected QueryResultRetrieverBase(DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, FieldsToFetch fieldsToFetch, DocumentsStorage documentsStorage, JsonOperationContext context, bool reduceResults, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand)
{
_database = database;
_query = query;
_context = context;
_includeDocumentsCommand = includeDocumentsCommand;
_includeCompareExchangeValuesCommand = includeCompareExchangeValuesCommand;
DocumentsStorage = documentsStorage;
RetrieverScope = queryTimings?.For(nameof(QueryTimingsScope.Names.Retriever), start: false);
FieldsToFetch = fieldsToFetch;
DocumentFields = query?.DocumentFields ?? DocumentFields.All;
_blittableTraverser = reduceResults ? BlittableJsonTraverser.FlatMapReduceResults : BlittableJsonTraverser.Default;
}
public Enumerator(DocumentDatabase database, DocumentsStorage documents, FieldsToFetch fieldsToFetch, string collection, bool isAllDocsCollection,
IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext context, IncludeDocumentsCommand includeDocumentsCommand,
IncludeRevisionsCommand includeRevisionsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, Reference <int> totalResults,
Reference <int> scannedResults, string startAfterId, Reference <long> alreadySeenIdsCount, DocumentFields fields, Reference <long> skippedResults, CancellationToken token)
{
_documents = documents;
_fieldsToFetch = fieldsToFetch;
_collection = collection;
_isAllDocsCollection = isAllDocsCollection;
_query = query;
_queryTimings = queryTimings;
_context = context;
_totalResults = totalResults;
_scannedResults = scannedResults;
_totalResults.Value = 0;
_startAfterId = startAfterId;
_alreadySeenIdsCount = alreadySeenIdsCount;
_fields = fields;
_skippedResults = skippedResults;
_token = token;
if (_fieldsToFetch.IsDistinct)
{
_alreadySeenProjections = new HashSet <ulong>();
}
_resultsRetriever = new MapQueryResultRetriever(database, query, queryTimings, documents, context, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand);
(_ids, _startsWith) = ExtractIdsFromQuery(query, context);
if (_query.Metadata.FilterScript != null)
{
var key = new FilterKey(_query.Metadata);
_releaseFilterScriptRunner = database.Scripts.GetScriptRunner(key, readOnly: true, patchRun: out _filterScriptRun);
}
}
private void ExecuteCollectionQuery(QueryResultServerSide <Document> resultToFill, IndexQueryServerSide query, string collection, QueryOperationContext context, bool pulseReadingTransaction, CancellationToken cancellationToken)
{
using (var queryScope = query.Timings?.For(nameof(QueryTimingsScope.Names.Query)))
{
QueryTimingsScope gatherScope = null;
QueryTimingsScope fillScope = null;
if (queryScope != null && query.Metadata.Includes?.Length > 0)
{
var includesScope = queryScope.For(nameof(QueryTimingsScope.Names.Includes), start: false);
gatherScope = includesScope.For(nameof(QueryTimingsScope.Names.Gather), start: false);
fillScope = includesScope.For(nameof(QueryTimingsScope.Names.Fill), start: false);
}
// we optimize for empty queries without sorting options, appending CollectionIndexPrefix to be able to distinguish index for collection vs. physical index
resultToFill.IsStale = false;
resultToFill.LastQueryTime = DateTime.MinValue;
resultToFill.IndexTimestamp = DateTime.MinValue;
resultToFill.IncludedPaths = query.Metadata.Includes;
var fieldsToFetch = new FieldsToFetch(query, null);
var includeDocumentsCommand = new IncludeDocumentsCommand(Database.DocumentsStorage, context.Documents, query.Metadata.Includes, fieldsToFetch.IsProjection);
var includeCompareExchangeValuesCommand = IncludeCompareExchangeValuesCommand.ExternalScope(context, query.Metadata.CompareExchangeValueIncludes);
var totalResults = new Reference <int>();
IEnumerator <Document> enumerator;
if (pulseReadingTransaction == false)
{
var documents = new CollectionQueryEnumerable(Database, Database.DocumentsStorage, fieldsToFetch, collection, query, queryScope, context.Documents, includeDocumentsCommand, includeCompareExchangeValuesCommand, totalResults);
enumerator = documents.GetEnumerator();
}
else
{
enumerator = new PulsedTransactionEnumerator <Document, CollectionQueryResultsIterationState>(context.Documents,
state =>
{
query.Start = state.Start;
query.PageSize = state.Take;
var documents = new CollectionQueryEnumerable(Database, Database.DocumentsStorage, fieldsToFetch, collection, query, queryScope, context.Documents, includeDocumentsCommand, includeCompareExchangeValuesCommand, totalResults);
return(documents);
},
new CollectionQueryResultsIterationState(context.Documents, Database.Configuration.Databases.PulseReadTransactionLimit)
{
Start = query.Start,
Take = query.PageSize
});
}
IncludeCountersCommand includeCountersCommand = null;
IncludeTimeSeriesCommand includeTimeSeriesCommand = null;
if (query.Metadata.CounterIncludes != null)
{
includeCountersCommand = new IncludeCountersCommand(
Database,
context.Documents,
query.Metadata.CounterIncludes.Counters);
}
if (query.Metadata.TimeSeriesIncludes != null)
{
includeTimeSeriesCommand = new IncludeTimeSeriesCommand(
context.Documents,
query.Metadata.TimeSeriesIncludes.TimeSeries);
}
try
{
using (enumerator)
{
while (enumerator.MoveNext())
{
var document = enumerator.Current;
cancellationToken.ThrowIfCancellationRequested();
resultToFill.AddResult(document);
using (gatherScope?.Start())
{
includeDocumentsCommand.Gather(document);
includeCompareExchangeValuesCommand?.Gather(document);
}
includeCountersCommand?.Fill(document);
includeTimeSeriesCommand?.Fill(document);
}
}
}
catch (Exception e)
{
if (resultToFill.SupportsExceptionHandling == false)
{
throw;
}
resultToFill.HandleException(e);
}
using (fillScope?.Start())
{
includeDocumentsCommand.Fill(resultToFill.Includes);
includeCompareExchangeValuesCommand.Materialize();
}
if (includeCompareExchangeValuesCommand != null)
{
resultToFill.AddCompareExchangeValueIncludes(includeCompareExchangeValuesCommand);
}
if (includeCountersCommand != null)
{
resultToFill.AddCounterIncludes(includeCountersCommand);
}
if (includeTimeSeriesCommand != null)
{
resultToFill.AddTimeSeriesIncludes(includeTimeSeriesCommand);
}
resultToFill.RegisterTimeSeriesFields(query, fieldsToFetch);
resultToFill.TotalResults = (totalResults.Value == 0 && resultToFill.Results.Count != 0) ? -1 : totalResults.Value;
if (query.Offset != null || query.Limit != null)
{
if (resultToFill.TotalResults == -1)
{
resultToFill.CappedMaxResults = query.Limit ?? -1;
}
else
{
resultToFill.CappedMaxResults = Math.Min(
query.Limit ?? int.MaxValue,
resultToFill.TotalResults - (query.Offset ?? 0)
);
}
}
}
}
protected StoredValueQueryResultRetriever(string storedValueFieldName, DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsStorage documentsStorage, JsonOperationContext context, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand)
: base(database, query, queryTimings, fieldsToFetch, documentsStorage, context, true, includeDocumentsCommand, includeCompareExchangeValuesCommand)
{
if (storedValueFieldName == null)
{
throw new ArgumentNullException(nameof(storedValueFieldName));
}
_storedValueFieldName = storedValueFieldName;
_context = context;
}
public static unsafe string ComputeEtagForDocuments(
List <Document> documents,
List <Document>?includes,
IncludeCountersCommand?includeCounters,
IncludeTimeSeriesCommand?includeTimeSeries,
IncludeCompareExchangeValuesCommand includeCompareExchangeValues)
{
// This method is efficient because we aren't materializing any values
// except the change vector, which we need
if (documents.Count == 1 &&
(includes == null || includes.Count == 0) &&
includeCounters == null &&
includeTimeSeries == null &&
includeCompareExchangeValues == null)
{
return(documents[0]?.ChangeVector ?? string.Empty);
}
var size = Sodium.crypto_generichash_bytes();
Debug.Assert((int)size == 32);
var cryptoGenerichashStatebytes = (int)Sodium.crypto_generichash_statebytes();
byte *state = stackalloc byte[cryptoGenerichashStatebytes];
if (Sodium.crypto_generichash_init(state, null, UIntPtr.Zero, size) != 0)
{
ThrowFailToInitHash();
}
HashNumber(state, documents.Count);
foreach (var doc in documents)
{
HashChangeVector(state, doc?.ChangeVector);
}
if (includes != null)
{
HashNumber(state, includes.Count);
foreach (var doc in includes)
{
if (doc is IncludeDocumentsCommand.ConflictDocument)
{
continue;
}
HashChangeVector(state, doc.ChangeVector);
}
}
if (includeCounters != null)
{
foreach (var countersResult in includeCounters.Results)
{
HashNumber(state, countersResult.Value.Count);
foreach (var counterDetail in countersResult.Value)
{
HashNumber(state, counterDetail?.Etag ?? 0);
}
}
}
if (includeTimeSeries != null)
{
foreach (var tsIncludesForDocument in includeTimeSeries.Results)
{
foreach (var kvp in tsIncludesForDocument.Value)
{
HashNumber(state, tsIncludesForDocument.Value.Count);
foreach (var rangeResult in kvp.Value)
{
HashNumber(state, rangeResult.Entries?.Length ?? 0);
HashChangeVector(state, rangeResult.Hash);
}
}
}
}
if (includeCompareExchangeValues != null)
{
if (includeCompareExchangeValues.Results == null || includeCompareExchangeValues.Results.Count == 0)
{
HashNumber(state, 0);
}
else
{
HashNumber(state, includeCompareExchangeValues.Results.Count);
foreach (var compareExchangeValueInclude in includeCompareExchangeValues.Results)
{
HashNumber(state, compareExchangeValueInclude.Value.Index);
}
}
}
return(FinalizeHash(size, state));
}
public CountersQueryResultRetriever(DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsStorage documentsStorage, JsonOperationContext context, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand)
: base(database, query, queryTimings, documentsStorage, context, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand)
{
}
public override IQueryResultRetriever GetQueryResultRetriever(IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext documentsContext, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand)
{
return(new MapQueryResultRetriever(DocumentDatabase, query, queryTimings, DocumentDatabase.DocumentsStorage, documentsContext, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand));
}
public override void AddCompareExchangeValueIncludes(IncludeCompareExchangeValuesCommand command)
{
_compareExchangeValueIncludes = command.Results;
}
public override IQueryResultRetriever GetQueryResultRetriever(IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext documentsContext, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand)
{
throw new NotSupportedException($"Index {Name} is in-memory implementation of a faulty index", _e);
}
public abstract void AddCompareExchangeValueIncludes(IncludeCompareExchangeValuesCommand command);
private async Task <TResult> ExecuteQuery <TResult>(TResult final, IndexQueryServerSide query, QueryOperationContext queryContext, long?existingResultEtag, OperationCancelToken token) where TResult : QueryResultServerSide <Document>
{
try
{
if (Database.ServerStore.Configuration.Core.FeaturesAvailability == FeaturesAvailability.Stable)
{
FeaturesAvailabilityException.Throw("Graph Queries");
}
using (QueryRunner.MarkQueryAsRunning(Constants.Documents.Indexing.DummyGraphIndexName, query, token))
using (var timingScope = new QueryTimingsScope())
{
var qr = await GetQueryResults(query, queryContext, existingResultEtag, token);
if (qr.NotModified)
{
final.NotModified = true;
return(final);
}
var q = query.Metadata.Query;
//TODO: handle order by, load, clauses
IncludeDocumentsCommand idc = null;
IncludeCompareExchangeValuesCommand icevc = null;
if (q.Select == null && q.SelectFunctionBody.FunctionText == null)
{
await HandleResultsWithoutSelectAsync(queryContext.Documents, qr.Matches, final, token.Token);
}
else if (q.Select != null)
{
//TODO : investigate fields to fetch
var fieldsToFetch = new FieldsToFetch(query, null);
idc = new IncludeDocumentsCommand(Database.DocumentsStorage, queryContext.Documents, query.Metadata.Includes, fieldsToFetch.IsProjection);
icevc = IncludeCompareExchangeValuesCommand.ExternalScope(queryContext, query.Metadata.CompareExchangeValueIncludes);
var resultRetriever = new GraphQueryResultRetriever(
q.GraphQuery,
Database,
query,
timingScope,
Database.DocumentsStorage,
queryContext.Documents,
fieldsToFetch,
idc,
icevc);
HashSet <ulong> alreadySeenProjections = null;
if (q.IsDistinct)
{
alreadySeenProjections = new HashSet <ulong>();
}
foreach (var match in qr.Matches)
{
if (match.Empty)
{
continue;
}
var result = resultRetriever.ProjectFromMatch(match, queryContext.Documents);
// ReSharper disable once PossibleNullReferenceException
if (q.IsDistinct && alreadySeenProjections.Add(result.DataHash) == false)
{
continue;
}
await final.AddResultAsync(result, token.Token);
}
}
if (idc == null)
{
idc = new IncludeDocumentsCommand(Database.DocumentsStorage, queryContext.Documents, query.Metadata.Includes, isProjection: false);
}
if (query.Metadata.Includes?.Length > 0)
{
foreach (var result in final.Results)
{
idc.Gather(result);
}
}
idc.Fill(final.Includes);
final.TotalResults = final.Results.Count;
if (query.Limit != null || query.Offset != null)
{
final.CappedMaxResults = Math.Min(
query.Limit ?? int.MaxValue,
final.TotalResults - (query.Offset ?? 0)
);
}
final.IsStale = qr.QueryPlan.IsStale;
final.ResultEtag = qr.QueryPlan.ResultEtag;
return(final);
}
}
catch (OperationCanceledException oce)
{
throw new OperationCanceledException($"Database:{Database} Query:{query.Metadata.Query} has been cancelled ", oce);
}
}
public MapReduceQueryResultRetriever(DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsStorage documentsStorage, JsonOperationContext context, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand)
: base(Constants.Documents.Indexing.Fields.ReduceKeyValueFieldName, database, query, queryTimings, documentsStorage, context, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand)
{
}
public override IQueryResultRetriever GetQueryResultRetriever(IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext documentsContext, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand)
{
throw new System.NotImplementedException();
}
