private void GetRevisionsQueryString(DocumentDatabase database, DocumentsOperationContext context, out IncludeRevisionsCommand includeRevisions) { includeRevisions = null; var rif = new RevisionIncludeField(); var revisionsByChangeVectors = GetStringValuesQueryString("revisions", required: false); var revisionByDateTimeBefore = GetStringValuesQueryString("revisionsBefore", required: false); if (revisionsByChangeVectors.Count == 0 && revisionByDateTimeBefore.Count == 0) { return; } if (DateTime.TryParseExact(revisionByDateTimeBefore.ToString(), DefaultFormat.DateTimeFormatsToRead, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var dateTime)) { rif.RevisionsBeforeDateTime = dateTime.ToUniversalTime(); } foreach (var changeVector in revisionsByChangeVectors) { rif.RevisionsChangeVectorsPaths.Add(changeVector); } includeRevisions = new IncludeRevisionsCommand(database, context, rif); }
public override void AddRevisionIncludes(IncludeRevisionsCommand command) { if (command.RevisionsChangeVectorResults is null) { return; } throw new NotSupportedException(); }
public GraphQueryResultRetriever(GraphQuery graphQuery, DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsStorage documentsStorage, DocumentsOperationContext context, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand) : base(database, query, queryTimings, fieldsToFetch, documentsStorage, context, false, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand) { _graphQuery = graphQuery; _context = context; }
public CollectionQueryEnumerable(DocumentDatabase database, DocumentsStorage documents, FieldsToFetch fieldsToFetch, string collection, IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext context, IncludeDocumentsCommand includeDocumentsCommand, IncludeRevisionsCommand includeRevisionsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, Reference <int> totalResults, Reference <int> scannedResults, Reference <long> skippedResults, CancellationToken token) { _database = database; _documents = documents; _fieldsToFetch = fieldsToFetch; _collection = collection; _isAllDocsCollection = collection == Constants.Documents.Collections.AllDocumentsCollection; _query = query; _queryTimings = queryTimings; _context = context; _includeDocumentsCommand = includeDocumentsCommand; _includeRevisionsCommand = includeRevisionsCommand; _includeCompareExchangeValuesCommand = includeCompareExchangeValuesCommand; _totalResults = totalResults; _token = token; _scannedResults = scannedResults; _skippedResults = skippedResults; }
protected QueryResultRetrieverBase( DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, FieldsToFetch fieldsToFetch, DocumentsStorage documentsStorage, JsonOperationContext context, bool reduceResults, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand) { _database = database; _query = query; _context = context; _includeDocumentsCommand = includeDocumentsCommand; _includeRevisionsCommand = includeRevisionsCommand; _includeCompareExchangeValuesCommand = includeCompareExchangeValuesCommand; ValidateFieldsToFetch(fieldsToFetch); FieldsToFetch = fieldsToFetch; DocumentsStorage = documentsStorage; RetrieverScope = queryTimings?.For(nameof(QueryTimingsScope.Names.Retriever), start: false); DocumentFields = query?.DocumentFields ?? DocumentFields.All; _blittableTraverser = reduceResults ? BlittableJsonTraverser.FlatMapReduceResults : BlittableJsonTraverser.Default; }
public Enumerator(DocumentDatabase database, DocumentsStorage documents, FieldsToFetch fieldsToFetch, string collection, bool isAllDocsCollection, IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext context, IncludeDocumentsCommand includeDocumentsCommand, IncludeRevisionsCommand includeRevisionsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, Reference <int> totalResults, Reference <int> scannedResults, string startAfterId, Reference <long> alreadySeenIdsCount, DocumentFields fields, Reference <long> skippedResults, CancellationToken token) { _documents = documents; _fieldsToFetch = fieldsToFetch; _collection = collection; _isAllDocsCollection = isAllDocsCollection; _query = query; _queryTimings = queryTimings; _context = context; _totalResults = totalResults; _scannedResults = scannedResults; _totalResults.Value = 0; _startAfterId = startAfterId; _alreadySeenIdsCount = alreadySeenIdsCount; _fields = fields; _skippedResults = skippedResults; _token = token; if (_fieldsToFetch.IsDistinct) { _alreadySeenProjections = new HashSet <ulong>(); } _resultsRetriever = new MapQueryResultRetriever(database, query, queryTimings, documents, context, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand); (_ids, _startsWith) = ExtractIdsFromQuery(query, context); if (_query.Metadata.FilterScript != null) { var key = new FilterKey(_query.Metadata); _releaseFilterScriptRunner = database.Scripts.GetScriptRunner(key, readOnly: true, patchRun: out _filterScriptRun); } }
public CountersQueryResultRetriever(DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsStorage documentsStorage, JsonOperationContext context, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand) : base(database, query, queryTimings, documentsStorage, context, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand) { }
public override void AddRevisionIncludes(IncludeRevisionsCommand command) { _revisionsChangeVectorResults = command.RevisionsChangeVectorResults; _revisionsDateTimeBeforeResults = command.IdByRevisionsByDateTimeResults; }
public override IQueryResultRetriever GetQueryResultRetriever(IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext documentsContext, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand) { throw new NotSupportedException($"Index {Name} is in-memory implementation of a faulty index", _e); }
public override IQueryResultRetriever GetQueryResultRetriever(IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext documentsContext, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand) { return(new TimeSeriesQueryResultRetriever(DocumentDatabase, query, queryTimings, DocumentDatabase.DocumentsStorage, documentsContext, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand)); }
protected StoredValueQueryResultRetriever(string storedValueFieldName, DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsStorage documentsStorage, JsonOperationContext context, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand) : base(database, query, queryTimings, fieldsToFetch, documentsStorage, context, true, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand) { if (storedValueFieldName == null) { throw new ArgumentNullException(nameof(storedValueFieldName)); } _storedValueFieldName = storedValueFieldName; _context = context; }
public MapReduceQueryResultRetriever(DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsStorage documentsStorage, JsonOperationContext context, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand) : base(Constants.Documents.Indexing.Fields.ReduceKeyValueFieldName, database, query, queryTimings, documentsStorage, context, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand) { }
public override IQueryResultRetriever GetQueryResultRetriever(IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext documentsContext, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, IncludeRevisionsCommand includeRevisionsCommand) { throw new System.NotImplementedException(); }
public abstract void AddRevisionIncludes(IncludeRevisionsCommand command);
private async Task <TResult> ExecuteQuery <TResult>(TResult final, IndexQueryServerSide query, QueryOperationContext queryContext, long?existingResultEtag, OperationCancelToken token) where TResult : QueryResultServerSide <Document> { try { Database.ServerStore.FeatureGuardian.Assert(Feature.GraphApi); using (QueryRunner.MarkQueryAsRunning(Constants.Documents.Indexing.DummyGraphIndexName, query, token)) using (var timingScope = new QueryTimingsScope()) { var qr = await GetQueryResults(query, queryContext, existingResultEtag, token); if (qr.NotModified) { final.NotModified = true; return(final); } var q = query.Metadata.Query; //TODO: handle order by, load, clauses IncludeDocumentsCommand idc = null; IncludeRevisionsCommand irc = null; IncludeCompareExchangeValuesCommand icevc = null; if (q.Select == null && q.SelectFunctionBody.FunctionText == null) { await HandleResultsWithoutSelectAsync(queryContext.Documents, qr.Matches, final, token.Token); } else if (q.Select != null) { //TODO : investigate fields to fetch var fieldsToFetch = new FieldsToFetch(query, null); idc = new IncludeDocumentsCommand(Database.DocumentsStorage, queryContext.Documents, query.Metadata.Includes, fieldsToFetch.IsProjection); icevc = IncludeCompareExchangeValuesCommand.ExternalScope(queryContext, query.Metadata.CompareExchangeValueIncludes); irc = new IncludeRevisionsCommand(database: Database, context: queryContext.Documents, query.Metadata.RevisionIncludes); var resultRetriever = new GraphQueryResultRetriever( q.GraphQuery, Database, query, timingScope, Database.DocumentsStorage, queryContext.Documents, fieldsToFetch, idc, icevc, irc); HashSet <ulong> alreadySeenProjections = null; if (q.IsDistinct) { alreadySeenProjections = new HashSet <ulong>(); } foreach (var match in qr.Matches) { if (match.Empty) { continue; } var result = resultRetriever.ProjectFromMatch(match, queryContext.Documents, token.Token); // ReSharper disable once PossibleNullReferenceException if (q.IsDistinct && alreadySeenProjections.Add(result.DataHash) == false) { continue; } await final.AddResultAsync(result, token.Token); } } if (idc == null) { idc = new IncludeDocumentsCommand(Database.DocumentsStorage, queryContext.Documents, query.Metadata.Includes, isProjection: false); } if (query.Metadata.Includes?.Length > 0) { foreach (var result in final.Results) { idc.Gather(result); } } idc.Fill(final.Includes); final.TotalResults = final.Results.Count; final.LongTotalResults = final.Results.Count; if (query.Limit != null || query.Offset != null) { final.CappedMaxResults = Math.Min( query.Limit ?? int.MaxValue, final.TotalResults - (query.Offset ?? 0) ); } final.IsStale = qr.QueryPlan.IsStale; final.ResultEtag = qr.QueryPlan.ResultEtag; return(final); } } catch (OperationCanceledException oce) { throw new OperationCanceledException($"Database:{Database} Query:{query.Metadata.Query} has been cancelled ", oce); } }
private async ValueTask ExecuteCollectionQueryAsync(QueryResultServerSide <Document> resultToFill, IndexQueryServerSide query, string collection, QueryOperationContext context, bool pulseReadingTransaction, CancellationToken token) { using (var queryScope = query.Timings?.For(nameof(QueryTimingsScope.Names.Query))) { QueryTimingsScope gatherScope = null; QueryTimingsScope fillScope = null; if (queryScope != null && query.Metadata.Includes?.Length > 0) { var includesScope = queryScope.For(nameof(QueryTimingsScope.Names.Includes), start: false); gatherScope = includesScope.For(nameof(QueryTimingsScope.Names.Gather), start: false); fillScope = includesScope.For(nameof(QueryTimingsScope.Names.Fill), start: false); } // we optimize for empty queries without sorting options, appending CollectionIndexPrefix to be able to distinguish index for collection vs. physical index resultToFill.IsStale = false; resultToFill.LastQueryTime = DateTime.MinValue; resultToFill.IndexTimestamp = DateTime.MinValue; resultToFill.IncludedPaths = query.Metadata.Includes; var fieldsToFetch = new FieldsToFetch(query, null); var includeDocumentsCommand = new IncludeDocumentsCommand(Database.DocumentsStorage, context.Documents, query.Metadata.Includes, fieldsToFetch.IsProjection); var includeRevisionsCommand = new IncludeRevisionsCommand(Database, context.Documents, query.Metadata.RevisionIncludes); var includeCompareExchangeValuesCommand = IncludeCompareExchangeValuesCommand.ExternalScope(context, query.Metadata.CompareExchangeValueIncludes); var totalResults = new Reference <int>(); var skippedResults = new Reference <long>(); var scannedResults = new Reference <int>(); IEnumerator <Document> enumerator; if (pulseReadingTransaction == false) { var documents = new CollectionQueryEnumerable(Database, Database.DocumentsStorage, fieldsToFetch, collection, query, queryScope, context.Documents, includeDocumentsCommand, includeRevisionsCommand: includeRevisionsCommand, includeCompareExchangeValuesCommand: includeCompareExchangeValuesCommand, totalResults: totalResults, scannedResults, skippedResults, token); enumerator = documents.GetEnumerator(); } else { enumerator = new PulsedTransactionEnumerator <Document, CollectionQueryResultsIterationState>(context.Documents, state => { query.Start = state.Start; query.PageSize = state.Take; var documents = new CollectionQueryEnumerable(Database, Database.DocumentsStorage, fieldsToFetch, collection, query, queryScope, context.Documents, includeDocumentsCommand, includeRevisionsCommand, includeCompareExchangeValuesCommand, totalResults, scannedResults, skippedResults, token); return(documents); }, new CollectionQueryResultsIterationState(context.Documents, Database.Configuration.Databases.PulseReadTransactionLimit) { Start = query.Start, Take = query.PageSize }); } IncludeCountersCommand includeCountersCommand = null; IncludeTimeSeriesCommand includeTimeSeriesCommand = null; if (query.Metadata.RevisionIncludes != null) { includeRevisionsCommand = new IncludeRevisionsCommand( Database, context.Documents, query.Metadata.RevisionIncludes); } if (query.Metadata.CounterIncludes != null) { includeCountersCommand = new IncludeCountersCommand( Database, context.Documents, query.Metadata.CounterIncludes.Counters); } if (query.Metadata.TimeSeriesIncludes != null) { includeTimeSeriesCommand = new IncludeTimeSeriesCommand( context.Documents, query.Metadata.TimeSeriesIncludes.TimeSeries); } try { using (enumerator) { while (enumerator.MoveNext()) { var document = enumerator.Current; token.ThrowIfCancellationRequested(); await resultToFill.AddResultAsync(document, token); using (gatherScope?.Start()) { includeDocumentsCommand.Gather(document); includeCompareExchangeValuesCommand?.Gather(document); } includeCountersCommand?.Fill(document); includeTimeSeriesCommand?.Fill(document); includeRevisionsCommand?.Fill(document); } } } catch (Exception e) { if (resultToFill.SupportsExceptionHandling == false) { throw; } await resultToFill.HandleExceptionAsync(e, token); } using (fillScope?.Start()) { includeDocumentsCommand.Fill(resultToFill.Includes); includeCompareExchangeValuesCommand.Materialize(); } if (includeCompareExchangeValuesCommand != null) { resultToFill.AddCompareExchangeValueIncludes(includeCompareExchangeValuesCommand); } if (includeCountersCommand != null) { resultToFill.AddCounterIncludes(includeCountersCommand); } if (includeTimeSeriesCommand != null) { resultToFill.AddTimeSeriesIncludes(includeTimeSeriesCommand); } if (includeRevisionsCommand != null) { resultToFill.AddRevisionIncludes(includeRevisionsCommand); } resultToFill.RegisterTimeSeriesFields(query, fieldsToFetch); resultToFill.TotalResults = (totalResults.Value == 0 && resultToFill.Results.Count != 0) ? -1 : totalResults.Value; resultToFill.LongTotalResults = resultToFill.TotalResults; resultToFill.SkippedResults = Convert.ToInt32(skippedResults.Value); resultToFill.ScannedResults = scannedResults.Value; if (query.Offset != null || query.Limit != null) { if (resultToFill.TotalResults == -1) { resultToFill.CappedMaxResults = query.Limit ?? -1; } else { resultToFill.CappedMaxResults = Math.Min( query.Limit ?? int.MaxValue, resultToFill.TotalResults - (query.Offset ?? 0) ); } } } }