public Enumerator(DocumentDatabase database, DocumentsStorage documents, FieldsToFetch fieldsToFetch, string collection, bool isAllDocsCollection, IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext context, IncludeDocumentsCommand includeDocumentsCommand, IncludeRevisionsCommand includeRevisionsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, Reference <int> totalResults, Reference <int> scannedResults, string startAfterId, Reference <long> alreadySeenIdsCount, DocumentFields fields, Reference <long> skippedResults, CancellationToken token) { _documents = documents; _fieldsToFetch = fieldsToFetch; _collection = collection; _isAllDocsCollection = isAllDocsCollection; _query = query; _queryTimings = queryTimings; _context = context; _totalResults = totalResults; _scannedResults = scannedResults; _totalResults.Value = 0; _startAfterId = startAfterId; _alreadySeenIdsCount = alreadySeenIdsCount; _fields = fields; _skippedResults = skippedResults; _token = token; if (_fieldsToFetch.IsDistinct) { _alreadySeenProjections = new HashSet <ulong>(); } _resultsRetriever = new MapQueryResultRetriever(database, query, queryTimings, documents, context, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand); (_ids, _startsWith) = ExtractIdsFromQuery(query, context); if (_query.Metadata.FilterScript != null) { var key = new FilterKey(_query.Metadata); _releaseFilterScriptRunner = database.Scripts.GetScriptRunner(key, readOnly: true, patchRun: out _filterScriptRun); } }
public override async Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, PatchRequest patch, BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token) { ObjectDisposedException lastException = null; for (var i = 0; i < NumberOfRetries; i++) { try { context.CloseTransaction(); return(await GetRunner(query).ExecutePatchQuery(query, options, patch, patchArgs, context, onProgress, token)); } catch (ObjectDisposedException e) { if (Database.DatabaseShutdown.IsCancellationRequested) { throw; } lastException = e; } } throw CreateRetriesFailedException(lastException); }
public FieldsToFetch(IndexQueryServerSide query, IndexDefinitionBase indexDefinition) : this(query.Metadata.SelectFields, indexDefinition) { IsDistinct = query.Metadata.IsDistinct && IsProjection; }
public CollectionQueryEnumerable(DocumentsStorage documents, FieldsToFetch fieldsToFetch, string collection, IndexQueryServerSide query, DocumentsOperationContext context) { _documents = documents; _fieldsToFetch = fieldsToFetch; _collection = collection; _isAllDocsCollection = collection == Constants.Indexing.AllDocumentsCollection; _query = query; _context = context; }
public static IndexQueryServerSide Create(HttpContext httpContext, int start, int pageSize, JsonOperationContext context) { var result = new IndexQueryServerSide { // all defaults which need to have custom value Start = start, PageSize = pageSize }; DynamicJsonValue transformerParameters = null; HashSet <string> includes = null; foreach (var item in httpContext.Request.Query) { try { switch (item.Key) { case "query": result.Query = item.Value[0]; break; case RequestHandler.StartParameter: case RequestHandler.PageSizeParameter: break; case "cutOffEtag": result.CutoffEtag = long.Parse(item.Value[0]); break; case "waitForNonStaleResultsAsOfNow": result.WaitForNonStaleResultsAsOfNow = bool.Parse(item.Value[0]); break; case "waitForNonStaleResultsTimeout": result.WaitForNonStaleResultsTimeout = TimeSpan.Parse(item.Value[0]); break; case "fetch": result.FieldsToFetch = item.Value; break; case "operator": result.DefaultOperator = "And".Equals(item.Value[0], StringComparison.OrdinalIgnoreCase) ? QueryOperator.And : QueryOperator.Or; break; case "defaultField": result.DefaultField = item.Value; break; case "sort": result.SortedFields = item.Value.Select(y => new SortedField(y)).ToArray(); break; case "mapReduce": result.DynamicMapReduceFields = ParseDynamicMapReduceFields(item.Value); break; case "include": if (includes == null) { includes = new HashSet <string>(StringComparer.OrdinalIgnoreCase); } includes.Add(item.Value[0]); break; case "distinct": result.IsDistinct = bool.Parse(item.Value[0]); break; case "transformer": result.Transformer = item.Value[0]; break; case "skipDuplicateChecking": result.SkipDuplicateChecking = bool.Parse(item.Value[0]); break; case "allowMultipleIndexEntriesForSameDocumentToResultTransformer": result.AllowMultipleIndexEntriesForSameDocumentToResultTransformer = bool.Parse(item.Value[0]); break; default: if (item.Key.StartsWith(TransformerParameter.Prefix, StringComparison.OrdinalIgnoreCase)) { if (transformerParameters == null) { transformerParameters = new DynamicJsonValue(); } transformerParameters[item.Key.Substring(TransformerParameter.Prefix.Length)] = item.Value[0]; } break; // TODO: HighlightedFields, HighlighterPreTags, HighlighterPostTags, HighlighterKeyName, ExplainScores // TODO: ShowTimings and spatial stuff // TODO: We also need to make sure that we aren't using headers } } catch (Exception e) { throw new ArgumentException($"Could not handle query string parameter '{item.Key}' (value: {item.Value})", e); } } if (includes != null) { result.Includes = includes.ToArray(); } if (transformerParameters != null) { result.TransformerParameters = context.ReadObject(transformerParameters, "transformer/parameters"); } if (result.Query == null) { result.Query = string.Empty; } return(result); }
public async Task <SuggestionQueryResult> ExecuteSuggestionQuery(IndexQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token) { if (query.Metadata.IsDynamic) { throw new InvalidQueryException("Suggestion query must be executed against static index.", query.Metadata.QueryText, query.QueryParameters); } ObjectDisposedException lastException = null; for (var i = 0; i < NumberOfRetries; i++) { try { context.CloseTransaction(); var sw = Stopwatch.StartNew(); if (query.Metadata.SelectFields.Length != 1 || query.Metadata.SelectFields[0].IsSuggest == false) { throw new InvalidQueryException("Suggestion query must have one suggest token in SELECT.", query.Metadata.QueryText, query.QueryParameters); } var selectField = (SuggestionField)query.Metadata.SelectFields[0]; var index = GetIndex(query.Metadata.IndexName); var indexDefinition = index.GetIndexDefinition(); if (indexDefinition.Fields.TryGetValue(selectField.Name, out IndexFieldOptions field) == false) { throw new InvalidOperationException($"Index '{query.Metadata.IndexName}' does not have a field '{selectField.Name}'."); } if (field.Suggestions == null) { throw new InvalidOperationException($"Index '{query.Metadata.IndexName}' does not have suggestions configured for field '{selectField.Name}'."); } if (field.Suggestions.Value == false) { throw new InvalidOperationException($"Index '{query.Metadata.IndexName}' have suggestions explicitly disabled for field '{selectField.Name}'."); } if (existingResultEtag.HasValue) { var etag = index.GetIndexEtag(query.Metadata); if (etag == existingResultEtag.Value) { return(SuggestionQueryResult.NotModifiedResult); } } var result = await index.SuggestionQuery(query, context, token); result.DurationInMs = (int)sw.Elapsed.TotalMilliseconds; return(result); } catch (ObjectDisposedException e) { if (Database.DatabaseShutdown.IsCancellationRequested) { throw; } lastException = e; } } throw CreateRetriesFailedException(lastException); }
public abstract Task <IOperationResult> ExecuteDeleteQuery(IndexQueryServerSide query, QueryOperationOptions options, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token);
public abstract Task ExecuteStreamQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, HttpResponse response, IStreamDocumentQueryResultWriter writer, OperationCancelToken token);
public override Task <IndexEntriesQueryResult> ExecuteIndexEntriesQuery(IndexQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token) { throw new NotSupportedException("Graph queries do not expose index queries"); }
public override Task <IOperationResult> ExecuteDeleteQuery(IndexQueryServerSide query, QueryOperationOptions options, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token) { throw new NotSupportedException("You cannot delete based on graph query"); }
private async Task <(List <Match> Matches, GraphQueryPlan QueryPlan, bool NotModified)> GetQueryResults(IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token, bool collectIntermediateResults = false) { var q = query.Metadata.Query; var qp = new GraphQueryPlan(query, documentsContext, existingResultEtag, token, Database) { CollectIntermediateResults = collectIntermediateResults }; qp.BuildQueryPlan(); qp.OptimizeQueryPlan(); //TODO: audit optimization if (query.WaitForNonStaleResults) { qp.IsStale = await qp.WaitForNonStaleResults(); } else { await qp.CreateAutoIndexesAndWaitIfNecessary(); } //for the case where we don't wait for non stale results we will override IsStale in the QueryQueryStep steps if (documentsContext.Transaction == null || documentsContext.Transaction.Disposed) { documentsContext.OpenReadTransaction(); } qp.ResultEtag = DocumentsStorage.ReadLastEtag(documentsContext.Transaction.InnerTransaction); if (existingResultEtag.HasValue) { if (qp.ResultEtag == existingResultEtag) { return(null, null, true); } } await qp.Initialize(); var matchResults = qp.Execute(); if (query.Metadata.OrderBy != null) { Sort(matchResults, query.Metadata.OrderBy, Database.Name, query.Query); } var filter = q.GraphQuery.Where; if (filter != null) { for (int i = 0; i < matchResults.Count; i++) { var resultAsJson = new DynamicJsonValue(); matchResults[i].PopulateVertices(resultAsJson); using (var result = documentsContext.ReadObject(resultAsJson, "graph/result")) { if (filter.IsMatchedBy(result, query.QueryParameters) == false) { matchResults[i] = default; } } } } if (query.Start > 0) { matchResults.RemoveRange(0, Math.Min(query.Start, matchResults.Count)); } if (query.PageSize < matchResults.Count) { matchResults.RemoveRange(query.PageSize, matchResults.Count - query.PageSize); } return(matchResults, qp, false); }
private async Task <TResult> ExecuteQuery <TResult>(TResult final, IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token) where TResult : QueryResultServerSide <Document> { try { if (Database.ServerStore.Configuration.Core.FeaturesAvailability == FeaturesAvailability.Stable) { FeaturesAvailabilityException.Throw("Graph Queries"); } using (QueryRunner.MarkQueryAsRunning(Constants.Documents.Indexing.DummyGraphIndexName, query, token)) using (var timingScope = new QueryTimingsScope()) { var qr = await GetQueryResults(query, documentsContext, existingResultEtag, token); if (qr.NotModified) { final.NotModified = true; return(final); } var q = query.Metadata.Query; //TODO: handle order by, load, clauses IncludeDocumentsCommand idc = null; if (q.Select == null && q.SelectFunctionBody.FunctionText == null) { HandleResultsWithoutSelect(documentsContext, qr.Matches, final); } else if (q.Select != null) { //TODO : investigate fields to fetch var fieldsToFetch = new FieldsToFetch(query, null); idc = new IncludeDocumentsCommand(Database.DocumentsStorage, documentsContext, query.Metadata.Includes, fieldsToFetch.IsProjection); var resultRetriever = new GraphQueryResultRetriever( q.GraphQuery, Database, query, timingScope, Database.DocumentsStorage, documentsContext, fieldsToFetch, idc); HashSet <ulong> alreadySeenProjections = null; if (q.IsDistinct) { alreadySeenProjections = new HashSet <ulong>(); } foreach (var match in qr.Matches) { if (match.Empty) { continue; } var result = resultRetriever.ProjectFromMatch(match, documentsContext); // ReSharper disable once PossibleNullReferenceException if (q.IsDistinct && alreadySeenProjections.Add(result.DataHash) == false) { continue; } final.AddResult(result); } } if (idc == null) { idc = new IncludeDocumentsCommand(Database.DocumentsStorage, documentsContext, query.Metadata.Includes, isProjection: false); } if (query.Metadata.Includes?.Length > 0) { foreach (var result in final.Results) { idc.Gather(result); } } idc.Fill(final.Includes); final.TotalResults = final.Results.Count; if (query.Limit != null || query.Offset != null) { final.CappedMaxResults = Math.Min( query.Limit ?? int.MaxValue, final.TotalResults - (query.Offset ?? 0) ); } final.IsStale = qr.QueryPlan.IsStale; final.ResultEtag = qr.QueryPlan.ResultEtag; return(final); } } catch (OperationCanceledException oce) { throw new OperationCanceledException($"Database:{Database} Query:{query.Metadata.Query} has been cancelled ", oce); } }
public override Task ExecuteStreamIndexEntriesQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, HttpResponse response, IStreamQueryResultWriter <BlittableJsonReaderObject> writer, OperationCancelToken token) { throw new NotImplementedException(); }
public override async Task <IndexEntriesQueryResult> ExecuteIndexEntriesQuery(IndexQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token) { var index = GetIndex(query.Metadata.IndexName); if (existingResultEtag.HasValue) { var etag = index.GetIndexEtag(query.Metadata); if (etag == existingResultEtag) { return(IndexEntriesQueryResult.NotModifiedResult); } } using (QueryRunner.MarkQueryAsRunning(index.Name, query, token)) { return(await index.IndexEntries(query, context, token)); } }
public List <DynamicQueryToIndexMatcher.Explanation> ExplainDynamicIndexSelection(IndexQueryServerSide query) { if (query.Metadata.IsDynamic == false) { throw new InvalidOperationException("Explain can only work on dynamic indexes"); } return(_dynamic.ExplainIndexSelection(query)); }
public override Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, PatchRequest patch, BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token) { var index = GetIndex(query.Metadata.IndexName); return(ExecutePatch(query, index, options, patch, patchArgs, context, onProgress, token)); }
public override Task ExecuteStreamQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, HttpResponse response, IStreamDocumentQueryResultWriter writer, OperationCancelToken token) { var index = GetIndex(query.Metadata.IndexName); return(index.StreamQuery(response, writer, query, documentsContext, token)); }
public override Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, Patch.PatchRequest patch, BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token) { throw new NotSupportedException("You cannot patch based on graph query"); }
public abstract Task <IndexEntriesQueryResult> ExecuteIndexEntriesQuery(IndexQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token);
public override Task <SuggestionQueryResult> ExecuteSuggestionQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token) { throw new NotSupportedException("You cannot suggest based on graph query"); }
public abstract Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, PatchRequest patch, BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token);
public override Task <IndexEntriesQueryResult> ExecuteIndexEntriesQuery(IndexQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token) { return(GetRunner(query).ExecuteIndexEntriesQuery(query, context, existingResultEtag, token)); }
public override async Task <IndexEntriesQueryResult> ExecuteIndexEntriesQuery(IndexQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token) { ObjectDisposedException lastException = null; for (var i = 0; i < NumberOfRetries; i++) { try { context.CloseTransaction(); return(await GetRunner(query).ExecuteIndexEntriesQuery(query, context, existingResultEtag, token)); } catch (ObjectDisposedException e) { if (Database.DatabaseShutdown.IsCancellationRequested) { throw; } lastException = e; } } throw CreateRetriesFailedException(lastException); }
public override Task <IOperationResult> ExecuteDeleteQuery(IndexQueryServerSide query, QueryOperationOptions options, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token) { return(GetRunner(query).ExecuteDeleteQuery(query, options, context, onProgress, token)); }
public List <DynamicQueryToIndexMatcher.Explanation> ExplainDynamicIndexSelection(IndexQueryServerSide query, DocumentsOperationContext context, out string indexName) { if (query.Metadata.IsDynamic == false) { throw new InvalidOperationException("Explain can only work on dynamic indexes"); } if (_dynamic is DynamicQueryRunner d) { return(d.ExplainIndexSelection(query, context, out indexName)); } throw new NotSupportedException(InvalidQueryRunner.ErrorMessage); }
public override Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, PatchRequest patch, BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token) { return(GetRunner(query).ExecutePatchQuery(query, options, patch, patchArgs, context, onProgress, token)); }
private List <Slice> ExtractIdsFromQuery(IndexQueryServerSide query) { if (string.IsNullOrWhiteSpace(query.Query)) { return(null); } var q = new StringSegment(query.Query.Replace(" ", string.Empty), 0); if (q.Length <= EqualPrefix.Length) { return(null); } var documentId = q.SubSegment(0, EqualPrefix.Length); if (documentId.Equals(EqualPrefix)) { var id = q.SubSegment(EqualPrefix.Length); Slice key; Slice.From(_context.Allocator, id, out key); _context.Allocator.ToLowerCase(ref key.Content); return(new List <Slice> { key }); } if (q.Length <= InPrefix.Length) { return(null); } var @in = q.SubSegment(0, InPrefix.Length); if (@in.Equals(InPrefix) == false) { return(null); } var ids = q.SubSegment(InPrefix.Length + 1, q.Length - InPrefix.Length - 2); var results = new Slice[0]; int indexOfComma; do { indexOfComma = ids.IndexOfAny(InSeparator, 0); StringSegment id; if (indexOfComma != -1) { id = ids.SubSegment(0, indexOfComma); ids = ids.SubSegment(indexOfComma + 1); } else { id = ids; } Slice key; Slice.From(_context.Allocator, id, out key); _context.Allocator.ToLowerCase(ref key.Content); Array.Resize(ref results, results.Length + 1); results[results.Length - 1] = key; } while (indexOfComma != -1); return(results .OrderBy(x => x, SliceComparer.Instance) .ToList()); }
public override Task ExecuteStreamQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, HttpResponse response, IStreamDocumentQueryResultWriter writer, OperationCancelToken token) { return(GetRunner(query).ExecuteStreamQuery(query, documentsContext, response, writer, token)); }
public Enumerator(DocumentsStorage documents, FieldsToFetch fieldsToFetch, string collection, bool isAllDocsCollection, IndexQueryServerSide query, DocumentsOperationContext context) { _documents = documents; _fieldsToFetch = fieldsToFetch; _collection = collection; _isAllDocsCollection = isAllDocsCollection; _query = query; _context = context; if (_fieldsToFetch.IsDistinct) { _alreadySeenProjections = new HashSet <ulong>(); } _ids = ExtractIdsFromQuery(query); _sort = ExtractSortFromQuery(query); }
public override Task <IOperationResult> ExecuteDeleteQuery(IndexQueryServerSide query, QueryOperationOptions options, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token) { var index = GetIndex(query.Metadata.IndexName); return(ExecuteDelete(query, index, options, context, onProgress, token)); }