public Enumerator(DocumentDatabase database, DocumentsStorage documents, FieldsToFetch fieldsToFetch, string collection, bool isAllDocsCollection,
IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext context, IncludeDocumentsCommand includeDocumentsCommand,
IncludeRevisionsCommand includeRevisionsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand, Reference <int> totalResults,
Reference <int> scannedResults, string startAfterId, Reference <long> alreadySeenIdsCount, DocumentFields fields, Reference <long> skippedResults, CancellationToken token)
{
_documents = documents;
_fieldsToFetch = fieldsToFetch;
_collection = collection;
_isAllDocsCollection = isAllDocsCollection;
_query = query;
_queryTimings = queryTimings;
_context = context;
_totalResults = totalResults;
_scannedResults = scannedResults;
_totalResults.Value = 0;
_startAfterId = startAfterId;
_alreadySeenIdsCount = alreadySeenIdsCount;
_fields = fields;
_skippedResults = skippedResults;
_token = token;
if (_fieldsToFetch.IsDistinct)
{
_alreadySeenProjections = new HashSet <ulong>();
}
_resultsRetriever = new MapQueryResultRetriever(database, query, queryTimings, documents, context, fieldsToFetch, includeDocumentsCommand, includeCompareExchangeValuesCommand, includeRevisionsCommand);
(_ids, _startsWith) = ExtractIdsFromQuery(query, context);
if (_query.Metadata.FilterScript != null)
{
var key = new FilterKey(_query.Metadata);
_releaseFilterScriptRunner = database.Scripts.GetScriptRunner(key, readOnly: true, patchRun: out _filterScriptRun);
}
}
public override async Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, PatchRequest patch, BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
ObjectDisposedException lastException = null;
for (var i = 0; i < NumberOfRetries; i++)
{
try
{
context.CloseTransaction();
return(await GetRunner(query).ExecutePatchQuery(query, options, patch, patchArgs, context, onProgress, token));
}
catch (ObjectDisposedException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
lastException = e;
}
}
throw CreateRetriesFailedException(lastException);
}
public FieldsToFetch(IndexQueryServerSide query, IndexDefinitionBase indexDefinition)
: this(query.Metadata.SelectFields, indexDefinition)
{
IsDistinct = query.Metadata.IsDistinct && IsProjection;
}
public CollectionQueryEnumerable(DocumentsStorage documents, FieldsToFetch fieldsToFetch, string collection, IndexQueryServerSide query, DocumentsOperationContext context)
{
_documents = documents;
_fieldsToFetch = fieldsToFetch;
_collection = collection;
_isAllDocsCollection = collection == Constants.Indexing.AllDocumentsCollection;
_query = query;
_context = context;
}
public static IndexQueryServerSide Create(HttpContext httpContext, int start, int pageSize, JsonOperationContext context)
{
var result = new IndexQueryServerSide
{
// all defaults which need to have custom value
Start = start,
PageSize = pageSize
};
DynamicJsonValue transformerParameters = null;
HashSet <string> includes = null;
foreach (var item in httpContext.Request.Query)
{
try
{
switch (item.Key)
{
case "query":
result.Query = item.Value[0];
break;
case RequestHandler.StartParameter:
case RequestHandler.PageSizeParameter:
break;
case "cutOffEtag":
result.CutoffEtag = long.Parse(item.Value[0]);
break;
case "waitForNonStaleResultsAsOfNow":
result.WaitForNonStaleResultsAsOfNow = bool.Parse(item.Value[0]);
break;
case "waitForNonStaleResultsTimeout":
result.WaitForNonStaleResultsTimeout = TimeSpan.Parse(item.Value[0]);
break;
case "fetch":
result.FieldsToFetch = item.Value;
break;
case "operator":
result.DefaultOperator = "And".Equals(item.Value[0], StringComparison.OrdinalIgnoreCase) ?
QueryOperator.And : QueryOperator.Or;
break;
case "defaultField":
result.DefaultField = item.Value;
break;
case "sort":
result.SortedFields = item.Value.Select(y => new SortedField(y)).ToArray();
break;
case "mapReduce":
result.DynamicMapReduceFields = ParseDynamicMapReduceFields(item.Value);
break;
case "include":
if (includes == null)
{
includes = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
}
includes.Add(item.Value[0]);
break;
case "distinct":
result.IsDistinct = bool.Parse(item.Value[0]);
break;
case "transformer":
result.Transformer = item.Value[0];
break;
case "skipDuplicateChecking":
result.SkipDuplicateChecking = bool.Parse(item.Value[0]);
break;
case "allowMultipleIndexEntriesForSameDocumentToResultTransformer":
result.AllowMultipleIndexEntriesForSameDocumentToResultTransformer = bool.Parse(item.Value[0]);
break;
default:
if (item.Key.StartsWith(TransformerParameter.Prefix, StringComparison.OrdinalIgnoreCase))
{
if (transformerParameters == null)
{
transformerParameters = new DynamicJsonValue();
}
transformerParameters[item.Key.Substring(TransformerParameter.Prefix.Length)] = item.Value[0];
}
break;
// TODO: HighlightedFields, HighlighterPreTags, HighlighterPostTags, HighlighterKeyName, ExplainScores
// TODO: ShowTimings and spatial stuff
// TODO: We also need to make sure that we aren't using headers
}
}
catch (Exception e)
{
throw new ArgumentException($"Could not handle query string parameter '{item.Key}' (value: {item.Value})", e);
}
}
if (includes != null)
{
result.Includes = includes.ToArray();
}
if (transformerParameters != null)
{
result.TransformerParameters = context.ReadObject(transformerParameters, "transformer/parameters");
}
if (result.Query == null)
{
result.Query = string.Empty;
}
return(result);
}
public async Task <SuggestionQueryResult> ExecuteSuggestionQuery(IndexQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token)
{
if (query.Metadata.IsDynamic)
{
throw new InvalidQueryException("Suggestion query must be executed against static index.", query.Metadata.QueryText, query.QueryParameters);
}
ObjectDisposedException lastException = null;
for (var i = 0; i < NumberOfRetries; i++)
{
try
{
context.CloseTransaction();
var sw = Stopwatch.StartNew();
if (query.Metadata.SelectFields.Length != 1 || query.Metadata.SelectFields[0].IsSuggest == false)
{
throw new InvalidQueryException("Suggestion query must have one suggest token in SELECT.", query.Metadata.QueryText, query.QueryParameters);
}
var selectField = (SuggestionField)query.Metadata.SelectFields[0];
var index = GetIndex(query.Metadata.IndexName);
var indexDefinition = index.GetIndexDefinition();
if (indexDefinition.Fields.TryGetValue(selectField.Name, out IndexFieldOptions field) == false)
{
throw new InvalidOperationException($"Index '{query.Metadata.IndexName}' does not have a field '{selectField.Name}'.");
}
if (field.Suggestions == null)
{
throw new InvalidOperationException($"Index '{query.Metadata.IndexName}' does not have suggestions configured for field '{selectField.Name}'.");
}
if (field.Suggestions.Value == false)
{
throw new InvalidOperationException($"Index '{query.Metadata.IndexName}' have suggestions explicitly disabled for field '{selectField.Name}'.");
}
if (existingResultEtag.HasValue)
{
var etag = index.GetIndexEtag(query.Metadata);
if (etag == existingResultEtag.Value)
{
return(SuggestionQueryResult.NotModifiedResult);
}
}
var result = await index.SuggestionQuery(query, context, token);
result.DurationInMs = (int)sw.Elapsed.TotalMilliseconds;
return(result);
}
catch (ObjectDisposedException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
lastException = e;
}
}
throw CreateRetriesFailedException(lastException);
}
public abstract Task <IOperationResult> ExecuteDeleteQuery(IndexQueryServerSide query, QueryOperationOptions options, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token);
public abstract Task ExecuteStreamQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, HttpResponse response,
IStreamDocumentQueryResultWriter writer, OperationCancelToken token);
public override Task <IndexEntriesQueryResult> ExecuteIndexEntriesQuery(IndexQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token)
{
throw new NotSupportedException("Graph queries do not expose index queries");
}
public override Task <IOperationResult> ExecuteDeleteQuery(IndexQueryServerSide query, QueryOperationOptions options, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
throw new NotSupportedException("You cannot delete based on graph query");
}
private async Task <(List <Match> Matches, GraphQueryPlan QueryPlan, bool NotModified)> GetQueryResults(IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token, bool collectIntermediateResults = false)
{
var q = query.Metadata.Query;
var qp = new GraphQueryPlan(query, documentsContext, existingResultEtag, token, Database)
{
CollectIntermediateResults = collectIntermediateResults
};
qp.BuildQueryPlan();
qp.OptimizeQueryPlan(); //TODO: audit optimization
if (query.WaitForNonStaleResults)
{
qp.IsStale = await qp.WaitForNonStaleResults();
}
else
{
await qp.CreateAutoIndexesAndWaitIfNecessary();
}
//for the case where we don't wait for non stale results we will override IsStale in the QueryQueryStep steps
if (documentsContext.Transaction == null || documentsContext.Transaction.Disposed)
{
documentsContext.OpenReadTransaction();
}
qp.ResultEtag = DocumentsStorage.ReadLastEtag(documentsContext.Transaction.InnerTransaction);
if (existingResultEtag.HasValue)
{
if (qp.ResultEtag == existingResultEtag)
{
return(null, null, true);
}
}
await qp.Initialize();
var matchResults = qp.Execute();
if (query.Metadata.OrderBy != null)
{
Sort(matchResults, query.Metadata.OrderBy, Database.Name, query.Query);
}
var filter = q.GraphQuery.Where;
if (filter != null)
{
for (int i = 0; i < matchResults.Count; i++)
{
var resultAsJson = new DynamicJsonValue();
matchResults[i].PopulateVertices(resultAsJson);
using (var result = documentsContext.ReadObject(resultAsJson, "graph/result"))
{
if (filter.IsMatchedBy(result, query.QueryParameters) == false)
{
matchResults[i] = default;
}
}
}
}
if (query.Start > 0)
{
matchResults.RemoveRange(0, Math.Min(query.Start, matchResults.Count));
}
if (query.PageSize < matchResults.Count)
{
matchResults.RemoveRange(query.PageSize, matchResults.Count - query.PageSize);
}
return(matchResults, qp, false);
}
private async Task <TResult> ExecuteQuery <TResult>(TResult final, IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token) where TResult : QueryResultServerSide <Document>
{
try
{
if (Database.ServerStore.Configuration.Core.FeaturesAvailability == FeaturesAvailability.Stable)
{
FeaturesAvailabilityException.Throw("Graph Queries");
}
using (QueryRunner.MarkQueryAsRunning(Constants.Documents.Indexing.DummyGraphIndexName, query, token))
using (var timingScope = new QueryTimingsScope())
{
var qr = await GetQueryResults(query, documentsContext, existingResultEtag, token);
if (qr.NotModified)
{
final.NotModified = true;
return(final);
}
var q = query.Metadata.Query;
//TODO: handle order by, load, clauses
IncludeDocumentsCommand idc = null;
if (q.Select == null && q.SelectFunctionBody.FunctionText == null)
{
HandleResultsWithoutSelect(documentsContext, qr.Matches, final);
}
else if (q.Select != null)
{
//TODO : investigate fields to fetch
var fieldsToFetch = new FieldsToFetch(query, null);
idc = new IncludeDocumentsCommand(Database.DocumentsStorage, documentsContext, query.Metadata.Includes, fieldsToFetch.IsProjection);
var resultRetriever = new GraphQueryResultRetriever(
q.GraphQuery,
Database,
query,
timingScope,
Database.DocumentsStorage,
documentsContext,
fieldsToFetch,
idc);
HashSet <ulong> alreadySeenProjections = null;
if (q.IsDistinct)
{
alreadySeenProjections = new HashSet <ulong>();
}
foreach (var match in qr.Matches)
{
if (match.Empty)
{
continue;
}
var result = resultRetriever.ProjectFromMatch(match, documentsContext);
// ReSharper disable once PossibleNullReferenceException
if (q.IsDistinct && alreadySeenProjections.Add(result.DataHash) == false)
{
continue;
}
final.AddResult(result);
}
}
if (idc == null)
{
idc = new IncludeDocumentsCommand(Database.DocumentsStorage, documentsContext, query.Metadata.Includes, isProjection: false);
}
if (query.Metadata.Includes?.Length > 0)
{
foreach (var result in final.Results)
{
idc.Gather(result);
}
}
idc.Fill(final.Includes);
final.TotalResults = final.Results.Count;
if (query.Limit != null || query.Offset != null)
{
final.CappedMaxResults = Math.Min(
query.Limit ?? int.MaxValue,
final.TotalResults - (query.Offset ?? 0)
);
}
final.IsStale = qr.QueryPlan.IsStale;
final.ResultEtag = qr.QueryPlan.ResultEtag;
return(final);
}
}
catch (OperationCanceledException oce)
{
throw new OperationCanceledException($"Database:{Database} Query:{query.Metadata.Query} has been cancelled ", oce);
}
}
public override Task ExecuteStreamIndexEntriesQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, HttpResponse response, IStreamQueryResultWriter <BlittableJsonReaderObject> writer,
OperationCancelToken token)
{
throw new NotImplementedException();
}
public override async Task <IndexEntriesQueryResult> ExecuteIndexEntriesQuery(IndexQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token)
{
var index = GetIndex(query.Metadata.IndexName);
if (existingResultEtag.HasValue)
{
var etag = index.GetIndexEtag(query.Metadata);
if (etag == existingResultEtag)
{
return(IndexEntriesQueryResult.NotModifiedResult);
}
}
using (QueryRunner.MarkQueryAsRunning(index.Name, query, token))
{
return(await index.IndexEntries(query, context, token));
}
}
public List <DynamicQueryToIndexMatcher.Explanation> ExplainDynamicIndexSelection(IndexQueryServerSide query)
{
if (query.Metadata.IsDynamic == false)
{
throw new InvalidOperationException("Explain can only work on dynamic indexes");
}
return(_dynamic.ExplainIndexSelection(query));
}
public override Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, PatchRequest patch, BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
var index = GetIndex(query.Metadata.IndexName);
return(ExecutePatch(query, index, options, patch, patchArgs, context, onProgress, token));
}
public override Task ExecuteStreamQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, HttpResponse response, IStreamDocumentQueryResultWriter writer, OperationCancelToken token)
{
var index = GetIndex(query.Metadata.IndexName);
return(index.StreamQuery(response, writer, query, documentsContext, token));
}
public override Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, Patch.PatchRequest patch, BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
throw new NotSupportedException("You cannot patch based on graph query");
}
public abstract Task <IndexEntriesQueryResult> ExecuteIndexEntriesQuery(IndexQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token);
public override Task <SuggestionQueryResult> ExecuteSuggestionQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token)
{
throw new NotSupportedException("You cannot suggest based on graph query");
}
public abstract Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, PatchRequest patch,
BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token);
public override Task <IndexEntriesQueryResult> ExecuteIndexEntriesQuery(IndexQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token)
{
return(GetRunner(query).ExecuteIndexEntriesQuery(query, context, existingResultEtag, token));
}
public override async Task <IndexEntriesQueryResult> ExecuteIndexEntriesQuery(IndexQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token)
{
ObjectDisposedException lastException = null;
for (var i = 0; i < NumberOfRetries; i++)
{
try
{
context.CloseTransaction();
return(await GetRunner(query).ExecuteIndexEntriesQuery(query, context, existingResultEtag, token));
}
catch (ObjectDisposedException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
lastException = e;
}
}
throw CreateRetriesFailedException(lastException);
}
public override Task <IOperationResult> ExecuteDeleteQuery(IndexQueryServerSide query, QueryOperationOptions options, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
return(GetRunner(query).ExecuteDeleteQuery(query, options, context, onProgress, token));
}
public List <DynamicQueryToIndexMatcher.Explanation> ExplainDynamicIndexSelection(IndexQueryServerSide query, DocumentsOperationContext context, out string indexName)
{
if (query.Metadata.IsDynamic == false)
{
throw new InvalidOperationException("Explain can only work on dynamic indexes");
}
if (_dynamic is DynamicQueryRunner d)
{
return(d.ExplainIndexSelection(query, context, out indexName));
}
throw new NotSupportedException(InvalidQueryRunner.ErrorMessage);
}
public override Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, PatchRequest patch, BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
return(GetRunner(query).ExecutePatchQuery(query, options, patch, patchArgs, context, onProgress, token));
}
private List <Slice> ExtractIdsFromQuery(IndexQueryServerSide query)
{
if (string.IsNullOrWhiteSpace(query.Query))
{
return(null);
}
var q = new StringSegment(query.Query.Replace(" ", string.Empty), 0);
if (q.Length <= EqualPrefix.Length)
{
return(null);
}
var documentId = q.SubSegment(0, EqualPrefix.Length);
if (documentId.Equals(EqualPrefix))
{
var id = q.SubSegment(EqualPrefix.Length);
Slice key;
Slice.From(_context.Allocator, id, out key);
_context.Allocator.ToLowerCase(ref key.Content);
return(new List <Slice>
{
key
});
}
if (q.Length <= InPrefix.Length)
{
return(null);
}
var @in = q.SubSegment(0, InPrefix.Length);
if (@in.Equals(InPrefix) == false)
{
return(null);
}
var ids = q.SubSegment(InPrefix.Length + 1, q.Length - InPrefix.Length - 2);
var results = new Slice[0];
int indexOfComma;
do
{
indexOfComma = ids.IndexOfAny(InSeparator, 0);
StringSegment id;
if (indexOfComma != -1)
{
id = ids.SubSegment(0, indexOfComma);
ids = ids.SubSegment(indexOfComma + 1);
}
else
{
id = ids;
}
Slice key;
Slice.From(_context.Allocator, id, out key);
_context.Allocator.ToLowerCase(ref key.Content);
Array.Resize(ref results, results.Length + 1);
results[results.Length - 1] = key;
} while (indexOfComma != -1);
return(results
.OrderBy(x => x, SliceComparer.Instance)
.ToList());
}
public override Task ExecuteStreamQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, HttpResponse response, IStreamDocumentQueryResultWriter writer, OperationCancelToken token)
{
return(GetRunner(query).ExecuteStreamQuery(query, documentsContext, response, writer, token));
}
public Enumerator(DocumentsStorage documents, FieldsToFetch fieldsToFetch, string collection, bool isAllDocsCollection, IndexQueryServerSide query, DocumentsOperationContext context)
{
_documents = documents;
_fieldsToFetch = fieldsToFetch;
_collection = collection;
_isAllDocsCollection = isAllDocsCollection;
_query = query;
_context = context;
if (_fieldsToFetch.IsDistinct)
{
_alreadySeenProjections = new HashSet <ulong>();
}
_ids = ExtractIdsFromQuery(query);
_sort = ExtractSortFromQuery(query);
}
public override Task <IOperationResult> ExecuteDeleteQuery(IndexQueryServerSide query, QueryOperationOptions options, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
var index = GetIndex(query.Metadata.IndexName);
return(ExecuteDelete(query, index, options, context, onProgress, token));
}
