/// <summary>
/// Register the query as a lazy query in the session and return a lazy
/// instance that will evaluate the query only when needed
/// </summary>
public Lazy <IEnumerable <S> > Lazily <S>(Expression expression, Action <IEnumerable <S> > onEval)
{
var processor = GetQueryProviderProcessor <S>();
var query = processor.GetDocumentQueryFor(expression);
if (afterQueryExecuted != null)
{
query.AfterQueryExecuted(afterQueryExecuted);
}
var renamedFields = FieldsToFetch.Select(field =>
{
var renamedField = FieldsToRename.FirstOrDefault(x => x.OriginalField == field);
if (renamedField != null)
{
return(renamedField.NewField ?? field);
}
return(field);
}).ToArray();
if (renamedFields.Length > 0)
{
query.AfterQueryExecuted(processor.RenameResults);
}
if (FieldsToFetch.Count > 0)
{
query = query.SelectFields <S>(FieldsToFetch.ToArray(), renamedFields);
}
return(query.Lazily(onEval));
}
public override int GetHashCode()
{
unchecked
{
var hashCode = PageSizeSet.GetHashCode();
hashCode = (hashCode * 397) ^ (Query != null ? Query.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ (TotalSize != null ? TotalSize.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ (QueryInputs != null ? QueryInputs.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ Start;
hashCode = (hashCode * 397) ^ (IsDistinct ? 1 : 0);
hashCode = (hashCode * 397) ^ (FieldsToFetch != null ? FieldsToFetch.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ (SortedFields != null ? SortedFields.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ Cutoff.GetHashCode();
hashCode = (hashCode * 397) ^ WaitForNonStaleResultsAsOfNow.GetHashCode();
hashCode = (hashCode * 397) ^ (CutoffEtag != null ? CutoffEtag.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ (DefaultField != null ? DefaultField.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ (int)DefaultOperator;
hashCode = (hashCode * 397) ^ SkipTransformResults.GetHashCode();
hashCode = (hashCode * 397) ^ (SkippedResults != null ? SkippedResults.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ DebugOptionGetIndexEntries.GetHashCode();
hashCode = (hashCode * 397) ^ (HighlightedFields != null ? HighlightedFields.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ (HighlighterPreTags != null ? HighlighterPreTags.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ (HighlighterPostTags != null ? HighlighterPostTags.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ (ResultsTransformer != null ? ResultsTransformer.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ DisableCaching.GetHashCode();
return(hashCode);
}
}
protected virtual void ValidateFieldsToFetch(FieldsToFetch fieldsToFetch)
{
if (fieldsToFetch == null)
{
throw new ArgumentNullException(nameof(fieldsToFetch));
}
}
private static BlittableJsonReaderObject GetReduceResult(ulong reduceKeyHash, Index index, TransactionOperationContext context)
{
using (var reader = index.IndexPersistence.OpenIndexReader(context.Transaction.InnerTransaction))
{
var queryParameters = context.ReadObject(new DynamicJsonValue
{
["p0"] = reduceKeyHash.ToString()
}, "query/parameters");
var query = new IndexQueryServerSide($"FROM INDEX '{index.Name}' WHERE '{Constants.Documents.Indexing.Fields.ReduceKeyHashFieldName}' = $p0", queryParameters);
var fieldsToFetch = new FieldsToFetch(query, index.Definition);
var retriever = new MapReduceQueryResultRetriever(null, null, null, context, fieldsToFetch, null);
var result = reader
.Query(query, fieldsToFetch, new Reference <int>(), new Reference <int>(), retriever, context, null, CancellationToken.None)
.ToList();
if (result.Count != 1)
{
throw new InvalidOperationException("Cannot have multiple reduce results for a single reduce key");
}
return(result[0].Data);
}
}
protected override void ValidateFieldsToFetch(FieldsToFetch fieldsToFetch)
{
base.ValidateFieldsToFetch(fieldsToFetch);
if (fieldsToFetch.Projection.MustExtractFromDocument)
{
throw new InvalidQueryException($"Invalid projection behavior '{_query.ProjectionBehavior}'. You can only extract values from index.", _query.Query, _query.QueryParameters);
}
}
protected QueryResultRetrieverBase(DocumentDatabase database, IndexQueryServerSide query, FieldsToFetch fieldsToFetch, DocumentsStorage documentsStorage, JsonOperationContext context, bool reduceResults, IncludeDocumentsCommand includeDocumentsCommand)
{
_database = database;
_query = query;
_context = context;
_includeDocumentsCommand = includeDocumentsCommand;
DocumentsStorage = documentsStorage;
FieldsToFetch = fieldsToFetch;
_blittableTraverser = reduceResults ? BlittableJsonTraverser.FlatMapReduceResults : BlittableJsonTraverser.Default;
}
protected override void ValidateFieldsToFetch(FieldsToFetch fieldsToFetch)
{
base.ValidateFieldsToFetch(fieldsToFetch);
if (_query.ProjectionBehavior == null || _query.ProjectionBehavior == ProjectionBehavior.Default)
{
return;
}
throw new InvalidQueryException($"Invalid projection behavior '{_query.ProjectionBehavior}'.Only default projection behavior can be used for graph queries.", _query.Query, _query.QueryParameters);
}
private JsonDocument RetrieveDocumentInternal(
IndexQueryResult queryResult,
HashSet<string> loadedIds,
FieldsToFetch fieldsToFetch,
IndexDefinition indexDefinition)
{
if (queryResult.Projection == null)
{
// duplicate document, filter it out
if (loadedIds.Add(queryResult.Key) == false)
return null;
return GetDocumentWithCaching(queryResult.Key);
}
if (fieldsToFetch.IsProjection)
{
if (indexDefinition.IsMapReduce == false)
{
bool hasStoredFields = false;
foreach (var fieldToFetch in fieldsToFetch)
{
FieldStorage value;
if (indexDefinition.Stores.TryGetValue(fieldToFetch, out value) == false &&
value != FieldStorage.No)
continue;
hasStoredFields = true;
}
if (hasStoredFields == false)
{
// duplicate document, filter it out
if (loadedIds.Add(queryResult.Key) == false)
return null;
}
}
var fieldsToFetchFromDocument = fieldsToFetch.Where(fieldToFetch => queryResult.Projection.Property(fieldToFetch) == null);
var doc = GetDocumentWithCaching(queryResult.Key);
if (doc != null)
{
var result = doc.DataAsJson.SelectTokenWithRavenSyntax(fieldsToFetchFromDocument.ToArray());
foreach (var property in result.Properties())
{
if(property.Value == null || property.Value.Type == JTokenType.Null)
continue;
queryResult.Projection[property.Name] = property.Value;
}
}
}
return new JsonDocument
{
Key = queryResult.Key,
Projection = queryResult.Projection,
};
}
public GraphQueryResultRetriever(GraphQuery graphQuery, DocumentDatabase database,
IndexQueryServerSide query,
QueryTimingsScope queryTimings,
DocumentsStorage documentsStorage,
DocumentsOperationContext context,
FieldsToFetch fieldsToFetch,
IncludeDocumentsCommand includeDocumentsCommand)
: base(database, query, queryTimings, fieldsToFetch, documentsStorage, context, false, includeDocumentsCommand)
{
_graphQuery = graphQuery;
_context = context;
}
public void AppendQueryString(StringBuilder path)
{
path.Append("?");
AppendMinimalQueryString(path);
if (Start != 0)
{
path.Append("&start=").Append(Start);
}
path.Append("&pageSize=").Append(PageSize);
if (AggregationOperation != AggregationOperation.None)
{
path.Append("&aggregation=").Append(AggregationOperation);
}
FieldsToFetch.ApplyIfNotNull(field => path.Append("&fetch=").Append(Uri.EscapeDataString(field)));
GroupBy.ApplyIfNotNull(field => path.Append("&groupBy=").Append(Uri.EscapeDataString(field)));
SortedFields.ApplyIfNotNull(
field => path.Append("&sort=").Append(field.Descending ? "-" : "").Append(Uri.EscapeDataString(field.Field)));
if (SkipTransformResults)
{
path.Append("&skipTransformResults=true");
}
if (Cutoff != null)
{
var cutOffAsString =
Uri.EscapeUriString(Uri.EscapeDataString(Cutoff.Value.ToString("o", CultureInfo.InvariantCulture)));
path.Append("&cutOff=").Append(cutOffAsString);
}
if (CutoffEtag != null)
{
path.Append("&cutOffEtag=").Append(CutoffEtag.Value.ToString());
}
this.HighlightedFields.ApplyIfNotNull(field => path.Append("&highlight=").Append(field));
this.HighlighterPreTags.ApplyIfNotNull(tag => path.Append("&preTags=").Append(tag));
this.HighlighterPostTags.ApplyIfNotNull(tag => path.Append("&postTags=").Append(tag));
if (DebugOptionGetIndexEntries)
{
path.Append("&debug=entries");
}
}
protected QueryResultRetrieverBase(DocumentDatabase database, IndexQueryServerSide query, QueryTimingsScope queryTimings, FieldsToFetch fieldsToFetch, DocumentsStorage documentsStorage, JsonOperationContext context, bool reduceResults, IncludeDocumentsCommand includeDocumentsCommand)
{
_database = database;
_query = query;
_context = context;
_includeDocumentsCommand = includeDocumentsCommand;
DocumentsStorage = documentsStorage;
RetrieverScope = queryTimings?.For(nameof(QueryTimingsScope.Names.Retriever), start: false);
FieldsToFetch = fieldsToFetch;
_blittableTraverser = reduceResults ? BlittableJsonTraverser.FlatMapReduceResults : BlittableJsonTraverser.Default;
}
public void AppendQueryString(StringBuilder path)
{
path
.Append("?query=");
path.Append(Uri.EscapeUriString(Uri.EscapeDataString(Query ?? "")))
.Append("&start=").Append(Start)
.Append("&pageSize=").Append(PageSize)
.Append("&aggregation=").Append(AggregationOperation);
FieldsToFetch.ApplyIfNotNull(field => path.Append("&fetch=").Append(Uri.EscapeDataString(field)));
GroupBy.ApplyIfNotNull(field => path.Append("&groupBy=").Append(Uri.EscapeDataString(field)));
SortedFields.ApplyIfNotNull(
field => path.Append("&sort=").Append(field.Descending ? "-" : "").Append(Uri.EscapeDataString(field.Field)));
if (string.IsNullOrEmpty(DefaultField) == false)
{
path.Append("&defaultField=").Append(Uri.EscapeDataString(DefaultField));
}
if (DefaultOperator != QueryOperator.Or)
{
path.Append("&operator=AND");
}
if (SkipTransformResults)
{
path.Append("&skipTransformResults=true");
}
if (Cutoff != null)
{
var cutOffAsString =
Uri.EscapeUriString(Uri.EscapeDataString(Cutoff.Value.ToString("o", CultureInfo.InvariantCulture)));
path.Append("&cutOff=").Append(cutOffAsString);
}
if (CutoffEtag != null)
{
path.Append("&cutOffEtag=").Append(CutoffEtag.Value.ToString());
}
var vars = GetCustomQueryStringVariables();
if (!string.IsNullOrEmpty(vars))
{
path.Append(vars.StartsWith("&") ? vars : ("&" + vars));
}
if (DebugOptionGetIndexEntries)
{
path.Append("&debug=entries");
}
}
public Lazy <IEnumerable <S> > Lazily <S>(Expression expression, Action <IEnumerable <S> > onEval)
{
var processor = GetQueryProviderProcessor <S>();
var query = processor.GetLuceneQueryFor(expression);
if (afterQueryExecuted != null)
{
query.AfterQueryExecuted(afterQueryExecuted);
}
if (FieldsToFetch.Count > 0)
{
query = query.SelectFields <S>(FieldsToFetch.ToArray());
}
return(query.Lazily(onEval));
}
public IndexQueryingScope(IndexType indexType, IndexQueryServerSide query, FieldsToFetch fieldsToFetch, IndexSearcher searcher, IQueryResultRetriever retriever, IState state)
{
_indexType = indexType;
_query = query;
_fieldsToFetch = fieldsToFetch;
_searcher = searcher;
_retriever = retriever;
_state = state;
_isSortingQuery = query.Metadata.OrderBy != null;
if (_fieldsToFetch.IsDistinct)
{
_alreadySeenProjections = new HashSet <ulong>();
}
}
public static Document GetProjectionFromDocument(Document doc, float score, FieldsToFetch fieldsToFetch, JsonOperationContext context)
{
var result = new DynamicJsonValue();
if (fieldsToFetch.IsDistinct == false && doc.Key != null)
{
result[Constants.Indexing.Fields.DocumentIdFieldName] = doc.Key;
}
foreach (var fieldToFetch in fieldsToFetch.Fields.Values)
{
MaybeExtractValueFromDocument(fieldToFetch, doc, result);
}
return(ReturnProjection(result, doc, score, context));
}
public override int GetHashCode()
{
unchecked
{
var hashCode = PageSizeSet.GetHashCode();
hashCode = (hashCode * 397) ^ (Query?.GetHashCode() ?? 0);
hashCode = (hashCode * 397) ^ Start;
hashCode = (hashCode * 397) ^ (IsDistinct ? 1 : 0);
hashCode = (hashCode * 397) ^ (FieldsToFetch?.GetHashCode() ?? 0);
hashCode = (hashCode * 397) ^ (WaitForNonStaleResultsTimeout != null ? WaitForNonStaleResultsTimeout.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ WaitForNonStaleResultsAsOfNow.GetHashCode();
hashCode = (hashCode * 397) ^ (CutoffEtag != null ? CutoffEtag.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ (DefaultField?.GetHashCode() ?? 0);
hashCode = (hashCode * 397) ^ (int)DefaultOperator;
return(hashCode);
}
}
private void AddToFieldsToFetch(string docField, string renamedField)
{
var identityProperty = luceneQuery.DocumentConvention.GetIdentityProperty(typeof(T));
if (identityProperty != null && identityProperty.Name == docField)
{
FieldsToFetch.Add(Constants.DocumentIdFieldName);
}
else
{
FieldsToFetch.Add(docField);
}
if (docField != renamedField)
{
FieldsToRename[docField] = renamedField;
}
}
private void ExecuteCollectionQuery(QueryResultServerSide resultToFill, IndexQueryServerSide query, string collection)
{
var isAllDocsCollection = collection == Constants.Documents.Collections.AllDocumentsCollection;
// we optimize for empty queries without sorting options, appending CollectionIndexPrefix to be able to distinguish index for collection vs. physical index
resultToFill.IndexName = isAllDocsCollection ? "AllDocs" : CollectionIndexPrefix + collection;
resultToFill.IsStale = false;
resultToFill.LastQueryTime = DateTime.MinValue;
resultToFill.IndexTimestamp = DateTime.MinValue;
resultToFill.IncludedPaths = query.Metadata.Includes;
var includeDocumentsCommand = new IncludeDocumentsCommand(_documents, _context, query.Metadata.Includes);
var fieldsToFetch = new FieldsToFetch(query, null);
var totalResults = new Reference <int>();
var documents = new CollectionQueryEnumerable(_database, _documents, fieldsToFetch, collection, query, _context, includeDocumentsCommand, totalResults);
var cancellationToken = _token.Token;
try
{
foreach (var document in documents)
{
cancellationToken.ThrowIfCancellationRequested();
resultToFill.AddResult(document);
includeDocumentsCommand.Gather(document);
}
}
catch (Exception e)
{
if (resultToFill.SupportsExceptionHandling == false)
{
throw;
}
resultToFill.HandleException(e);
}
includeDocumentsCommand.Fill(resultToFill.Includes);
resultToFill.TotalResults = totalResults.Value;
}
private static BlittableJsonReaderObject GetReduceResult(ulong reduceKeyHash, Index index, TransactionOperationContext context)
{
using (var reader = index.IndexPersistence.OpenIndexReader(context.Transaction.InnerTransaction))
{
var query = new IndexQueryServerSide
{
Query = $"{Constants.Indexing.Fields.ReduceKeyFieldName}:{reduceKeyHash}"
};
var fieldsToFetch = new FieldsToFetch(query, index.Definition, null);
var result = reader.Query(query, fieldsToFetch, new Reference <int>(), new Reference <int>(),
new MapReduceQueryResultRetriever(context, fieldsToFetch), CancellationToken.None).ToList();
if (result.Count != 1)
{
throw new InvalidOperationException("Cannot have multiple reduce results for a single reduce key");
}
return(result[0].Data);
}
}
protected static Document AddProjectionToResult(Document doc, float score, FieldsToFetch fieldsToFetch, DynamicJsonValue result, string key, object fieldVal)
{
if (fieldsToFetch.SingleBodyOrMethodWithNoAlias)
{
Document newDoc = null;
if (fieldVal is BlittableJsonReaderObject nested)
{
newDoc = new Document
{
Id = doc.Id,
ChangeVector = doc.ChangeVector,
Data = nested,
Etag = doc.Etag,
Flags = doc.Flags,
IndexScore = score,
LastModified = doc.LastModified,
LowerId = doc.LowerId,
NonPersistentFlags = doc.NonPersistentFlags,
StorageId = doc.StorageId,
TransactionMarker = doc.TransactionMarker
};
}
else if (fieldVal is Document d)
{
newDoc = d;
newDoc.IndexScore = score;
}
else
{
ThrowInvalidQueryBodyResponse(fieldVal);
}
return(newDoc);
}
AddProjectionToResult(result, key, fieldVal);
return(null);
}
/// <summary>
/// Convert the expression to a Lucene query
/// </summary>
public IDocumentQuery <TResult> ToDocumentQuery <TResult>(Expression expression)
{
var processor = GetQueryProviderProcessor <T>();
var result = (IDocumentQuery <TResult>)processor.GetDocumentQueryFor(expression);
result.SetResultTransformer(ResultTransformer);
var renamedFields = FieldsToFetch.Select(field =>
{
var renamedField = FieldsToRename.FirstOrDefault(x => x.OriginalField == field);
if (renamedField != null)
{
return(renamedField.NewField ?? field);
}
return(field);
}).ToArray();
if (renamedFields.Length > 0)
{
result.AfterStreamExecuted(processor.RenameSingleResult);
}
return(result);
}
/// <summary>
/// Gets the index query URL.
/// </summary>
/// <param name="operationUrl">The operation URL.</param>
/// <param name="index">The index.</param>
/// <param name="operationName">Name of the operation.</param>
/// <returns></returns>
public string GetIndexQueryUrl(string operationUrl, string index, string operationName)
{
if (operationUrl.EndsWith("/"))
{
operationUrl = operationUrl.Substring(0, operationUrl.Length - 1);
}
var path = new StringBuilder()
.Append(operationUrl)
.Append("/")
.Append(operationName)
.Append("/")
.Append(index)
.Append("?query=").Append(Uri.EscapeUriString(Uri.EscapeDataString(Query ?? "")))
.Append("&start=").Append(Start)
.Append("&pageSize=").Append(PageSize)
.Append("&aggregation=").Append(AggregationOperation);
FieldsToFetch.ApplyIfNotNull(field => path.Append("&fetch=").Append(Uri.EscapeDataString(field)));
GroupBy.ApplyIfNotNull(field => path.Append("&groupBy=").Append(Uri.EscapeDataString(field)));
SortedFields.ApplyIfNotNull(field => path.Append("&sort=").Append(field.Descending ? "-" : "").Append(Uri.EscapeDataString(field.Field)));
if (Cutoff != null)
{
var cutOffAsString =
Uri.EscapeUriString(Uri.EscapeDataString(Cutoff.Value.ToString("o", CultureInfo.InvariantCulture)));
path.Append("&cutOff=").Append(cutOffAsString);
}
var vars = GetCustomQueryStringVariables();
if (!string.IsNullOrEmpty(vars))
{
path.Append(vars.StartsWith("&") ? vars : ("&" + vars));
}
return(path.ToString());
}
public JsonDocument RetrieveDocumentForQuery(IndexQueryResult queryResult, IndexDefinition indexDefinition, FieldsToFetch fieldsToFetch, bool skipDuplicateCheck)
{
return(ExecuteReadTriggers(ProcessReadVetoes(
RetrieveDocumentInternal(queryResult, loadedIdsForRetrieval, fieldsToFetch, indexDefinition, skipDuplicateCheck),
null, ReadOperation.Query), null, ReadOperation.Query));
}
public bool ShouldIncludeResultInQuery(IndexQueryResult arg, IndexDefinition indexDefinition, FieldsToFetch fieldsToFetch, bool skipDuplicateCheck)
{
JsonDocument doc;
if (arg.DocumentLoaded)
{
doc = arg.Document;
}
else
{
doc = RetrieveDocumentInternal(arg, loadedIdsForFilter, fieldsToFetch, indexDefinition, skipDuplicateCheck);
arg.Document = doc;
arg.DocumentLoaded = true;
}
if (doc == null)
{
return(false);
}
doc = ProcessReadVetoes(doc, null, ReadOperation.Query);
return(doc != null);
}
private JsonDocument RetrieveDocumentInternal(
IndexQueryResult queryResult,
HashSet <string> loadedIds,
FieldsToFetch fieldsToFetch,
IndexDefinition indexDefinition,
bool skipDuplicateCheck)
{
var queryScore = queryResult.Score;
if (float.IsNaN(queryScore))
{
queryScore = 0f;
}
if (queryResult.Projection == null)
{
// duplicate document, filter it out
if (skipDuplicateCheck == false && loadedIds.Add(queryResult.Key) == false)
{
return(null);
}
var document = GetDocumentWithCaching(queryResult);
if (document == null)
{
return(null);
}
document.Metadata = GetMetadata(document);
if (skipDuplicateCheck == false)
{
document.Metadata[Constants.TemporaryScoreValue] = queryScore;
}
return(document);
}
JsonDocument doc = null;
if (fieldsToFetch.IsProjection)
{
if (indexDefinition.IsMapReduce == false)
{
bool hasStoredFields = false;
FieldStorage value;
if (indexDefinition.Stores.TryGetValue(Constants.AllFields, out value))
{
hasStoredFields = value != FieldStorage.No;
}
foreach (var fieldToFetch in fieldsToFetch.Fields)
{
if (indexDefinition.Stores.TryGetValue(fieldToFetch, out value) == false && value != FieldStorage.No)
{
continue;
}
hasStoredFields = true;
}
if (hasStoredFields == false)
{
// duplicate document, filter it out
if (loadedIds.Add(queryResult.Key) == false)
{
return(null);
}
}
}
// We have to load the document if user explicitly asked for the id, since
// we normalize the casing for the document id on the index, and we need to return
// the id to the user with the same casing they gave us.
var fetchingId = fieldsToFetch.HasField(Constants.DocumentIdFieldName);
var fieldsToFetchFromDocument = fieldsToFetch.Fields.Where(fieldToFetch => queryResult.Projection[fieldToFetch] == null).ToArray();
if (fieldsToFetchFromDocument.Length > 0 || fetchingId)
{
switch (configuration.ImplicitFetchFieldsFromDocumentMode)
{
case ImplicitFetchFieldsMode.Enabled:
doc = GetDocumentWithCaching(queryResult);
if (doc != null)
{
if (fetchingId)
{
queryResult.Projection[Constants.DocumentIdFieldName] = doc.Key;
}
var result = doc.DataAsJson.SelectTokenWithRavenSyntax(fieldsToFetchFromDocument.ToArray());
foreach (var property in result)
{
if (property.Value == null)
{
continue;
}
queryResult.Projection[property.Key] = property.Value;
}
}
break;
case ImplicitFetchFieldsMode.DoNothing:
break;
case ImplicitFetchFieldsMode.Exception:
string message = string.Format("Implicit fetching of fields from the document is disabled." + Environment.NewLine +
"Check your index ({0}) to make sure that all fields you want to project are stored in the index." + Environment.NewLine +
"You can control this behavior using the Raven/ImplicitFetchFieldsFromDocumentMode setting." + Environment.NewLine +
"Fields to fetch from document are: {1}" + Environment.NewLine +
"Fetching id: {2}", indexDefinition.Name, string.Join(", ", fieldsToFetchFromDocument), fetchingId);
throw new ImplicitFetchFieldsFromDocumentNotAllowedException(message);
default:
throw new ArgumentOutOfRangeException(configuration.ImplicitFetchFieldsFromDocumentMode.ToString());
}
}
}
else if (fieldsToFetch.FetchAllStoredFields && string.IsNullOrEmpty(queryResult.Key) == false &&
(fieldsToFetch.Query == null || fieldsToFetch.Query.AllowMultipleIndexEntriesForSameDocumentToResultTransformer == false)
)
{
// duplicate document, filter it out
if (loadedIds.Add(queryResult.Key) == false)
{
return(null);
}
doc = GetDocumentWithCaching(queryResult);
}
var metadata = GetMetadata(doc);
metadata.Remove("@id");
metadata[Constants.TemporaryScoreValue] = queryScore;
return(new JsonDocument
{
Key = queryResult.Key,
DataAsJson = queryResult.Projection,
Metadata = metadata
});
}
private JsonDocument RetrieveDocumentInternal(
IndexQueryResult queryResult,
HashSet <string> loadedIds,
FieldsToFetch fieldsToFetch,
IndexDefinition indexDefinition)
{
var queryScore = queryResult.Score;
if (float.IsNaN(queryScore))
{
queryScore = 0f;
}
if (queryResult.Projection == null)
{
// duplicate document, filter it out
if (loadedIds.Add(queryResult.Key) == false)
{
return(null);
}
var document = GetDocumentWithCaching(queryResult.Key);
if (document != null)
{
document.Metadata[Constants.TemporaryScoreValue] = queryScore;
}
return(document);
}
if (fieldsToFetch.IsProjection)
{
if (indexDefinition.IsMapReduce == false)
{
bool hasStoredFields = false;
foreach (var fieldToFetch in fieldsToFetch)
{
FieldStorage value;
if (indexDefinition.Stores.TryGetValue(fieldToFetch, out value) == false &&
value != FieldStorage.No)
{
continue;
}
hasStoredFields = true;
}
if (hasStoredFields == false)
{
// duplicate document, filter it out
if (loadedIds.Add(queryResult.Key) == false)
{
return(null);
}
}
}
var fieldsToFetchFromDocument = fieldsToFetch.Where(fieldToFetch => queryResult.Projection[fieldToFetch] == null);
var doc = GetDocumentWithCaching(queryResult.Key);
if (doc != null)
{
var result = doc.DataAsJson.SelectTokenWithRavenSyntax(fieldsToFetchFromDocument.ToArray());
foreach (var property in result)
{
if (property.Value == null || property.Value.Type == JTokenType.Null)
{
continue;
}
queryResult.Projection[property.Key] = property.Value;
}
}
}
return(new JsonDocument
{
Key = queryResult.Key,
DataAsJson = queryResult.Projection,
Metadata = new RavenJObject {
{ Constants.TemporaryScoreValue, queryScore }
}
});
}
public bool ShouldIncludeResultInQuery(IndexQueryResult arg, IndexDefinition indexDefinition, FieldsToFetch fieldsToFetch)
{
var doc = RetrieveDocumentInternal(arg, loadedIdsForFilter, fieldsToFetch, indexDefinition);
if (doc == null)
{
return(false);
}
doc = ProcessReadVetoes(doc, null, ReadOperation.Query);
return(doc != null);
}
public override IQueryResultRetriever GetQueryResultRetriever(IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext documentsContext, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand)
{
return(new MapQueryResultRetriever(DocumentDatabase, query, queryTimings, DocumentDatabase.DocumentsStorage, documentsContext, fieldsToFetch, includeDocumentsCommand));
}
public void AppendQueryString(StringBuilder path, bool includePageSizeEvenIfNotExplicitlySet = true, bool includeQuery = true)
{
path.Append("?");
AppendMinimalQueryString(path, includeQuery);
if (Start != 0)
{
path.Append("&start=").Append(Start);
}
if (includePageSizeEvenIfNotExplicitlySet || PageSizeSet)
{
path.Append("&pageSize=").Append(PageSize);
}
if (AllowMultipleIndexEntriesForSameDocumentToResultTransformer)
{
path.Append("&allowMultipleIndexEntriesForSameDocumentToResultTransformer=true");
}
if (IsDistinct)
{
path.Append("&distinct=true");
}
if (ShowTimings)
{
path.Append("&showTimings=true");
}
if (SkipDuplicateChecking)
{
path.Append("&skipDuplicateChecking=true");
}
FieldsToFetch.ApplyIfNotNull(field => path.Append("&fetch=").Append(Uri.EscapeDataString(field)));
Includes.ApplyIfNotNull(include => path.AppendFormat("&include={0}", Uri.EscapeDataString(include)));
DynamicMapReduceFields.ApplyIfNotNull(field => path.Append("&mapReduce=")
.Append(Uri.EscapeDataString(field.Name))
.Append("-")
.Append(field.OperationType)
.Append("-")
.Append(field.IsGroupBy));
SortedFields.ApplyIfNotNull(
field => path.Append("&sort=").Append(field.Descending ? "-" : "").Append(Uri.EscapeDataString(field.Field)));
if (string.IsNullOrEmpty(Transformer) == false)
{
path.AppendFormat("&transformer={0}", Uri.EscapeDataString(Transformer));
}
if (TransformerParameters != null)
{
foreach (var input in TransformerParameters)
{
path.AppendFormat("&tp-{0}={1}", input.Key, input.Value);
}
}
if (CutoffEtag != null)
{
path.Append("&cutOffEtag=").Append(CutoffEtag);
}
if (WaitForNonStaleResultsAsOfNow)
{
path.Append("&waitForNonStaleResultsAsOfNow=true");
}
if (WaitForNonStaleResultsTimeout != null)
{
path.AppendLine("&waitForNonStaleResultsTimeout=" + WaitForNonStaleResultsTimeout);
}
HighlightedFields.ApplyIfNotNull(field => path.Append("&highlight=").Append(field));
HighlighterPreTags.ApplyIfNotNull(tag => path.Append("&preTags=").Append(tag));
HighlighterPostTags.ApplyIfNotNull(tag => path.Append("&postTags=").Append(tag));
if (string.IsNullOrEmpty(HighlighterKeyName) == false)
{
path.AppendFormat("&highlighterKeyName={0}", Uri.EscapeDataString(HighlighterKeyName));
}
if (DebugOptionGetIndexEntries)
{
path.Append("&debug=entries");
}
if (ExplainScores)
{
path.Append("&explainScores=true");
}
}
private void ExecuteCollectionQuery(QueryResultServerSide <Document> resultToFill, IndexQueryServerSide query, string collection, DocumentsOperationContext context, CancellationToken cancellationToken)
{
using (var queryScope = query.Timings?.For(nameof(QueryTimingsScope.Names.Query)))
{
QueryTimingsScope gatherScope = null;
QueryTimingsScope fillScope = null;
if (queryScope != null && query.Metadata.Includes?.Length > 0)
{
var includesScope = queryScope.For(nameof(QueryTimingsScope.Names.Includes), start: false);
gatherScope = includesScope.For(nameof(QueryTimingsScope.Names.Gather), start: false);
fillScope = includesScope.For(nameof(QueryTimingsScope.Names.Fill), start: false);
}
var isAllDocsCollection = collection == Constants.Documents.Collections.AllDocumentsCollection;
// we optimize for empty queries without sorting options, appending CollectionIndexPrefix to be able to distinguish index for collection vs. physical index
resultToFill.IndexName = isAllDocsCollection ? "AllDocs" : CollectionIndexPrefix + collection;
resultToFill.IsStale = false;
resultToFill.LastQueryTime = DateTime.MinValue;
resultToFill.IndexTimestamp = DateTime.MinValue;
resultToFill.IncludedPaths = query.Metadata.Includes;
var fieldsToFetch = new FieldsToFetch(query, null);
var includeDocumentsCommand = new IncludeDocumentsCommand(Database.DocumentsStorage, context, query.Metadata.Includes, fieldsToFetch.IsProjection);
var totalResults = new Reference <int>();
var documents = new CollectionQueryEnumerable(Database, Database.DocumentsStorage, fieldsToFetch, collection, query, queryScope, context, includeDocumentsCommand, totalResults);
IncludeCountersCommand includeCountersCommand = null;
if (query.Metadata.HasCounters)
{
includeCountersCommand = new IncludeCountersCommand(
Database,
context,
query.Metadata.CounterIncludes.Counters);
}
try
{
foreach (var document in documents)
{
cancellationToken.ThrowIfCancellationRequested();
resultToFill.AddResult(document);
using (gatherScope?.Start())
includeDocumentsCommand.Gather(document);
includeCountersCommand?.Fill(document);
}
}
catch (Exception e)
{
if (resultToFill.SupportsExceptionHandling == false)
{
throw;
}
resultToFill.HandleException(e);
}
using (fillScope?.Start())
includeDocumentsCommand.Fill(resultToFill.Includes);
if (includeCountersCommand != null)
{
resultToFill.AddCounterIncludes(includeCountersCommand);
}
resultToFill.TotalResults = (totalResults.Value == 0 && resultToFill.Results.Count != 0) ? -1 : totalResults.Value;
}
}
