private bool TryGetFieldValueFromDocument(Document document, FieldsToFetch.FieldToFetch field, out object value)
{
if (field.IsDocumentId)
{
value = document.Id;
}
else if (field.IsCompositeField == false)
{
if (BlittableJsonTraverserHelper.TryRead(_blittableTraverser, document, field.Name, out value) == false)
{
return(false);
}
}
else
{
var component = new DynamicJsonValue();
foreach (var componentField in field.Components)
{
if (BlittableJsonTraverserHelper.TryRead(_blittableTraverser, document, componentField, out var componentValue))
{
component[componentField] = componentValue;
}
}
value = component;
}
return(true);
}
private static void MaybeExtractValueFromDocument(FieldsToFetch.FieldToFetch fieldToFetch, Document document, DynamicJsonValue toFill)
{
object value;
if (BlittableJsonTraverserHelper.TryRead(BlittableJsonTraverser.Default, document, fieldToFetch.Name, out value) == false)
{
return;
}
toFill[fieldToFetch.Name.Value] = value;
}
protected object GetFunctionValue(FieldsToFetch.FieldToFetch fieldToFetch, string documentId, object[] args)
{
using (_functionScope = _functionScope?.Start() ?? _projectionScope?.For(nameof(QueryTimingsScope.Names.JavaScript)))
{
args[args.Length - 1] = _query.QueryParameters;
var value = InvokeFunction(
fieldToFetch.QueryField.Name,
_query.Metadata.Query,
documentId,
args,
_functionScope);
return(value);
}
}
private bool TryExtractValueFromIndex(FieldsToFetch.FieldToFetch fieldToFetch, Lucene.Net.Documents.Document indexDocument, DynamicJsonValue toFill, IState state)
{
if (fieldToFetch.CanExtractFromIndex == false)
{
return(false);
}
var name = fieldToFetch.ProjectedName ?? fieldToFetch.Name.Value;
DynamicJsonArray array = null;
FieldType fieldType = null;
var anyExtracted = false;
foreach (var field in indexDocument.GetFields(fieldToFetch.Name.Value))
{
if (fieldType == null)
{
fieldType = GetFieldType(field.Name, indexDocument);
}
var fieldValue = ConvertType(_context, field, fieldType, state);
if (fieldType.IsArray)
{
if (array == null)
{
array = new DynamicJsonArray();
toFill[name] = array;
}
array.Add(fieldValue);
anyExtracted = true;
continue;
}
toFill[name] = fieldValue;
anyExtracted = true;
}
return(anyExtracted);
}
private static void ThrowOnlyArrayFieldCanHaveMultipleValues(FieldsToFetch.FieldToFetch fieldToFetch)
{
throw new NotSupportedException(
$"Attempted to read multiple values in field {fieldToFetch.ProjectedName ?? fieldToFetch.Name.Value}, but it isn't an array and should have only a single value, did you forget '[]' ?");
}
protected bool TryGetValue(FieldsToFetch.FieldToFetch fieldToFetch, Document document, Lucene.Net.Documents.Document luceneDoc, IState state, Dictionary <string, IndexField> indexFields, bool?anyDynamicIndexFields, out string key, out object value)
{
key = fieldToFetch.ProjectedName ?? fieldToFetch.Name.Value;
if (fieldToFetch.QueryField == null)
{
return(TryGetFieldValueFromDocument(document, fieldToFetch, out value));
}
if (fieldToFetch.QueryField.Function != null)
{
var args = new object[fieldToFetch.QueryField.FunctionArgs.Length + 1];
for (int i = 0; i < fieldToFetch.FunctionArgs.Length; i++)
{
TryGetValue(fieldToFetch.FunctionArgs[i], document, luceneDoc, state, indexFields, anyDynamicIndexFields, out _, out args[i]);
if (ReferenceEquals(args[i], document))
{
args[i] = Tuple.Create(document, luceneDoc, state, indexFields, anyDynamicIndexFields);
}
}
value = GetFunctionValue(fieldToFetch, document.Id, args);
return(true);
}
if (fieldToFetch.QueryField.IsCounter)
{
string name;
string id = document.Id;
if (fieldToFetch.QueryField.IsParameter)
{
if (_query.QueryParameters == null)
{
throw new InvalidQueryException("The query is parametrized but the actual values of parameters were not provided", _query.Query, null);
}
if (_query.QueryParameters.TryGetMember(fieldToFetch.QueryField.Name, out var nameObject) == false)
{
throw new InvalidQueryException($"Value of parameter '{fieldToFetch.QueryField.Name}' was not provided", _query.Query, _query.QueryParameters);
}
name = nameObject.ToString();
key = fieldToFetch.QueryField.Alias ?? name;
}
else
{
name = fieldToFetch.Name.Value;
}
if (fieldToFetch.QueryField.SourceAlias != null &&
BlittableJsonTraverser.Default.TryRead(document.Data, fieldToFetch.QueryField.SourceAlias, out var sourceId, out _))
{
id = sourceId.ToString();
}
if (fieldToFetch.QueryField.FunctionArgs != null)
{
value = GetCounterRaw(id, name);
}
else
{
value = GetCounter(id, name);
}
return(true);
}
if (fieldToFetch.QueryField.ValueTokenType != null)
{
var val = fieldToFetch.QueryField.Value;
if (fieldToFetch.QueryField.ValueTokenType.Value == ValueTokenType.Parameter)
{
if (_query == null)
{
value = null;
return(false); // only happens for debug endpoints and more like this
}
_query.QueryParameters.TryGet((string)val, out val);
}
value = val;
return(true);
}
if (fieldToFetch.QueryField.HasSourceAlias == false)
{
return(TryGetFieldValueFromDocument(document, fieldToFetch, out value));
}
if (_loadedDocumentIds == null)
{
_loadedDocumentIds = new HashSet <string>();
_loadedDocuments = new Dictionary <string, Document>();
_loadedDocumentsByAliasName = new Dictionary <string, Document>();
}
_loadedDocumentIds.Clear();
//_loadedDocuments.Clear(); - explicitly not clearing this, we want to cache this for the duration of the query
_loadedDocuments[document.Id ?? string.Empty] = document;
if (fieldToFetch.QueryField.SourceAlias != null)
{
if (fieldToFetch.QueryField.IsQuoted)
{
_loadedDocumentIds.Add(fieldToFetch.QueryField.SourceAlias);
}
else if (fieldToFetch.QueryField.IsParameter)
{
if (_query.QueryParameters == null)
{
throw new InvalidQueryException("The query is parametrized but the actual values of parameters were not provided", _query.Query, (BlittableJsonReaderObject)null);
}
if (_query.QueryParameters.TryGetMember(fieldToFetch.QueryField.SourceAlias, out var id) == false)
{
throw new InvalidQueryException($"Value of parameter '{fieldToFetch.QueryField.SourceAlias}' was not provided", _query.Query, _query.QueryParameters);
}
_loadedDocumentIds.Add(id.ToString());
}
else if (fieldToFetch.QueryField.LoadFromAlias != null)
{
if (_loadedDocumentsByAliasName.TryGetValue(fieldToFetch.QueryField.LoadFromAlias, out var loadedDoc))
{
IncludeUtil.GetDocIdFromInclude(loadedDoc.Data, fieldToFetch.QueryField.SourceAlias, _loadedDocumentIds, _database.IdentityPartsSeparator);
}
}
else if (fieldToFetch.CanExtractFromIndex)
{
if (luceneDoc != null)
{
var field = luceneDoc.GetField(fieldToFetch.QueryField.SourceAlias);
if (field != null)
{
var fieldValue = ConvertType(_context, field, GetFieldType(field.Name, luceneDoc), state);
_loadedDocumentIds.Add(fieldValue.ToString());
}
}
}
else
{
IncludeUtil.GetDocIdFromInclude(document.Data, fieldToFetch.QueryField.SourceAlias, _loadedDocumentIds, _database.IdentityPartsSeparator);
}
}
else
{
_loadedDocumentIds.Add(document.Id ?? string.Empty); // null source alias is the root doc
_loadedDocumentsByAliasName.Clear();
}
if (_loadedDocumentIds.Count == 0)
{
if (fieldToFetch.QueryField.SourceIsArray)
{
value = new List <object>();
return(true);
}
value = null;
return(false);
}
var buffer = new List <object>();
foreach (var docId in _loadedDocumentIds)
{
if (docId == null)
{
continue;
}
if (_loadedDocuments.TryGetValue(docId, out var doc) == false)
{
using (_loadScope = _loadScope?.Start() ?? _projectionScope?.For(nameof(QueryTimingsScope.Names.Load)))
_loadedDocuments[docId] = doc = LoadDocument(docId);
}
if (doc == null)
{
continue;
}
if (fieldToFetch.QueryField.Alias != null)
{
_loadedDocumentsByAliasName[fieldToFetch.QueryField.Alias] = doc;
}
if (string.IsNullOrEmpty(fieldToFetch.Name.Value)) // we need the whole document here
{
buffer.Add(doc);
continue;
}
if (TryGetFieldValueFromDocument(doc, fieldToFetch, out var val))
{
if (val is string == false && val is LazyStringValue == false && val is System.Collections.IEnumerable items)
{
// we flatten arrays in projections
foreach (var item in items)
{
buffer.Add(item);
}
fieldToFetch.QueryField.SourceIsArray = true;
}
else
{
buffer.Add(val);
}
}
}
if (fieldToFetch.QueryField.SourceIsArray)
{
value = buffer;
return(true);
}
if (buffer.Count > 0)
{
if (buffer.Count > 1)
{
ThrowOnlyArrayFieldCanHaveMultipleValues(fieldToFetch);
}
value = buffer[0];
return(true);
}
value = null;
return(false);
}
private bool TryGetValue(FieldsToFetch.FieldToFetch fieldToFetch, Document document, Lucene.Net.Documents.Document luceneDoc, IState state, out object value)
{
if (fieldToFetch.QueryField == null)
{
return(TryGetFieldValueFromDocument(document, fieldToFetch, out value));
}
if (fieldToFetch.QueryField.Function != null)
{
var args = new object[fieldToFetch.QueryField.FunctionArgs.Length + 1];
for (int i = 0; i < fieldToFetch.FunctionArgs.Length; i++)
{
TryGetValue(fieldToFetch.FunctionArgs[i], document, luceneDoc, state, out args[i]);
if (ReferenceEquals(args[i], document))
{
args[i] = Tuple.Create(document, luceneDoc, state);
}
}
args[args.Length - 1] = _query.QueryParameters;
value = InvokeFunction(
fieldToFetch.QueryField.Name,
_query.Metadata.Query,
args);
return(true);
}
if (fieldToFetch.QueryField.ValueTokenType != null)
{
var val = fieldToFetch.QueryField.Value;
if (fieldToFetch.QueryField.ValueTokenType.Value == ValueTokenType.Parameter)
{
if (_query == null)
{
value = null;
return(false); // only happens for debug endpoints and more like this
}
_query.QueryParameters.TryGet((string)val, out val);
}
value = val;
return(true);
}
if (fieldToFetch.QueryField.HasSourceAlias == false)
{
return(TryGetFieldValueFromDocument(document, fieldToFetch, out value));
}
if (_loadedDocumentIds == null)
{
_loadedDocumentIds = new HashSet <string>();
_loadedDocuments = new Dictionary <string, Document>();
}
_loadedDocumentIds.Clear();
//_loadedDocuments.Clear(); - explicitly not clearing this, we want to cahce this for the duration of the query
_loadedDocuments[document.Id ?? string.Empty] = document;
if (fieldToFetch.QueryField.SourceAlias != null)
{
IncludeUtil.GetDocIdFromInclude(document.Data, fieldToFetch.QueryField.SourceAlias, _loadedDocumentIds);
}
else
{
_loadedDocumentIds.Add(document.Id ?? string.Empty); // null source alias is the root doc
}
if (_loadedDocumentIds.Count == 0)
{
value = null;
return(false);
}
var buffer = new List <object>();
foreach (var docId in _loadedDocumentIds)
{
if (docId == null)
{
continue;
}
if (_loadedDocuments.TryGetValue(docId, out var doc) == false)
{
_loadedDocuments[docId] = doc = LoadDocument(docId);
}
if (doc == null)
{
continue;
}
if (string.IsNullOrEmpty(fieldToFetch.Name)) // we need the whole document here
{
buffer.Add(doc);
continue;
}
if (TryGetFieldValueFromDocument(doc, fieldToFetch, out var val))
{
buffer.Add(val);
}
}
if (fieldToFetch.QueryField.SourceIsArray)
{
value = buffer;
return(true);
}
if (buffer.Count > 0)
{
if (buffer.Count > 1)
{
ThrowOnlyArrayFieldCanHaveMultipleValues(fieldToFetch);
}
value = buffer[0];
return(true);
}
value = null;
return(false);
}
private bool TryGetValue(FieldsToFetch.FieldToFetch fieldToFetch, Document document, Lucene.Net.Documents.Document luceneDoc, IState state, out object value)
{
if (fieldToFetch.QueryField == null)
{
return(TryGetFieldValueFromDocument(document, fieldToFetch, out value));
}
if (fieldToFetch.QueryField.Function != null)
{
var args = new object[fieldToFetch.QueryField.FunctionArgs.Length + 1];
for (int i = 0; i < fieldToFetch.FunctionArgs.Length; i++)
{
TryGetValue(fieldToFetch.FunctionArgs[i], document, luceneDoc, state, out args[i]);
if (ReferenceEquals(args[i], document))
{
args[i] = Tuple.Create(document, luceneDoc, state);
}
}
args[args.Length - 1] = _query.QueryParameters;
value = InvokeFunction(
fieldToFetch.QueryField.Name,
_query.Metadata.Query,
args);
return(true);
}
if (fieldToFetch.QueryField.ValueTokenType != null)
{
var val = fieldToFetch.QueryField.Value;
if (fieldToFetch.QueryField.ValueTokenType.Value == ValueTokenType.Parameter)
{
if (_query == null)
{
value = null;
return(false); // only happens for debug endpoints and more like this
}
_query.QueryParameters.TryGet((string)val, out val);
}
value = val;
return(true);
}
if (fieldToFetch.QueryField.HasSourceAlias == false)
{
return(TryGetFieldValueFromDocument(document, fieldToFetch, out value));
}
if (_loadedDocumentIds == null)
{
_loadedDocumentIds = new HashSet <string>();
_loadedDocuments = new Dictionary <string, Document>();
_loadedDocumentsByAliasName = new Dictionary <string, Document>();
}
_loadedDocumentIds.Clear();
//_loadedDocuments.Clear(); - explicitly not clearing this, we want to cache this for the duration of the query
_loadedDocuments[document.Id ?? string.Empty] = document;
if (fieldToFetch.QueryField.SourceAlias != null)
{
if (fieldToFetch.QueryField.IsQuoted)
{
_loadedDocumentIds.Add(fieldToFetch.QueryField.SourceAlias);
}
else if (fieldToFetch.QueryField.IsParameter)
{
if (_query.QueryParameters == null)
{
throw new InvalidQueryException("The query is parametrized but the actual values of parameters were not provided", _query.Query, (BlittableJsonReaderObject)null);
}
if (_query.QueryParameters.TryGetMember(fieldToFetch.QueryField.SourceAlias, out var id) == false)
{
throw new InvalidQueryException($"Value of parameter '{fieldToFetch.QueryField.SourceAlias}' was not provided", _query.Query, _query.QueryParameters);
}
_loadedDocumentIds.Add(id.ToString());
}
else if (fieldToFetch.QueryField.LoadFromAlias != null)
{
if (_loadedDocumentsByAliasName.TryGetValue(fieldToFetch.QueryField.LoadFromAlias, out var loadedDoc))
{
IncludeUtil.GetDocIdFromInclude(loadedDoc.Data, fieldToFetch.QueryField.SourceAlias, _loadedDocumentIds);
}
}
else
{
IncludeUtil.GetDocIdFromInclude(document.Data, fieldToFetch.QueryField.SourceAlias, _loadedDocumentIds);
}
}
else
{
_loadedDocumentIds.Add(document.Id ?? string.Empty); // null source alias is the root doc
_loadedDocumentsByAliasName.Clear();
}
if (_loadedDocumentIds.Count == 0)
{
if (fieldToFetch.QueryField.SourceIsArray)
{
value = new List <object>();
return(true);
}
value = null;
return(false);
}
var buffer = new List <object>();
foreach (var docId in _loadedDocumentIds)
{
if (docId == null)
{
continue;
}
if (_loadedDocuments.TryGetValue(docId, out var doc) == false)
{
_loadedDocuments[docId] = doc = LoadDocument(docId);
}
if (doc == null)
{
continue;
}
if (fieldToFetch.QueryField.Alias != null)
{
_loadedDocumentsByAliasName[fieldToFetch.QueryField.Alias] = doc;
}
if (string.IsNullOrEmpty(fieldToFetch.Name)) // we need the whole document here
{
buffer.Add(doc);
continue;
}
if (TryGetFieldValueFromDocument(doc, fieldToFetch, out var val))
{
if (val is string == false && val is System.Collections.IEnumerable items)
{
// we flatten arrays in projections
foreach (var item in items)
{
buffer.Add(item);
}
fieldToFetch.QueryField.SourceIsArray = true;
}
else
{
buffer.Add(val);
}
}
}
if (fieldToFetch.QueryField.SourceIsArray)
{
value = buffer;
return(true);
}
if (buffer.Count > 0)
{
if (buffer.Count > 1)
{
ThrowOnlyArrayFieldCanHaveMultipleValues(fieldToFetch);
}
value = buffer[0];
return(true);
}
value = null;
return(false);
}
protected Document GetProjection(Lucene.Net.Documents.Document input, float score, string id)
{
Document doc = null;
if (_fieldsToFetch.AnyExtractableFromIndex == false)
{
doc = DirectGet(input, id);
if (doc == null)
{
return(null);
}
return(GetProjectionFromDocument(doc, score, _fieldsToFetch, _context));
}
var documentLoaded = false;
var result = new DynamicJsonValue();
if (_fieldsToFetch.IsDistinct == false && string.IsNullOrEmpty(id) == false)
{
result[Constants.Indexing.Fields.DocumentIdFieldName] = id;
}
Dictionary <string, FieldsToFetch.FieldToFetch> fields;
if (_fieldsToFetch.ExtractAllFromIndexAndDocument)
{
fields = input.GetFields()
.Where(x => x.Name != Constants.Indexing.Fields.DocumentIdFieldName &&
x.Name != Constants.Indexing.Fields.ReduceKeyFieldName &&
x.Name != Constants.Indexing.Fields.ReduceValueFieldName)
.Distinct(UniqueFieldNames.Instance)
.ToDictionary(x => x.Name, x => new FieldsToFetch.FieldToFetch(x.Name, x.IsStored));
doc = DirectGet(input, id);
documentLoaded = true;
if (doc != null)
{
foreach (var name in doc.Data.GetPropertyNames())
{
if (fields.ContainsKey(name))
{
continue;
}
fields[name] = new FieldsToFetch.FieldToFetch(name, canExtractFromIndex: false);
}
}
}
else
{
fields = _fieldsToFetch.Fields;
}
foreach (var fieldToFetch in fields.Values)
{
if (TryExtractValueFromIndex(fieldToFetch, input, result))
{
continue;
}
if (documentLoaded == false)
{
doc = DirectGet(input, id);
documentLoaded = true;
}
if (doc == null)
{
continue;
}
MaybeExtractValueFromDocument(fieldToFetch, doc, result);
}
if (doc == null)
{
doc = new Document
{
Key = _context.GetLazyString(id)
};
}
return(ReturnProjection(result, doc, score, _context));
}
