protected abstract int GetFields<T>(T instance, LazyStringValue key, object document, JsonOperationContext indexContext, IWriteOperationBuffer writeBuffer) where T : ILuceneDocumentWrapper;
protected override int GetFields <T>(T instance, LazyStringValue key, object document, JsonOperationContext indexContext, IWriteOperationBuffer writeBuffer)
{
if (!(document is ObjectInstance documentToProcess))
{
return(0);
}
int newFields = 0;
if (key != null)
{
instance.Add(GetOrCreateKeyField(key));
newFields++;
}
if (_reduceOutput)
{
var reduceResult = JsBlittableBridge.Translate(indexContext,
documentToProcess.Engine,
documentToProcess);
instance.Add(GetReduceResultValueField(reduceResult, writeBuffer));
newFields++;
}
foreach (var(property, propertyDescriptor) in documentToProcess.GetOwnProperties())
{
if (_fields.TryGetValue(property, out var field) == false)
{
field = _fields[property] = IndexField.Create(property, new IndexFieldOptions(), _allFields);
}
object value;
var actualValue = propertyDescriptor.Value;
if (actualValue.IsObject() && actualValue.IsArray() == false)
{
//In case TryDetectDynamicFieldCreation finds a dynamic field it will populate 'field.Name' with the actual property name
//so we must use field.Name and not property from this point on.
var val = TryDetectDynamicFieldCreation(property, actualValue.AsObject(), field);
if (val != null)
{
if (val.IsObject() && val.AsObject().TryGetValue("$spatial", out _))
{
actualValue = val; //Here we populate the dynamic spatial field that will be handled below.
}
else
{
value = TypeConverter.ToBlittableSupportedType(val, flattenArrays: false, forIndexing: true, engine: documentToProcess.Engine, context: indexContext);
newFields += GetRegularFields(instance, field, value, indexContext, out _);
continue;
}
}
var objectValue = actualValue.AsObject();
if (objectValue.HasOwnProperty("$spatial") && objectValue.TryGetValue("$spatial", out var inner))
{
SpatialField spatialField;
IEnumerable <AbstractField> spatial;
if (inner.IsString())
{
spatialField = StaticIndexBase.GetOrCreateSpatialField(field.Name);
spatial = StaticIndexBase.CreateSpatialField(spatialField, inner.AsString());
}
else if (inner.IsObject())
{
var innerObject = inner.AsObject();
if (innerObject.HasOwnProperty("Lat") && innerObject.HasOwnProperty("Lng") && innerObject.TryGetValue("Lat", out var lat) &&
lat.IsNumber() && innerObject.TryGetValue("Lng", out var lng) && lng.IsNumber())
{
spatialField = StaticIndexBase.GetOrCreateSpatialField(field.Name);
spatial = StaticIndexBase.CreateSpatialField(spatialField, lat.AsNumber(), lng.AsNumber());
}
else
{
continue; //Ignoring bad spatial field
}
}
else
{
continue; //Ignoring bad spatial field
}
newFields += GetRegularFields(instance, field, spatial, indexContext, out _);
continue;
}
}
value = TypeConverter.ToBlittableSupportedType(propertyDescriptor.Value, flattenArrays: false, forIndexing: true, engine: documentToProcess.Engine, context: indexContext);
newFields += GetRegularFields(instance, field, value, indexContext, out _);
if (value is IDisposable toDispose)
{
// the value was converted to a lucene field and isn't needed anymore
toDispose.Dispose();
}
}
return(newFields);
}
// returned document needs to be written do index right after conversion because the same cached instance is used here
public IDisposable SetDocument(LazyStringValue key, object document, JsonOperationContext indexContext, IWriteOperationBuffer writeBuffer, out bool shouldSkip)
{
Document.GetFields().Clear();
var scope = CurrentIndexingScope.Current;
if (scope != null)
scope.CreatedFieldsCount = 0;
int numberOfFields = GetFields(new DefaultDocumentLuceneWrapper(Document), key, document, indexContext, writeBuffer);
if (_fields.Count > 0)
{
shouldSkip = _indexEmptyEntries == false && numberOfFields <= 1; // there is always a key field, but we want to filter-out empty documents
}
else
{
shouldSkip = numberOfFields <= 0; // if we have no entries, we might have an index on the id only, so retain it
}
return Scope;
}
protected override int GetFields <T>(T instance, LazyStringValue key, object doc, JsonOperationContext indexContext, IWriteOperationBuffer writeBuffer)
{
int newFields = 0;
var document = (Document)doc;
if (key != null)
{
Debug.Assert(document.LowerId == null || (key == document.LowerId));
instance.Add(GetOrCreateKeyField(key));
newFields++;
}
if (_reduceOutput)
{
instance.Add(GetReduceResultValueField(document.Data, writeBuffer));
newFields++;
}
foreach (var indexField in _fields.Values)
{
object value;
if (indexField.Spatial is AutoSpatialOptions spatialOptions)
{
var spatialField = CurrentIndexingScope.Current.GetOrCreateSpatialField(indexField.Name);
switch (spatialOptions.MethodType)
{
case AutoSpatialOptions.AutoSpatialMethodType.Wkt:
if (BlittableJsonTraverserHelper.TryRead(_blittableTraverser, document, spatialOptions.MethodArguments[0], out var wktValue) == false)
{
continue;
}
value = StaticIndexBase.CreateSpatialField(spatialField, wktValue);
break;
case AutoSpatialOptions.AutoSpatialMethodType.Point:
if (BlittableJsonTraverserHelper.TryRead(_blittableTraverser, document, spatialOptions.MethodArguments[0], out var latValue) == false)
{
continue;
}
if (BlittableJsonTraverserHelper.TryRead(_blittableTraverser, document, spatialOptions.MethodArguments[1], out var lngValue) == false)
{
continue;
}
value = StaticIndexBase.CreateSpatialField(spatialField, latValue, lngValue);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
else
{
if (BlittableJsonTraverserHelper.TryRead(_blittableTraverser, document, indexField.OriginalName ?? indexField.Name, out value) == false)
{
continue;
}
}
newFields += GetRegularFields(instance, indexField, value, indexContext, out _);
}
return(newFields);
}
protected override int GetFields <T>(T instance, LazyStringValue key, object document, JsonOperationContext indexContext, IWriteOperationBuffer writeBuffer)
{
int newFields = 0;
if (key != null)
{
instance.Add(GetOrCreateKeyField(key));
newFields++;
}
var boostedValue = document as BoostedValue;
var documentToProcess = boostedValue == null ? document : boostedValue.Value;
IPropertyAccessor accessor;
if (_isMultiMap == false)
{
accessor = _propertyAccessor ?? (_propertyAccessor = PropertyAccessor.Create(documentToProcess.GetType(), documentToProcess));
}
else
{
accessor = TypeConverter.GetPropertyAccessor(documentToProcess);
}
var reduceResult = _reduceOutput ? new DynamicJsonValue() : null;
foreach (var property in accessor.GetPropertiesInOrder(documentToProcess))
{
var value = property.Value;
IndexField field;
try
{
field = _fields[property.Key];
}
catch (KeyNotFoundException e)
{
throw new InvalidOperationException($"Field '{property.Key}' is not defined. Available fields: {string.Join(", ", _fields.Keys)}.", e);
}
var numberOfCreatedFields = GetRegularFields(instance, field, value, indexContext, out var shouldSkip);
newFields += numberOfCreatedFields;
if (boostedValue != null)
{
var fields = instance.GetFields();
for (int idx = fields.Count - 1; numberOfCreatedFields > 0; numberOfCreatedFields--, idx--)
{
var luceneField = fields[idx];
luceneField.Boost = boostedValue.Boost;
luceneField.OmitNorms = false;
}
}
if (reduceResult != null && shouldSkip == false)
{
reduceResult[property.Key] = TypeConverter.ToBlittableSupportedType(value, flattenArrays: true);
}
}
if (_reduceOutput)
{
instance.Add(GetReduceResultValueField(Scope.CreateJson(reduceResult, indexContext), writeBuffer));
newFields++;
}
return(newFields);
}
// returned document needs to be written do index right after conversion because the same cached instance is used here
public IDisposable SetDocument(LazyStringValue key, LazyStringValue sourceDocumentId, object document, JsonOperationContext indexContext, IWriteOperationBuffer writeBuffer, out bool shouldSkip)
{
Document.GetFields().Clear();
int numberOfFields = GetFields(new DefaultDocumentLuceneWrapper(Document), key, sourceDocumentId, document, indexContext, writeBuffer);
if (_fields.Count > 0)
{
shouldSkip = _indexEmptyEntries == false && numberOfFields <= _numberOfBaseFields; // there is always a key field, but we want to filter-out empty documents, some indexes (e.g. TS indexes contain more than 1 field by default)
}
else
{
shouldSkip = numberOfFields <= 0; // if we have no entries, we might have an index on the id only, so retain it
}
return(Scope);
}
protected override int GetFields <T>(T instance, LazyStringValue key, LazyStringValue sourceDocumentId, object document, JsonOperationContext indexContext, IWriteOperationBuffer writeBuffer)
{
if (!(document is ObjectInstance documentToProcess))
{
return(0);
}
int newFields = 0;
if (key != null)
{
instance.Add(GetOrCreateKeyField(key));
newFields++;
}
if (sourceDocumentId != null)
{
instance.Add(GetOrCreateSourceDocumentIdField(sourceDocumentId));
newFields++;
}
if (_storeValue)
{
var storedValue = JsBlittableBridge.Translate(indexContext,
documentToProcess.Engine,
documentToProcess);
instance.Add(GetStoredValueField(storedValue, writeBuffer));
newFields++;
}
if (TryGetBoostedValue(documentToProcess, out var boostedValue, out var documentBoost))
{
if (IsObject(boostedValue) == false)
{
throw new InvalidOperationException($"Invalid boosted value. Expected object but got '{boostedValue.Type}' with value '{boostedValue}'.");
}
documentToProcess = boostedValue.AsObject();
}
foreach (var(property, propertyDescriptor) in documentToProcess.GetOwnProperties())
{
var propertyAsString = property.AsString();
if (_fields.TryGetValue(propertyAsString, out var field) == false)
{
field = _fields[propertyAsString] = IndexField.Create(propertyAsString, new IndexFieldOptions(), _allFields);
}
object value;
float? propertyBoost = null;
int numberOfCreatedFields;
var actualValue = propertyDescriptor.Value;
var isObject = IsObject(actualValue);
if (isObject)
{
if (TryGetBoostedValue(actualValue.AsObject(), out boostedValue, out propertyBoost))
{
actualValue = boostedValue;
isObject = IsObject(actualValue);
}
if (isObject)
{
//In case TryDetectDynamicFieldCreation finds a dynamic field it will populate 'field.Name' with the actual property name
//so we must use field.Name and not property from this point on.
var val = TryDetectDynamicFieldCreation(propertyAsString, actualValue.AsObject(), field);
if (val != null)
{
if (val.IsObject() && val.AsObject().TryGetValue(SpatialPropertyName, out _))
{
actualValue = val; //Here we populate the dynamic spatial field that will be handled below.
}
else
{
value = TypeConverter.ToBlittableSupportedType(val, flattenArrays: false, forIndexing: true, engine: documentToProcess.Engine, context: indexContext);
numberOfCreatedFields = GetRegularFields(instance, field, CreateValueForIndexing(value, propertyBoost), indexContext, out _);
newFields += numberOfCreatedFields;
BoostDocument(instance, numberOfCreatedFields, documentBoost);
if (value is IDisposable toDispose1)
{
// the value was converted to a lucene field and isn't needed anymore
toDispose1.Dispose();
}
continue;
}
}
var objectValue = actualValue.AsObject();
if (objectValue.HasOwnProperty(SpatialPropertyName) && objectValue.TryGetValue(SpatialPropertyName, out var inner))
{
SpatialField spatialField;
IEnumerable <AbstractField> spatial;
if (inner.IsString())
{
spatialField = AbstractStaticIndexBase.GetOrCreateSpatialField(field.Name);
spatial = AbstractStaticIndexBase.CreateSpatialField(spatialField, inner.AsString());
}
else if (inner.IsObject())
{
var innerObject = inner.AsObject();
if (innerObject.HasOwnProperty("Lat") && innerObject.HasOwnProperty("Lng") && innerObject.TryGetValue("Lat", out var lat) &&
lat.IsNumber() && innerObject.TryGetValue("Lng", out var lng) && lng.IsNumber())
{
spatialField = AbstractStaticIndexBase.GetOrCreateSpatialField(field.Name);
spatial = AbstractStaticIndexBase.CreateSpatialField(spatialField, lat.AsNumber(), lng.AsNumber());
}
else
{
continue; //Ignoring bad spatial field
}
}
else
{
continue; //Ignoring bad spatial field
}
numberOfCreatedFields = GetRegularFields(instance, field, CreateValueForIndexing(spatial, propertyBoost), indexContext, out _);
newFields += numberOfCreatedFields;
BoostDocument(instance, numberOfCreatedFields, documentBoost);
continue;
}
}
}
value = TypeConverter.ToBlittableSupportedType(actualValue, flattenArrays: false, forIndexing: true, engine: documentToProcess.Engine, context: indexContext);
numberOfCreatedFields = GetRegularFields(instance, field, CreateValueForIndexing(value, propertyBoost), indexContext, out _);
newFields += numberOfCreatedFields;
BoostDocument(instance, numberOfCreatedFields, documentBoost);
if (value is IDisposable toDispose)
{
// the value was converted to a lucene field and isn't needed anymore
toDispose.Dispose();
}
}
return(newFields);
