private static object ToBlittableSupportedType(object root, object value, bool flattenArrays, int recursiveLevel, Engine engine, JsonOperationContext context)
{
if (recursiveLevel > MaxAllowedRecursiveLevelForType)
{
NestingLevelTooDeep(root);
}
if (value is JsValue js)
{
if (js.IsNull() || js.IsUndefined())
{
return(null);
}
if (js.IsString())
{
return(js.AsString());
}
if (js.IsBoolean())
{
return(js.AsBoolean());
}
if (js.IsNumber())
{
return(js.AsNumber());
}
if (js.IsDate())
{
return(js.AsDate().ToDateTime());
}
//object wrapper is an object so it must come before the object
if (js is ObjectWrapper ow)
{
var target = ow.Target;
switch (target)
{
case LazyStringValue lsv:
return(lsv);
case LazyCompressedStringValue lcsv:
return(lcsv);
case LazyNumberValue lnv:
return(lnv); //should be already blittable supported type.
}
ThrowInvalidObject(js);
}
//Array is an object in Jint
else if (js.IsArray())
{
var arr = js.AsArray();
var convertedArray = EnumerateArray(root, arr, flattenArrays, recursiveLevel + 1, engine, context);
return(new DynamicJsonArray(flattenArrays ? Flatten(convertedArray) : convertedArray));
}
else if (js.IsObject())
{
return(JsBlittableBridge.Translate(context, engine, js.AsObject()));
}
ThrowInvalidObject(js);
return(null);
}
if (value == null || value is DynamicNullObject)
{
return(null);
}
if (value is DynamicBlittableJson dynamicDocument)
{
return(dynamicDocument.BlittableJson);
}
if (value is string)
{
return(value);
}
if (value is LazyStringValue || value is LazyCompressedStringValue)
{
return(value);
}
if (value is bool)
{
return(value);
}
if (value is int || value is long || value is double || value is decimal || value is float || value is short || value is byte)
{
return(value);
}
if (value is LazyNumberValue)
{
return(value);
}
if (value is DateTime || value is DateTimeOffset || value is TimeSpan)
{
return(value);
}
if (value is Guid guid)
{
return(guid.ToString("D"));
}
if (value is Enum)
{
return(value.ToString());
}
if (value is IEnumerable <IFieldable> || value is IFieldable)
{
return("__ignored");
}
if (value is IDictionary dictionary)
{
var @object = new DynamicJsonValue();
foreach (var key in dictionary.Keys)
{
var keyAsString = KeyAsString(key: ToBlittableSupportedType(root, key, flattenArrays, recursiveLevel: recursiveLevel + 1, engine: engine, context: context));
@object[keyAsString] = ToBlittableSupportedType(root, dictionary[key], flattenArrays, recursiveLevel: recursiveLevel + 1, engine: engine, context: context);
}
return(@object);
}
if (value is IDictionary <object, object> dDictionary)
{
var @object = new DynamicJsonValue();
foreach (var key in dDictionary.Keys)
{
var keyAsString = KeyAsString(key: ToBlittableSupportedType(root, key, flattenArrays, recursiveLevel: recursiveLevel + 1, engine: engine, context: context));
@object[keyAsString] = ToBlittableSupportedType(root, dDictionary[key], flattenArrays, recursiveLevel: recursiveLevel + 1, engine: engine, context: context);
}
return(@object);
}
if (value is char[] chars)
{
return(new string(chars));
}
if (value is IEnumerable <char> charEnumerable)
{
return(new string(charEnumerable.ToArray()));
}
if (value is byte[] bytes)
{
return(System.Convert.ToBase64String(bytes));
}
if (value is IEnumerable enumerable)
{
if (ShouldTreatAsEnumerable(enumerable))
{
return(EnumerableToJsonArray(flattenArrays ? Flatten(enumerable) : enumerable, root, flattenArrays, recursiveLevel, engine, context));
}
}
var inner = new DynamicJsonValue();
var accessor = GetPropertyAccessor(value);
foreach (var property in accessor.GetPropertiesInOrder(value))
{
var propertyValue = property.Value;
if (propertyValue is IEnumerable <object> propertyValueAsEnumerable && ShouldTreatAsEnumerable(propertyValue))
{
inner[property.Key] = EnumerableToJsonArray(propertyValueAsEnumerable, root, flattenArrays, recursiveLevel, engine, context);
continue;
}
inner[property.Key] = ToBlittableSupportedType(root, propertyValue, flattenArrays, recursiveLevel + 1, engine, context);
}
return(inner);
}
public async Task TestJavaScriptIndex()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var input = await context.ReadForMemoryAsync(RequestBodyStream(), "TestJavaScriptIndex");
if (input.TryGet("Definition", out BlittableJsonReaderObject index) == false)
{
ThrowRequiredPropertyNameInRequest("Definition");
}
input.TryGet("Ids", out BlittableJsonReaderArray ids);
var indexDefinition = JsonDeserializationServer.IndexDefinition(index);
if (indexDefinition.Maps == null || indexDefinition.Maps.Count == 0)
{
throw new ArgumentException("Index must have a 'Maps' fields");
}
indexDefinition.Type = indexDefinition.DetectStaticIndexType();
if (indexDefinition.Type.IsJavaScript() == false)
{
throw new UnauthorizedAccessException("Testing indexes is only allowed for JavaScript indexes.");
}
var compiledIndex = new JavaScriptIndex(indexDefinition, Database.Configuration);
var inputSize = GetIntValueQueryString("inputSize", false) ?? defaultInputSizeForTestingJavaScriptIndex;
var collections = new HashSet <string>(compiledIndex.Maps.Keys);
var docsPerCollection = new Dictionary <string, List <DynamicBlittableJson> >();
using (context.OpenReadTransaction())
{
if (ids == null)
{
foreach (var collection in collections)
{
docsPerCollection.Add(collection,
Database.DocumentsStorage.GetDocumentsFrom(context, collection, 0, 0, inputSize).Select(d => new DynamicBlittableJson(d)).ToList());
}
}
else
{
var listOfIds = ids.Select(x => x.ToString());
var _ = new Reference <int>
{
Value = 0
};
var docs = Database.DocumentsStorage.GetDocuments(context, listOfIds, 0, int.MaxValue, _);
foreach (var doc in docs)
{
if (doc.TryGetMetadata(out var metadata) && metadata.TryGet(Constants.Documents.Metadata.Collection, out string collectionStr))
{
if (docsPerCollection.TryGetValue(collectionStr, out var listOfDocs) == false)
{
listOfDocs = docsPerCollection[collectionStr] = new List <DynamicBlittableJson>();
}
listOfDocs.Add(new DynamicBlittableJson(doc));
}
}
}
var mapRes = new List <ObjectInstance>();
//all maps
foreach (var ListOfFunctions in compiledIndex.Maps)
{
//multi maps per collection
foreach (var mapFunc in ListOfFunctions.Value)
{
if (docsPerCollection.TryGetValue(ListOfFunctions.Key, out var docs))
{
foreach (var res in mapFunc(docs))
{
mapRes.Add((ObjectInstance)res);
}
}
}
}
var first = true;
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName("MapResults");
writer.WriteStartArray();
foreach (var mapResult in mapRes)
{
if (JavaScriptIndexUtils.StringifyObject(mapResult) is JsString jsStr)
{
if (first == false)
{
writer.WriteComma();
}
writer.WriteString(jsStr.ToString());
first = false;
}
}
writer.WriteEndArray();
if (indexDefinition.Reduce != null)
{
using (var bufferPool = new UnmanagedBuffersPoolWithLowMemoryHandling("JavaScriptIndexTest", Database.Name))
{
compiledIndex.SetBufferPoolForTestingPurposes(bufferPool);
compiledIndex.SetAllocatorForTestingPurposes(context.Allocator);
first = true;
writer.WritePropertyName("ReduceResults");
writer.WriteStartArray();
var reduceResults = compiledIndex.Reduce(mapRes.Select(mr => new DynamicBlittableJson(JsBlittableBridge.Translate(context, mr.Engine, mr))));
foreach (JsValue reduceResult in reduceResults)
{
if (JavaScriptIndexUtils.StringifyObject(reduceResult) is JsString jsStr)
{
if (first == false)
{
writer.WriteComma();
}
writer.WriteString(jsStr.ToString());
first = false;
}
}
}
writer.WriteEndArray();
}
writer.WriteEndObject();
}
}
}
}
protected override int GetFields <T>(T instance, LazyStringValue key, object document, JsonOperationContext indexContext, IWriteOperationBuffer writeBuffer)
{
if (!(document is ObjectInstance documentToProcess))
{
return(0);
}
int newFields = 0;
if (key != null)
{
instance.Add(GetOrCreateKeyField(key));
newFields++;
}
if (_reduceOutput)
{
var reduceResult = JsBlittableBridge.Translate(indexContext,
documentToProcess.Engine,
documentToProcess);
instance.Add(GetReduceResultValueField(reduceResult, writeBuffer));
newFields++;
}
foreach (var(property, propertyDescriptor) in documentToProcess.GetOwnProperties())
{
if (_fields.TryGetValue(property, out var field) == false)
{
field = _fields[property] = IndexField.Create(property, new IndexFieldOptions(), _allFields);
}
object value;
var actualValue = propertyDescriptor.Value;
if (actualValue.IsObject() && actualValue.IsArray() == false)
{
//In case TryDetectDynamicFieldCreation finds a dynamic field it will populate 'field.Name' with the actual property name
//so we must use field.Name and not property from this point on.
var val = TryDetectDynamicFieldCreation(property, actualValue.AsObject(), field);
if (val != null)
{
if (val.IsObject() && val.AsObject().TryGetValue("$spatial", out _))
{
actualValue = val; //Here we populate the dynamic spatial field that will be handled below.
}
else
{
value = TypeConverter.ToBlittableSupportedType(val, flattenArrays: false, forIndexing: true, engine: documentToProcess.Engine, context: indexContext);
newFields += GetRegularFields(instance, field, value, indexContext, out _);
continue;
}
}
var objectValue = actualValue.AsObject();
if (objectValue.HasOwnProperty("$spatial") && objectValue.TryGetValue("$spatial", out var inner))
{
SpatialField spatialField;
IEnumerable <AbstractField> spatial;
if (inner.IsString())
{
spatialField = StaticIndexBase.GetOrCreateSpatialField(field.Name);
spatial = StaticIndexBase.CreateSpatialField(spatialField, inner.AsString());
}
else if (inner.IsObject())
{
var innerObject = inner.AsObject();
if (innerObject.HasOwnProperty("Lat") && innerObject.HasOwnProperty("Lng") && innerObject.TryGetValue("Lat", out var lat) &&
lat.IsNumber() && innerObject.TryGetValue("Lng", out var lng) && lng.IsNumber())
{
spatialField = StaticIndexBase.GetOrCreateSpatialField(field.Name);
spatial = StaticIndexBase.CreateSpatialField(spatialField, lat.AsNumber(), lng.AsNumber());
}
else
{
continue; //Ignoring bad spatial field
}
}
else
{
continue; //Ignoring bad spatial field
}
newFields += GetRegularFields(instance, field, spatial, indexContext, out _);
continue;
}
}
value = TypeConverter.ToBlittableSupportedType(propertyDescriptor.Value, flattenArrays: false, forIndexing: true, engine: documentToProcess.Engine, context: indexContext);
newFields += GetRegularFields(instance, field, value, indexContext, out _);
if (value is IDisposable toDispose)
{
// the value was converted to a lucene field and isn't needed anymore
toDispose.Dispose();
}
}
return(newFields);
}
protected override int GetFields <T>(T instance, LazyStringValue key, LazyStringValue sourceDocumentId, object document, JsonOperationContext indexContext, IWriteOperationBuffer writeBuffer)
{
if (!(document is ObjectInstance documentToProcess))
{
return(0);
}
int newFields = 0;
if (key != null)
{
instance.Add(GetOrCreateKeyField(key));
newFields++;
}
if (sourceDocumentId != null)
{
instance.Add(GetOrCreateSourceDocumentIdField(sourceDocumentId));
newFields++;
}
if (_storeValue)
{
var storedValue = JsBlittableBridge.Translate(indexContext,
documentToProcess.Engine,
documentToProcess);
instance.Add(GetStoredValueField(storedValue, writeBuffer));
newFields++;
}
if (TryGetBoostedValue(documentToProcess, out var boostedValue, out var documentBoost))
{
if (IsObject(boostedValue) == false)
{
throw new InvalidOperationException($"Invalid boosted value. Expected object but got '{boostedValue.Type}' with value '{boostedValue}'.");
}
documentToProcess = boostedValue.AsObject();
}
foreach (var(property, propertyDescriptor) in documentToProcess.GetOwnProperties())
{
var propertyAsString = property.AsString();
if (_fields.TryGetValue(propertyAsString, out var field) == false)
{
field = _fields[propertyAsString] = IndexField.Create(propertyAsString, new IndexFieldOptions(), _allFields);
}
object value;
float? propertyBoost = null;
int numberOfCreatedFields;
var actualValue = propertyDescriptor.Value;
var isObject = IsObject(actualValue);
if (isObject)
{
if (TryGetBoostedValue(actualValue.AsObject(), out boostedValue, out propertyBoost))
{
actualValue = boostedValue;
isObject = IsObject(actualValue);
}
if (isObject)
{
//In case TryDetectDynamicFieldCreation finds a dynamic field it will populate 'field.Name' with the actual property name
//so we must use field.Name and not property from this point on.
var val = TryDetectDynamicFieldCreation(propertyAsString, actualValue.AsObject(), field);
if (val != null)
{
if (val.IsObject() && val.AsObject().TryGetValue(SpatialPropertyName, out _))
{
actualValue = val; //Here we populate the dynamic spatial field that will be handled below.
}
else
{
value = TypeConverter.ToBlittableSupportedType(val, flattenArrays: false, forIndexing: true, engine: documentToProcess.Engine, context: indexContext);
numberOfCreatedFields = GetRegularFields(instance, field, CreateValueForIndexing(value, propertyBoost), indexContext, out _);
newFields += numberOfCreatedFields;
BoostDocument(instance, numberOfCreatedFields, documentBoost);
if (value is IDisposable toDispose1)
{
// the value was converted to a lucene field and isn't needed anymore
toDispose1.Dispose();
}
continue;
}
}
var objectValue = actualValue.AsObject();
if (objectValue.HasOwnProperty(SpatialPropertyName) && objectValue.TryGetValue(SpatialPropertyName, out var inner))
{
SpatialField spatialField;
IEnumerable <AbstractField> spatial;
if (inner.IsString())
{
spatialField = AbstractStaticIndexBase.GetOrCreateSpatialField(field.Name);
spatial = AbstractStaticIndexBase.CreateSpatialField(spatialField, inner.AsString());
}
else if (inner.IsObject())
{
var innerObject = inner.AsObject();
if (innerObject.HasOwnProperty("Lat") && innerObject.HasOwnProperty("Lng") && innerObject.TryGetValue("Lat", out var lat) &&
lat.IsNumber() && innerObject.TryGetValue("Lng", out var lng) && lng.IsNumber())
{
spatialField = AbstractStaticIndexBase.GetOrCreateSpatialField(field.Name);
spatial = AbstractStaticIndexBase.CreateSpatialField(spatialField, lat.AsNumber(), lng.AsNumber());
}
else
{
continue; //Ignoring bad spatial field
}
}
else
{
continue; //Ignoring bad spatial field
}
numberOfCreatedFields = GetRegularFields(instance, field, CreateValueForIndexing(spatial, propertyBoost), indexContext, out _);
newFields += numberOfCreatedFields;
BoostDocument(instance, numberOfCreatedFields, documentBoost);
continue;
}
}
}
value = TypeConverter.ToBlittableSupportedType(actualValue, flattenArrays: false, forIndexing: true, engine: documentToProcess.Engine, context: indexContext);
numberOfCreatedFields = GetRegularFields(instance, field, CreateValueForIndexing(value, propertyBoost), indexContext, out _);
newFields += numberOfCreatedFields;
BoostDocument(instance, numberOfCreatedFields, documentBoost);
if (value is IDisposable toDispose)
{
// the value was converted to a lucene field and isn't needed anymore
toDispose.Dispose();
}
}
return(newFields);
