private static object ToBlittableSupportedType(object root, object value, bool flattenArrays, int recursiveLevel, Engine engine, JsonOperationContext context) { if (recursiveLevel > MaxAllowedRecursiveLevelForType) { NestingLevelTooDeep(root); } if (value is JsValue js) { if (js.IsNull() || js.IsUndefined()) { return(null); } if (js.IsString()) { return(js.AsString()); } if (js.IsBoolean()) { return(js.AsBoolean()); } if (js.IsNumber()) { return(js.AsNumber()); } if (js.IsDate()) { return(js.AsDate().ToDateTime()); } //object wrapper is an object so it must come before the object if (js is ObjectWrapper ow) { var target = ow.Target; switch (target) { case LazyStringValue lsv: return(lsv); case LazyCompressedStringValue lcsv: return(lcsv); case LazyNumberValue lnv: return(lnv); //should be already blittable supported type. } ThrowInvalidObject(js); } //Array is an object in Jint else if (js.IsArray()) { var arr = js.AsArray(); var convertedArray = EnumerateArray(root, arr, flattenArrays, recursiveLevel + 1, engine, context); return(new DynamicJsonArray(flattenArrays ? Flatten(convertedArray) : convertedArray)); } else if (js.IsObject()) { return(JsBlittableBridge.Translate(context, engine, js.AsObject())); } ThrowInvalidObject(js); return(null); } if (value == null || value is DynamicNullObject) { return(null); } if (value is DynamicBlittableJson dynamicDocument) { return(dynamicDocument.BlittableJson); } if (value is string) { return(value); } if (value is LazyStringValue || value is LazyCompressedStringValue) { return(value); } if (value is bool) { return(value); } if (value is int || value is long || value is double || value is decimal || value is float || value is short || value is byte) { return(value); } if (value is LazyNumberValue) { return(value); } if (value is DateTime || value is DateTimeOffset || value is TimeSpan) { return(value); } if (value is Guid guid) { return(guid.ToString("D")); } if (value is Enum) { return(value.ToString()); } if (value is IEnumerable <IFieldable> || value is IFieldable) { return("__ignored"); } if (value is IDictionary dictionary) { var @object = new DynamicJsonValue(); foreach (var key in dictionary.Keys) { var keyAsString = KeyAsString(key: ToBlittableSupportedType(root, key, flattenArrays, recursiveLevel: recursiveLevel + 1, engine: engine, context: context)); @object[keyAsString] = ToBlittableSupportedType(root, dictionary[key], flattenArrays, recursiveLevel: recursiveLevel + 1, engine: engine, context: context); } return(@object); } if (value is IDictionary <object, object> dDictionary) { var @object = new DynamicJsonValue(); foreach (var key in dDictionary.Keys) { var keyAsString = KeyAsString(key: ToBlittableSupportedType(root, key, flattenArrays, recursiveLevel: recursiveLevel + 1, engine: engine, context: context)); @object[keyAsString] = ToBlittableSupportedType(root, dDictionary[key], flattenArrays, recursiveLevel: recursiveLevel + 1, engine: engine, context: context); } return(@object); } if (value is char[] chars) { return(new string(chars)); } if (value is IEnumerable <char> charEnumerable) { return(new string(charEnumerable.ToArray())); } if (value is byte[] bytes) { return(System.Convert.ToBase64String(bytes)); } if (value is IEnumerable enumerable) { if (ShouldTreatAsEnumerable(enumerable)) { return(EnumerableToJsonArray(flattenArrays ? Flatten(enumerable) : enumerable, root, flattenArrays, recursiveLevel, engine, context)); } } var inner = new DynamicJsonValue(); var accessor = GetPropertyAccessor(value); foreach (var property in accessor.GetPropertiesInOrder(value)) { var propertyValue = property.Value; if (propertyValue is IEnumerable <object> propertyValueAsEnumerable && ShouldTreatAsEnumerable(propertyValue)) { inner[property.Key] = EnumerableToJsonArray(propertyValueAsEnumerable, root, flattenArrays, recursiveLevel, engine, context); continue; } inner[property.Key] = ToBlittableSupportedType(root, propertyValue, flattenArrays, recursiveLevel + 1, engine, context); } return(inner); }
public async Task TestJavaScriptIndex() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var input = await context.ReadForMemoryAsync(RequestBodyStream(), "TestJavaScriptIndex"); if (input.TryGet("Definition", out BlittableJsonReaderObject index) == false) { ThrowRequiredPropertyNameInRequest("Definition"); } input.TryGet("Ids", out BlittableJsonReaderArray ids); var indexDefinition = JsonDeserializationServer.IndexDefinition(index); if (indexDefinition.Maps == null || indexDefinition.Maps.Count == 0) { throw new ArgumentException("Index must have a 'Maps' fields"); } indexDefinition.Type = indexDefinition.DetectStaticIndexType(); if (indexDefinition.Type.IsJavaScript() == false) { throw new UnauthorizedAccessException("Testing indexes is only allowed for JavaScript indexes."); } var compiledIndex = new JavaScriptIndex(indexDefinition, Database.Configuration); var inputSize = GetIntValueQueryString("inputSize", false) ?? defaultInputSizeForTestingJavaScriptIndex; var collections = new HashSet <string>(compiledIndex.Maps.Keys); var docsPerCollection = new Dictionary <string, List <DynamicBlittableJson> >(); using (context.OpenReadTransaction()) { if (ids == null) { foreach (var collection in collections) { docsPerCollection.Add(collection, Database.DocumentsStorage.GetDocumentsFrom(context, collection, 0, 0, inputSize).Select(d => new DynamicBlittableJson(d)).ToList()); } } else { var listOfIds = ids.Select(x => x.ToString()); var _ = new Reference <int> { Value = 0 }; var docs = Database.DocumentsStorage.GetDocuments(context, listOfIds, 0, int.MaxValue, _); foreach (var doc in docs) { if (doc.TryGetMetadata(out var metadata) && metadata.TryGet(Constants.Documents.Metadata.Collection, out string collectionStr)) { if (docsPerCollection.TryGetValue(collectionStr, out var listOfDocs) == false) { listOfDocs = docsPerCollection[collectionStr] = new List <DynamicBlittableJson>(); } listOfDocs.Add(new DynamicBlittableJson(doc)); } } } var mapRes = new List <ObjectInstance>(); //all maps foreach (var ListOfFunctions in compiledIndex.Maps) { //multi maps per collection foreach (var mapFunc in ListOfFunctions.Value) { if (docsPerCollection.TryGetValue(ListOfFunctions.Key, out var docs)) { foreach (var res in mapFunc(docs)) { mapRes.Add((ObjectInstance)res); } } } } var first = true; using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName("MapResults"); writer.WriteStartArray(); foreach (var mapResult in mapRes) { if (JavaScriptIndexUtils.StringifyObject(mapResult) is JsString jsStr) { if (first == false) { writer.WriteComma(); } writer.WriteString(jsStr.ToString()); first = false; } } writer.WriteEndArray(); if (indexDefinition.Reduce != null) { using (var bufferPool = new UnmanagedBuffersPoolWithLowMemoryHandling("JavaScriptIndexTest", Database.Name)) { compiledIndex.SetBufferPoolForTestingPurposes(bufferPool); compiledIndex.SetAllocatorForTestingPurposes(context.Allocator); first = true; writer.WritePropertyName("ReduceResults"); writer.WriteStartArray(); var reduceResults = compiledIndex.Reduce(mapRes.Select(mr => new DynamicBlittableJson(JsBlittableBridge.Translate(context, mr.Engine, mr)))); foreach (JsValue reduceResult in reduceResults) { if (JavaScriptIndexUtils.StringifyObject(reduceResult) is JsString jsStr) { if (first == false) { writer.WriteComma(); } writer.WriteString(jsStr.ToString()); first = false; } } } writer.WriteEndArray(); } writer.WriteEndObject(); } } } }
protected override int GetFields <T>(T instance, LazyStringValue key, object document, JsonOperationContext indexContext, IWriteOperationBuffer writeBuffer) { if (!(document is ObjectInstance documentToProcess)) { return(0); } int newFields = 0; if (key != null) { instance.Add(GetOrCreateKeyField(key)); newFields++; } if (_reduceOutput) { var reduceResult = JsBlittableBridge.Translate(indexContext, documentToProcess.Engine, documentToProcess); instance.Add(GetReduceResultValueField(reduceResult, writeBuffer)); newFields++; } foreach (var(property, propertyDescriptor) in documentToProcess.GetOwnProperties()) { if (_fields.TryGetValue(property, out var field) == false) { field = _fields[property] = IndexField.Create(property, new IndexFieldOptions(), _allFields); } object value; var actualValue = propertyDescriptor.Value; if (actualValue.IsObject() && actualValue.IsArray() == false) { //In case TryDetectDynamicFieldCreation finds a dynamic field it will populate 'field.Name' with the actual property name //so we must use field.Name and not property from this point on. var val = TryDetectDynamicFieldCreation(property, actualValue.AsObject(), field); if (val != null) { if (val.IsObject() && val.AsObject().TryGetValue("$spatial", out _)) { actualValue = val; //Here we populate the dynamic spatial field that will be handled below. } else { value = TypeConverter.ToBlittableSupportedType(val, flattenArrays: false, forIndexing: true, engine: documentToProcess.Engine, context: indexContext); newFields += GetRegularFields(instance, field, value, indexContext, out _); continue; } } var objectValue = actualValue.AsObject(); if (objectValue.HasOwnProperty("$spatial") && objectValue.TryGetValue("$spatial", out var inner)) { SpatialField spatialField; IEnumerable <AbstractField> spatial; if (inner.IsString()) { spatialField = StaticIndexBase.GetOrCreateSpatialField(field.Name); spatial = StaticIndexBase.CreateSpatialField(spatialField, inner.AsString()); } else if (inner.IsObject()) { var innerObject = inner.AsObject(); if (innerObject.HasOwnProperty("Lat") && innerObject.HasOwnProperty("Lng") && innerObject.TryGetValue("Lat", out var lat) && lat.IsNumber() && innerObject.TryGetValue("Lng", out var lng) && lng.IsNumber()) { spatialField = StaticIndexBase.GetOrCreateSpatialField(field.Name); spatial = StaticIndexBase.CreateSpatialField(spatialField, lat.AsNumber(), lng.AsNumber()); } else { continue; //Ignoring bad spatial field } } else { continue; //Ignoring bad spatial field } newFields += GetRegularFields(instance, field, spatial, indexContext, out _); continue; } } value = TypeConverter.ToBlittableSupportedType(propertyDescriptor.Value, flattenArrays: false, forIndexing: true, engine: documentToProcess.Engine, context: indexContext); newFields += GetRegularFields(instance, field, value, indexContext, out _); if (value is IDisposable toDispose) { // the value was converted to a lucene field and isn't needed anymore toDispose.Dispose(); } } return(newFields); }
protected override int GetFields <T>(T instance, LazyStringValue key, LazyStringValue sourceDocumentId, object document, JsonOperationContext indexContext, IWriteOperationBuffer writeBuffer) { if (!(document is ObjectInstance documentToProcess)) { return(0); } int newFields = 0; if (key != null) { instance.Add(GetOrCreateKeyField(key)); newFields++; } if (sourceDocumentId != null) { instance.Add(GetOrCreateSourceDocumentIdField(sourceDocumentId)); newFields++; } if (_storeValue) { var storedValue = JsBlittableBridge.Translate(indexContext, documentToProcess.Engine, documentToProcess); instance.Add(GetStoredValueField(storedValue, writeBuffer)); newFields++; } if (TryGetBoostedValue(documentToProcess, out var boostedValue, out var documentBoost)) { if (IsObject(boostedValue) == false) { throw new InvalidOperationException($"Invalid boosted value. Expected object but got '{boostedValue.Type}' with value '{boostedValue}'."); } documentToProcess = boostedValue.AsObject(); } foreach (var(property, propertyDescriptor) in documentToProcess.GetOwnProperties()) { var propertyAsString = property.AsString(); if (_fields.TryGetValue(propertyAsString, out var field) == false) { field = _fields[propertyAsString] = IndexField.Create(propertyAsString, new IndexFieldOptions(), _allFields); } object value; float? propertyBoost = null; int numberOfCreatedFields; var actualValue = propertyDescriptor.Value; var isObject = IsObject(actualValue); if (isObject) { if (TryGetBoostedValue(actualValue.AsObject(), out boostedValue, out propertyBoost)) { actualValue = boostedValue; isObject = IsObject(actualValue); } if (isObject) { //In case TryDetectDynamicFieldCreation finds a dynamic field it will populate 'field.Name' with the actual property name //so we must use field.Name and not property from this point on. var val = TryDetectDynamicFieldCreation(propertyAsString, actualValue.AsObject(), field); if (val != null) { if (val.IsObject() && val.AsObject().TryGetValue(SpatialPropertyName, out _)) { actualValue = val; //Here we populate the dynamic spatial field that will be handled below. } else { value = TypeConverter.ToBlittableSupportedType(val, flattenArrays: false, forIndexing: true, engine: documentToProcess.Engine, context: indexContext); numberOfCreatedFields = GetRegularFields(instance, field, CreateValueForIndexing(value, propertyBoost), indexContext, out _); newFields += numberOfCreatedFields; BoostDocument(instance, numberOfCreatedFields, documentBoost); if (value is IDisposable toDispose1) { // the value was converted to a lucene field and isn't needed anymore toDispose1.Dispose(); } continue; } } var objectValue = actualValue.AsObject(); if (objectValue.HasOwnProperty(SpatialPropertyName) && objectValue.TryGetValue(SpatialPropertyName, out var inner)) { SpatialField spatialField; IEnumerable <AbstractField> spatial; if (inner.IsString()) { spatialField = AbstractStaticIndexBase.GetOrCreateSpatialField(field.Name); spatial = AbstractStaticIndexBase.CreateSpatialField(spatialField, inner.AsString()); } else if (inner.IsObject()) { var innerObject = inner.AsObject(); if (innerObject.HasOwnProperty("Lat") && innerObject.HasOwnProperty("Lng") && innerObject.TryGetValue("Lat", out var lat) && lat.IsNumber() && innerObject.TryGetValue("Lng", out var lng) && lng.IsNumber()) { spatialField = AbstractStaticIndexBase.GetOrCreateSpatialField(field.Name); spatial = AbstractStaticIndexBase.CreateSpatialField(spatialField, lat.AsNumber(), lng.AsNumber()); } else { continue; //Ignoring bad spatial field } } else { continue; //Ignoring bad spatial field } numberOfCreatedFields = GetRegularFields(instance, field, CreateValueForIndexing(spatial, propertyBoost), indexContext, out _); newFields += numberOfCreatedFields; BoostDocument(instance, numberOfCreatedFields, documentBoost); continue; } } } value = TypeConverter.ToBlittableSupportedType(actualValue, flattenArrays: false, forIndexing: true, engine: documentToProcess.Engine, context: indexContext); numberOfCreatedFields = GetRegularFields(instance, field, CreateValueForIndexing(value, propertyBoost), indexContext, out _); newFields += numberOfCreatedFields; BoostDocument(instance, numberOfCreatedFields, documentBoost); if (value is IDisposable toDispose) { // the value was converted to a lucene field and isn't needed anymore toDispose.Dispose(); } } return(newFields);