public JavaScriptIndex(IndexDefinition definition, RavenConfiguration configuration)
{
_definitions = definition;
// we create the Jint instance directly instead of using SingleRun
// because the index is single threaded and long lived
var resolver = new JintPreventResolvingTasksReferenceResolver();
_engine = new Engine(options =>
{
options.LimitRecursion(64)
.SetReferencesResolver(resolver)
.MaxStatements(configuration.Indexing.MaxStepsForScript)
.Strict()
.AddObjectConverter(new JintGuidConverter())
.AddObjectConverter(new JintStringConverter())
.AddObjectConverter(new JintEnumConverter())
.AddObjectConverter(new JintDateTimeConverter())
.AddObjectConverter(new JintTimeSpanConverter())
.LocalTimeZone(TimeZoneInfo.Utc);
});
InitializeEngine(definition, out var mapList);
var definitions = GetDefinitions();
ProcessMaps(configuration, definitions, resolver, mapList, out var collectionFunctions);
ProcessReduce(definition, definitions, resolver);
ProcessFields(definition, collectionFunctions);
}
public JavaScriptReduceOperation(ScriptFunctionInstance reduce, ScriptFunctionInstance key, Engine engine, JintPreventResolvingTasksReferenceResolver resolver)
{
Reduce = reduce;
Key = key;
Engine = engine;
_resolver = resolver;
GetReduceFieldsNames();
_groupedItems = null;
}
public JavaScriptReduceOperation(ArrowFunctionInstance reduce, ArrowFunctionInstance key, Engine engine, JintPreventResolvingTasksReferenceResolver resolver)
{
Reduce = reduce ?? throw new ArgumentNullException(nameof(reduce));
Key = key ?? throw new ArgumentNullException(nameof(key));
Engine = engine;
_resolver = resolver;
GetReduceFieldsNames();
_groupedItems = null;
}
protected AbstractJavaScriptIndex(IndexDefinition definition, RavenConfiguration configuration, Action <List <string> > modifyMappingFunctions, string mapCode, long indexVersion)
{
Definition = definition;
var indexConfiguration = new SingleIndexConfiguration(definition.Configuration, configuration);
// we create the Jint instance directly instead of using SingleRun
// because the index is single threaded and long lived
var resolver = new JintPreventResolvingTasksReferenceResolver();
_engine = new Engine(options =>
{
options.LimitRecursion(64)
.SetReferencesResolver(resolver)
.MaxStatements(indexConfiguration.MaxStepsForScript)
.Strict(configuration.Patching.StrictMode)
.AddObjectConverter(new JintGuidConverter())
.AddObjectConverter(new JintStringConverter())
.AddObjectConverter(new JintEnumConverter())
.AddObjectConverter(new JintDateTimeConverter())
.AddObjectConverter(new JintTimeSpanConverter())
.LocalTimeZone(TimeZoneInfo.Utc);
});
using (_engine.DisableMaxStatements())
{
var maps = GetMappingFunctions(modifyMappingFunctions);
var mapReferencedCollections = InitializeEngine(definition, maps, mapCode);
var definitions = GetDefinitions();
ProcessMaps(definitions, resolver, maps, mapReferencedCollections, out var collectionFunctions);
ProcessReduce(definition, definitions, resolver, indexVersion);
ProcessFields(definition, collectionFunctions);
}
_javaScriptUtils = new JavaScriptUtils(null, _engine);
}
public JavaScriptMapOperation(Engine engine, JintPreventResolvingTasksReferenceResolver resolver)
{
_engine = engine;
_resolver = resolver;
}
private void ProcessMaps(RavenConfiguration configuration, ObjectInstance definitions, JintPreventResolvingTasksReferenceResolver resolver, List <string> mapList,
List <HashSet <CollectionName> > mapReferencedCollections, out Dictionary <string, List <JavaScriptMapOperation> > collectionFunctions)
{
var mapsArray = definitions.GetProperty(MapsProperty).Value;
if (mapsArray.IsNull() || mapsArray.IsUndefined() || mapsArray.IsArray() == false)
{
ThrowIndexCreationException($"doesn't contain any map function or '{GlobalDefinitions}.{Maps}' was modified in the script");
}
var maps = mapsArray.AsArray();
collectionFunctions = new Dictionary <string, List <JavaScriptMapOperation> >();
for (int i = 0; i < maps.GetLength(); i++)
{
var mapObj = maps.Get(i.ToString());
if (mapObj.IsNull() || mapObj.IsUndefined() || mapObj.IsObject() == false)
{
ThrowIndexCreationException($"map function #{i} is not a valid object");
}
var map = mapObj.AsObject();
if (map.HasProperty(CollectionProperty) == false)
{
ThrowIndexCreationException($"map function #{i} is missing a collection name");
}
var mapCollectionStr = map.Get(CollectionProperty);
if (mapCollectionStr.IsString() == false)
{
ThrowIndexCreationException($"map function #{i} collection name isn't a string");
}
var mapCollection = mapCollectionStr.AsString();
if (collectionFunctions.TryGetValue(mapCollection, out var list) == false)
{
list = new List <JavaScriptMapOperation>();
collectionFunctions.Add(mapCollection, list);
}
if (map.HasProperty(MethodProperty) == false)
{
ThrowIndexCreationException($"map function #{i} is missing its {MethodProperty} property");
}
var funcInstance = map.Get(MethodProperty).As <FunctionInstance>();
if (funcInstance == null)
{
ThrowIndexCreationException($"map function #{i} {MethodProperty} property isn't a 'FunctionInstance'");
}
var operation = new JavaScriptMapOperation(_engine, resolver)
{
MapFunc = funcInstance,
IndexName = _definitions.Name,
Configuration = configuration,
MapString = mapList[i]
};
if (map.HasOwnProperty(MoreArgsProperty))
{
var moreArgsObj = map.Get(MoreArgsProperty);
if (moreArgsObj.IsArray())
{
var array = moreArgsObj.AsArray();
if (array.GetLength() > 0)
{
operation.MoreArguments = array;
}
}
}
operation.Analyze(_engine);
if (ReferencedCollections.TryGetValue(mapCollection, out var collectionNames) == false)
{
collectionNames = new HashSet <CollectionName>();
ReferencedCollections.Add(mapCollection, collectionNames);
}
collectionNames.UnionWith(mapReferencedCollections[i]);
list.Add(operation);
}
}
private void ProcessReduce(IndexDefinition definition, ObjectInstance definitions, JintPreventResolvingTasksReferenceResolver resolver)
{
var reduceObj = definitions.GetProperty(ReduceProperty)?.Value;
if (reduceObj != null && reduceObj.IsObject())
{
var reduceAsObj = reduceObj.AsObject();
var groupByKey = reduceAsObj.GetProperty(KeyProperty).Value.As <ArrowFunctionInstance>();
var reduce = reduceAsObj.GetProperty(AggregateByProperty).Value.As <ArrowFunctionInstance>();
ReduceOperation = new JavaScriptReduceOperation(reduce, groupByKey, _engine, resolver)
{
ReduceString = definition.Reduce
};
GroupByFields = ReduceOperation.GetReduceFieldsNames();
Reduce = ReduceOperation.IndexingFunction;
}
}
protected abstract void ProcessMaps(ObjectInstance definitions, JintPreventResolvingTasksReferenceResolver resolver, List <string> mapList, List <MapMetadata> mapReferencedCollections, out Dictionary <string, Dictionary <string, List <JavaScriptMapOperation> > > collectionFunctions);
public JavaScriptIndex(IndexDefinition definition, RavenConfiguration configuration)
{
_definitions = definition;
_resolver = new JintPreventResolvingTasksReferenceResolver();
// we create the Jint instance directly instead of using SingleRun
// because the index is single threaded and long lived
_engine = new Engine(options =>
{
options.LimitRecursion(64)
.SetReferencesResolver(_resolver)
.MaxStatements(configuration.Indexing.MaxStepsForScript)
.Strict()
.AddObjectConverter(new JintGuidConverter())
.AddObjectConverter(new JintStringConverter())
.AddObjectConverter(new JintEnumConverter())
.AddObjectConverter(new JintDateTimeConverter())
.AddObjectConverter(new JintTimeSpanConverter())
.LocalTimeZone(TimeZoneInfo.Utc);
});
_engine.SetValue("load", new ClrFunctionInstance(_engine, LoadDocument));
_engine.Execute(Code);
if (definition.AdditionalSources != null)
{
foreach (var script in definition.AdditionalSources.Values)
{
_engine.Execute(script);
}
}
var mapList = definition.Maps.ToList();
for (var i = 0; i < mapList.Count; i++)
{
_engine.Execute(mapList[i]);
}
if (definition.Reduce != null)
{
_engine.Execute(definition.Reduce);
}
var definitionsObj = _engine.GetValue(GlobalDefinitions);
if (definitionsObj.IsNull() || definitionsObj.IsUndefined() || definitionsObj.IsObject() == false)
{
ThrowIndexCreationException($"is missing its '{GlobalDefinitions}' global variable, are you modifying it in your script?");
}
var definitions = definitionsObj.AsObject();
if (definitions.HasProperty(MapsProperty) == false)
{
ThrowIndexCreationException("is missing its 'globalDefinition.maps' property, are you modifying it in your script?");
}
var mapsArray = definitions.GetProperty(MapsProperty).Value;
if (mapsArray.IsNull() || mapsArray.IsUndefined() || mapsArray.IsArray() == false)
{
ThrowIndexCreationException($"doesn't contain any map function or '{GlobalDefinitions}.{Maps}' was modified in the script");
}
var maps = mapsArray.AsArray();
var collectionFunctions = new Dictionary <string, List <JavaScriptMapOperation> >();
for (int i = 0; i < maps.GetLength(); i++)
{
var mapObj = maps.Get(i.ToString());
if (mapObj.IsNull() || mapObj.IsUndefined() || mapObj.IsObject() == false)
{
ThrowIndexCreationException($"map function #{i} is not a valid object");
}
var map = mapObj.AsObject();
if (map.HasProperty(CollectionProperty) == false)
{
ThrowIndexCreationException($"map function #{i} is missing a collection name");
}
var mapCollectionStr = map.Get(CollectionProperty);
if (mapCollectionStr.IsString() == false)
{
ThrowIndexCreationException($"map function #{i} collection name isn't a string");
}
var mapCollection = mapCollectionStr.AsString();
if (collectionFunctions.TryGetValue(mapCollection, out var list) == false)
{
list = new List <JavaScriptMapOperation>();
collectionFunctions.Add(mapCollection, list);
}
if (map.HasProperty(MethodProperty) == false)
{
ThrowIndexCreationException($"map function #{i} is missing its {MethodProperty} property");
}
var funcInstance = map.Get(MethodProperty).As <FunctionInstance>();
if (funcInstance == null)
{
ThrowIndexCreationException($"map function #{i} {MethodProperty} property isn't a 'FunctionInstance'");
}
var operation = new JavaScriptMapOperation(_engine, _resolver)
{
MapFunc = funcInstance,
IndexName = _definitions.Name,
Configuration = configuration,
MapString = mapList[i]
};
if (map.HasOwnProperty(MoreArgsProperty))
{
var moreArgsObj = map.Get(MoreArgsProperty);
if (moreArgsObj.IsArray())
{
var array = moreArgsObj.AsArray();
if (array.GetLength() > 0)
{
operation.MoreArguments = array;
}
}
}
operation.Analyze(_engine);
if (ReferencedCollections.TryGetValue(mapCollection, out var collectionNames) == false)
{
collectionNames = new HashSet <CollectionName>();
ReferencedCollections.Add(mapCollection, collectionNames);
}
collectionNames.UnionWith(operation.ReferencedCollection);
list.Add(operation);
}
var reduceObj = definitions.GetProperty(ReduceProperty)?.Value;
if (reduceObj != null && reduceObj.IsObject())
{
var reduceAsObj = reduceObj.AsObject();
var groupByKey = reduceAsObj.GetProperty(KeyProperty).Value.As <ScriptFunctionInstance>();
var reduce = reduceAsObj.GetProperty(AggregateByProperty).Value.As <ScriptFunctionInstance>();
ReduceOperation = new JavaScriptReduceOperation(reduce, groupByKey, _engine, _resolver)
{
ReduceString = definition.Reduce
};
GroupByFields = ReduceOperation.GetReduceFieldsNames();
Reduce = ReduceOperation.IndexingFunction;
}
var fields = new HashSet <string>();
HasDynamicFields = false;
foreach (var(key, val) in collectionFunctions)
{
Maps.Add(key, val.Select(x => (IndexingFunc)x.IndexingFunction).ToList());
//TODO: Validation of matches fields between group by / collections / etc
foreach (var operation in val)
{
HasDynamicFields |= operation.HasDynamicReturns;
fields.UnionWith(operation.Fields);
foreach ((var k, var v) in operation.FieldOptions)
{
_definitions.Fields.Add(k, v);
}
}
}
if (definition.Fields != null)
{
foreach (var item in definition.Fields)
{
fields.Add(item.Key);
}
}
OutputFields = fields.ToArray();
}
