private static FilterSpecPlanPathForge ComputePermutation(
FilterSpecParaForgeMap filterParamExprMap,
object[] permutation,
FilterSpecParaForgeMap[][] orNodesMaps,
FilterSpecCompilerArgs args)
{
var mapAll = new FilterSpecParaForgeMap();
mapAll.Add(filterParamExprMap);
// combine
for (var orNodeNum = 0; orNodeNum < permutation.Length; orNodeNum++) {
var orChildNodeNum = permutation[orNodeNum].AsInt32();
var mapOrSub = orNodesMaps[orNodeNum][orChildNodeNum];
mapAll.Add(mapOrSub);
}
// consolidate across
FilterSpecCompilerConsolidateUtil.Consolidate(mapAll, args.statementRawInfo.StatementName);
IList<FilterSpecPlanPathTripletForge> filterParams = new List<FilterSpecPlanPathTripletForge>(mapAll.Triplets);
var countUnassigned = mapAll.CountUnassignedExpressions();
if (countUnassigned != 0) {
FilterSpecPlanPathTripletForge node = MakeRemainingNode(mapAll.UnassignedExpressions, args);
filterParams.Add(node);
}
FilterSpecPlanPathTripletForge[] triplets = filterParams.ToArray();
return new FilterSpecPlanPathForge(triplets, null);
}
private static IList<FilterSpecParamForge> ComputePermutation(
FilterSpecParaForgeMap filterParamExprMap,
object[] permutation,
FilterSpecParaForgeMap[][] orNodesMaps,
FilterSpecCompilerArgs args)
{
FilterSpecParaForgeMap mapAll = new FilterSpecParaForgeMap();
mapAll.Add(filterParamExprMap);
// combine
for (int orNodeNum = 0; orNodeNum < permutation.Length; orNodeNum++) {
int orChildNodeNum = (int) permutation[orNodeNum];
FilterSpecParaForgeMap mapOrSub = orNodesMaps[orNodeNum][orChildNodeNum];
mapAll.Add(mapOrSub);
}
// consolidate across
FilterSpecCompilerConsolidateUtil.Consolidate(mapAll, args.statementRawInfo.StatementName);
IList<FilterSpecParamForge> filterParams = new List<FilterSpecParamForge>();
filterParams.AddAll(mapAll.FilterParams);
int countUnassigned = mapAll.CountUnassignedExpressions();
if (countUnassigned == 0) {
return filterParams;
}
FilterSpecParamForge node = MakeRemainingNode(mapAll.UnassignedExpressions, args);
filterParams.Add(node);
return filterParams;
}
public OrChildNodeNVNegated(
ExprNode node,
FilterSpecParaForgeMap map,
ExprNode control) : base(node, map)
{
this.Control = control;
}
public OrChildNodeNV(
ExprNode node,
FilterSpecParaForgeMap map)
{
this.Node = node;
this.Map = map;
}
private static FilterSpecPlanForge PlanFilterParametersInternal(
IList<ExprNode> validatedNodes,
FilterSpecCompilerArgs args)
{
if (validatedNodes.IsEmpty()) {
return EMPTY;
}
if (args.compileTimeServices.Configuration.Compiler.Execution.FilterIndexPlanning == ConfigurationCompilerExecution.FilterIndexPlanningEnum.NONE) {
DecomposeCheckAggregation(validatedNodes);
return BuildNoPlan(validatedNodes, args);
}
var performConditionPlanning = HasLevelOrHint(FilterSpecCompilerIndexPlannerHint.CONDITIONS, args.statementRawInfo, args.compileTimeServices);
var filterParamExprMap = new FilterSpecParaForgeMap();
// Make filter parameter for each expression node, if it can be optimized.
// Optionally receive a top-level control condition that negates
var topLevelNegation = DecomposePopulateConsolidate(filterParamExprMap, performConditionPlanning, validatedNodes, args);
// Use all filter parameter and unassigned expressions
var countUnassigned = filterParamExprMap.CountUnassignedExpressions();
// we are done if there are no remaining nodes
if (countUnassigned == 0) {
return MakePlanFromTriplets(filterParamExprMap.Triplets, topLevelNegation, args);
}
// determine max-width
var filterServiceMaxFilterWidth = args.compileTimeServices.Configuration.Compiler.Execution.FilterServiceMaxFilterWidth;
var hint = HintEnum.MAX_FILTER_WIDTH.GetHint(args.statementRawInfo.Annotations);
if (hint != null) {
var hintValue = HintEnum.MAX_FILTER_WIDTH.GetHintAssignedValue(hint);
filterServiceMaxFilterWidth = int.Parse(hintValue);
}
FilterSpecPlanForge plan = null;
if (filterServiceMaxFilterWidth > 0) {
if (performConditionPlanning) {
plan = PlanRemainingNodesWithConditions(filterParamExprMap, args, filterServiceMaxFilterWidth, topLevelNegation);
}
else {
plan = PlanRemainingNodesBasic(filterParamExprMap, args, filterServiceMaxFilterWidth);
}
}
if (plan != null) {
return plan;
}
// handle no-plan
var triplets = new List<FilterSpecPlanPathTripletForge>(filterParamExprMap.Triplets);
var unassignedExpressions = filterParamExprMap.UnassignedExpressions;
var triplet = MakeRemainingNode(unassignedExpressions, args);
triplets.Add(triplet);
return MakePlanFromTriplets(triplets, topLevelNegation, args);
}
private static IList<FilterSpecParamForge>[] PlanRemainingNodesIfFeasible(
FilterSpecParaForgeMap overallExpressions,
FilterSpecCompilerArgs args,
int filterServiceMaxFilterWidth)
{
IList<ExprNode> unassigned = overallExpressions.UnassignedExpressions;
IList<ExprOrNode> orNodes = new List<ExprOrNode>(unassigned.Count);
foreach (ExprNode node in unassigned) {
if (node is ExprOrNode) {
orNodes.Add((ExprOrNode) node);
}
}
FilterSpecParaForgeMap expressionsWithoutOr = new FilterSpecParaForgeMap();
expressionsWithoutOr.Add(overallExpressions);
// first dimension: or-node index
// second dimension: or child node index
FilterSpecParaForgeMap[][] orNodesMaps = new FilterSpecParaForgeMap[orNodes.Count][];
int countOr = 0;
int sizeFactorized = 1;
int[] sizePerOr = new int[orNodes.Count];
foreach (ExprOrNode orNode in orNodes) {
expressionsWithoutOr.RemoveNode(orNode);
orNodesMaps[countOr] = new FilterSpecParaForgeMap[orNode.ChildNodes.Length];
int len = orNode.ChildNodes.Length;
for (int i = 0; i < len; i++) {
FilterSpecParaForgeMap map = new FilterSpecParaForgeMap();
orNodesMaps[countOr][i] = map;
IList<ExprNode> nodes = Collections.SingletonList(orNode.ChildNodes[i]);
DecomposePopulateConsolidate(map, nodes, args);
}
sizePerOr[countOr] = len;
sizeFactorized = sizeFactorized * len;
countOr++;
}
// we become too large
if (sizeFactorized > filterServiceMaxFilterWidth) {
return null;
}
// combine
IList<FilterSpecParamForge>[] result = new IList<FilterSpecParamForge>[sizeFactorized];
IEnumerable<object[]> permutations = CombinationEnumeration.FromZeroBasedRanges(sizePerOr);
int count = 0;
foreach (var permutation in permutations) {
result[count] = ComputePermutation(expressionsWithoutOr, permutation, orNodesMaps, args);
count++;
}
return result;
}
internal static FilterSpecPlanForge PlanRemainingNodesBasic(
FilterSpecParaForgeMap overallExpressions,
FilterSpecCompilerArgs args,
int filterServiceMaxFilterWidth)
{
var unassigned = overallExpressions.UnassignedExpressions;
IList<ExprOrNode> orNodes = new List<ExprOrNode>(unassigned.Count);
foreach (var node in unassigned) {
if (node is ExprOrNode) {
orNodes.Add((ExprOrNode) node);
}
}
var expressionsWithoutOr = new FilterSpecParaForgeMap();
expressionsWithoutOr.Add(overallExpressions);
// first dimension: or-node index
// second dimension: or child node index
var orNodesMaps = new FilterSpecParaForgeMap[orNodes.Count][];
var countOr = 0;
var sizeFactorized = 1;
var sizePerOr = new int[orNodes.Count];
foreach (var orNode in orNodes) {
expressionsWithoutOr.RemoveNode(orNode);
orNodesMaps[countOr] = new FilterSpecParaForgeMap[orNode.ChildNodes.Length];
var len = orNode.ChildNodes.Length;
for (var i = 0; i < len; i++) {
var map = new FilterSpecParaForgeMap();
orNodesMaps[countOr][i] = map;
var nodes = Collections.SingletonList(orNode.ChildNodes[i]);
DecomposePopulateConsolidate(map, false, nodes, args);
}
sizePerOr[countOr] = len;
sizeFactorized = sizeFactorized * len;
countOr++;
}
// we become too large
if (sizeFactorized > filterServiceMaxFilterWidth) {
return null;
}
// combine
var result = new FilterSpecPlanPathForge[sizeFactorized];
var count = 0;
foreach (var permutation in CombinationEnumeration.FromZeroBasedRanges(sizePerOr)) {
result[count] = ComputePermutation(expressionsWithoutOr, permutation, orNodesMaps, args);
count++;
}
return new FilterSpecPlanForge(result, null, null, null);
}
internal static ExprNode DecomposePopulateConsolidate(
FilterSpecParaForgeMap filterParamExprMap,
bool performConditionPlanning,
IList<ExprNode> validatedNodes,
FilterSpecCompilerArgs args)
{
var constituents = DecomposeCheckAggregation(validatedNodes);
// Remove constituents that are value-expressions
ExprNode topLevelControl = null;
if (performConditionPlanning) {
IList<ExprNode> valueOnlyConstituents = null;
foreach (var node in constituents) {
var visitor = new FilterSpecExprNodeVisitorValueLimitedExpr();
node.Accept(visitor);
if (visitor.IsLimited) {
if (valueOnlyConstituents == null) {
valueOnlyConstituents = new List<ExprNode>();
}
valueOnlyConstituents.Add(node);
}
}
if (valueOnlyConstituents != null) {
constituents.RemoveAll(valueOnlyConstituents);
topLevelControl = ExprNodeUtilityMake.ConnectExpressionsByLogicalAndWhenNeeded(valueOnlyConstituents);
}
}
// Make filter parameter for each expression node, if it can be optimized
foreach (var constituent in constituents) {
var triplet = FilterSpecCompilerIndexPlannerConstituent.MakeFilterParam(
constituent,
performConditionPlanning,
args.taggedEventTypes,
args.arrayEventTypes,
args.allTagNamesOrdered,
args.statementRawInfo.StatementName,
args.streamTypeService,
args.statementRawInfo,
args.compileTimeServices);
filterParamExprMap.Put(constituent, triplet); // accepts null values as the expression may not be optimized
}
// Consolidate entries as possible, i.e. (a != 5 and a != 6) is (a not in (5,6))
// Removes duplicates for same property and same filter operator for filter service index optimizations
FilterSpecCompilerConsolidateUtil.Consolidate(filterParamExprMap, args.statementRawInfo.StatementName);
return topLevelControl;
}
// consolidate "val != 3 and val != 4 and val != 5"
// to "val not in (3, 4, 5)"
private static void HandleConsolidateNotEqual(
IList<FilterSpecPlanPathTripletForge> parameters,
FilterSpecParaForgeMap filterParamExprMap,
string statementName)
{
IList<FilterSpecParamInValueForge> values = new List<FilterSpecParamInValueForge>();
ExprNode lastNotEqualsExprNode = null;
foreach (FilterSpecPlanPathTripletForge triplet in parameters) {
FilterSpecParamForge paramForge = triplet.Param;
if (paramForge is FilterSpecParamConstantForge) {
var constantParam = (FilterSpecParamConstantForge) paramForge;
var constant = constantParam.FilterConstant;
values.Add(new FilterForEvalConstantAnyTypeForge(constant));
}
else if (paramForge is FilterSpecParamEventPropForge) {
var eventProp = (FilterSpecParamEventPropForge) paramForge;
values.Add(
new FilterForEvalEventPropForge(
eventProp.ResultEventAsName,
eventProp.ResultEventProperty,
eventProp.ExprIdentNodeEvaluator,
eventProp.IsMustCoerce,
eventProp.CoercionType.GetBoxedType()));
}
else if (paramForge is FilterSpecParamEventPropIndexedForge) {
var eventProp = (FilterSpecParamEventPropIndexedForge) paramForge;
values.Add(
new FilterForEvalEventPropIndexedForge(
eventProp.ResultEventAsName,
eventProp.ResultEventIndex,
eventProp.ResultEventProperty,
eventProp.EventType,
eventProp.IsMustCoerce,
eventProp.CoercionType.GetBoxedType()));
}
else {
throw new ArgumentException("Unknown filter parameter:" + paramForge);
}
lastNotEqualsExprNode = filterParamExprMap.RemoveEntry(triplet);
}
FilterSpecParamInForge param = new FilterSpecParamInForge(
parameters[0].Param.Lookupable, FilterOperator.NOT_IN_LIST_OF_VALUES, values);
FilterSpecPlanPathTripletForge tripletForge = new FilterSpecPlanPathTripletForge(param, null);
filterParamExprMap.Put(lastNotEqualsExprNode, tripletForge);
}
public static IList<FilterSpecParamForge>[] PlanFilterParameters(
IList<ExprNode> validatedNodes,
FilterSpecCompilerArgs args)
{
if (validatedNodes.IsEmpty()) {
return AllocateListArray(0);
}
FilterSpecParaForgeMap filterParamExprMap = new FilterSpecParaForgeMap();
// Make filter parameter for each expression node, if it can be optimized
DecomposePopulateConsolidate(filterParamExprMap, validatedNodes, args);
// Use all filter parameter and unassigned expressions
IList<FilterSpecParamForge> filterParams = new List<FilterSpecParamForge>();
filterParams.AddAll(filterParamExprMap.FilterParams);
int countUnassigned = filterParamExprMap.CountUnassignedExpressions();
// we are done if there are no remaining nodes
if (countUnassigned == 0) {
return AllocateListArraySizeOne(filterParams);
}
// determine max-width
int filterServiceMaxFilterWidth =
args.compileTimeServices.Configuration.Compiler.Execution.FilterServiceMaxFilterWidth;
var hint = HintEnum.MAX_FILTER_WIDTH.GetHint(args.statementRawInfo.Annotations);
if (hint != null) {
string hintValue = HintEnum.MAX_FILTER_WIDTH.GetHintAssignedValue(hint);
filterServiceMaxFilterWidth = Int32.Parse(hintValue);
}
IList<FilterSpecParamForge>[] plan = null;
if (filterServiceMaxFilterWidth > 0) {
plan = PlanRemainingNodesIfFeasible(filterParamExprMap, args, filterServiceMaxFilterWidth);
}
if (plan != null) {
return plan;
}
// handle no-plan
FilterSpecParamForge node = MakeRemainingNode(filterParamExprMap.UnassignedExpressions, args);
filterParams.Add(node);
return AllocateListArraySizeOne(filterParams);
}
// remove duplicate propertyName + filterOperator items making a judgement to optimize or simply remove the optimized form
private static void Consolidate(
IList<FilterSpecPlanPathTripletForge> items,
FilterSpecParaForgeMap filterParamExprMap,
string statementName)
{
var op = items[0].Param.FilterOperator;
if (op == FilterOperator.NOT_EQUAL) {
HandleConsolidateNotEqual(items, filterParamExprMap, statementName);
}
else {
// for all others we simple remove the second optimized form (filter param with same prop name and filter op)
// and thus the boolean expression that started this is included
for (var i = 1; i < items.Count; i++) {
filterParamExprMap.RemoveValue(items[i]);
}
}
}
private static void DecomposePopulateConsolidate(
FilterSpecParaForgeMap filterParamExprMap,
IList<ExprNode> validatedNodes,
FilterSpecCompilerArgs args)
{
IList<ExprNode> constituents = DecomposeCheckAggregation(validatedNodes);
// Make filter parameter for each expression node, if it can be optimized
foreach (ExprNode constituent in constituents) {
FilterSpecParamForge param = FilterSpecCompilerMakeParamUtil.MakeFilterParam(
constituent,
args.arrayEventTypes,
args.statementRawInfo.StatementName);
filterParamExprMap.Put(constituent, param); // accepts null values as the expression may not be optimized
}
// Consolidate entries as possible, i.e. (a != 5 and a != 6) is (a not in (5,6))
// Removes duplicates for same property and same filter operator for filter service index optimizations
FilterSpecCompilerConsolidateUtil.Consolidate(filterParamExprMap, args.statementRawInfo.StatementName);
}
protected internal static void Consolidate(
FilterSpecParaForgeMap filterParamExprMap,
string statementName)
{
// consolidate or place in a boolean expression (by removing filter spec param from the map)
// any filter parameter that feature the same property name and filter operator,
// i.e. we are looking for "a!=5 and a!=6" to transform to "a not in (5,6)" which can match faster
// considering that "a not in (5,6) and a not in (7,8)" is "a not in (5, 6, 7, 8)" therefore
// we need to consolidate until there is no more work to do
var mapOfParams =
new Dictionary<Pair<ExprFilterSpecLookupableForge, FilterOperator>, IList<FilterSpecParamForge>>();
bool haveConsolidated;
do {
haveConsolidated = false;
mapOfParams.Clear();
// sort into buckets of propertyName + filterOperator combination
foreach (var currentParam in filterParamExprMap.FilterParams) {
var lookupable = currentParam.Lookupable;
var op = currentParam.FilterOperator;
var key = new Pair<ExprFilterSpecLookupableForge, FilterOperator>(lookupable, op);
var existingParam = mapOfParams.Get(key);
if (existingParam == null) {
existingParam = new List<FilterSpecParamForge>();
mapOfParams.Put(key, existingParam);
}
existingParam.Add(currentParam);
}
foreach (var entry in mapOfParams.Values) {
if (entry.Count > 1) {
haveConsolidated = true;
Consolidate(entry, filterParamExprMap, statementName);
}
}
} while (haveConsolidated);
}
public void Add(FilterSpecParaForgeMap other)
{
exprNodes.PutAll(other.exprNodes);
specParams.PutAll(other.specParams);
}
internal static FilterSpecPlanForge PlanRemainingNodesWithConditions(
FilterSpecParaForgeMap overallExpressions,
FilterSpecCompilerArgs args,
int filterServiceMaxFilterWidth,
ExprNode topLevelNegator)
{
var unassigned = overallExpressions.UnassignedExpressions;
var orNodes = new List <ExprOrNode>(unassigned.Count);
foreach (var node in unassigned)
{
if (node is ExprOrNode)
{
orNodes.Add((ExprOrNode)node);
}
}
var expressionsWithoutOr = new FilterSpecParaForgeMap();
expressionsWithoutOr.Add(overallExpressions);
// first dimension: or-node index
// second dimension: or child node index
var countOr = 0;
var sizeFactorized = 1;
var sizePerOr = new int[orNodes.Count];
var orChildNodes = new OrChildNode[orNodes.Count][];
var hasControl = false;
foreach (var orNode in orNodes)
{
expressionsWithoutOr.RemoveNode(orNode);
// get value-nodes and non-value nodes
var nonValueNodes = GetNonValueChildNodes(orNode);
var valueNodes = new List <ExprNode>(Arrays.AsList(orNode.ChildNodes));
valueNodes.RemoveAll(nonValueNodes);
ExprNode singleValueNode = ExprNodeUtilityMake.ConnectExpressionsByLogicalOrWhenNeeded(valueNodes);
// get all child nodes; last one is confirm if present
IList <ExprNode> allChildNodes = new List <ExprNode>(nonValueNodes);
if (singleValueNode != null)
{
allChildNodes.Add(singleValueNode);
}
var len = allChildNodes.Count;
orChildNodes[countOr] = new OrChildNode[len];
for (var i = 0; i < len; i++)
{
var child = allChildNodes[i];
if (child == singleValueNode)
{
hasControl = true;
orChildNodes[countOr][i] = new OrChildNodeV(singleValueNode);
}
else
{
var map = new FilterSpecParaForgeMap();
var nodes = Collections.SingletonList(child);
var confirm = DecomposePopulateConsolidate(map, true, nodes, args);
if (confirm == null)
{
orChildNodes[countOr][i] = new OrChildNodeNV(child, map);
}
else
{
hasControl = true;
orChildNodes[countOr][i] = new OrChildNodeNVNegated(child, map, confirm);
}
}
}
sizePerOr[countOr] = len;
sizeFactorized = sizeFactorized * len;
countOr++;
}
// compute permutations
var permutations = new CombPermutationTriplets[sizeFactorized];
var combinationEnumeration = CombinationEnumeration.FromZeroBasedRanges(sizePerOr);
var count = 0;
foreach (var permutation in combinationEnumeration)
{
permutations[count] = ComputePermutation(expressionsWithoutOr, permutation, orChildNodes, hasControl, args);
count++;
}
// Remove any permutations that only have a control-confirm
var result = new List <FilterSpecPlanPathForge>(sizeFactorized);
var pathControlConfirm = new List <ExprNode>();
foreach (var permutation in permutations)
{
if (permutation.Triplets.Length > 0)
{
result.Add(new FilterSpecPlanPathForge(permutation.Triplets, permutation.NegateCondition));
}
else
{
pathControlConfirm.Add(permutation.NegateCondition);
}
}
if (result.Count > filterServiceMaxFilterWidth)
{
return(null);
}
var pathArray = result.ToArray();
ExprNode topLevelConfirmer = ExprNodeUtilityMake.ConnectExpressionsByLogicalOrWhenNeeded(pathControlConfirm);
// determine when the path-negate condition is the same as the root confirm-expression
if (topLevelConfirmer != null)
{
var not = new ExprNotNode();
not.AddChildNode(topLevelConfirmer);
foreach (var path in pathArray)
{
if (ExprNodeUtilityCompare.DeepEquals(not, path.PathNegate, true))
{
path.PathNegate = null;
}
}
}
var convertor = new MatchedEventConvertorForge(
args.taggedEventTypes,
args.arrayEventTypes,
args.allTagNamesOrdered,
null,
true);
return(new FilterSpecPlanForge(pathArray, topLevelConfirmer, topLevelNegator, convertor));
}
private static CombPermutationTriplets ComputePermutation(
FilterSpecParaForgeMap filterParamExprMap,
object[] permutation,
OrChildNode[][] orChildNodes,
bool hasControl,
FilterSpecCompilerArgs args)
{
var mapAll = new FilterSpecParaForgeMap();
mapAll.Add(filterParamExprMap);
// combine
IList <ExprNode> nvPerOr = new List <ExprNode>(permutation.Length);
IList <ExprNode> negatingPath = new List <ExprNode>(permutation.Length);
for (var orNodeNum = 0; orNodeNum < permutation.Length; orNodeNum++)
{
int orChildNodeNum = permutation[orNodeNum].AsInt32();
var current = orChildNodes[orNodeNum][orChildNodeNum];
if (current is OrChildNodeNV)
{
var nv = (OrChildNodeNV)current;
mapAll.Add(nv.Map);
if (current is OrChildNodeNVNegated)
{
negatingPath.Add(((OrChildNodeNVNegated)current).Control);
}
}
else
{
var v = (OrChildNodeV)current;
negatingPath.Add(v.Node);
}
var orChildNodesForCurrent = orChildNodes[orNodeNum];
foreach (var other in orChildNodesForCurrent)
{
if (current == other)
{
continue;
}
if (other is OrChildNodeV)
{
var v = (OrChildNodeV)other;
var not = new ExprNotNode();
not.AddChildNode(v.Node);
nvPerOr.Add(not);
}
}
}
// consolidate across
FilterSpecCompilerConsolidateUtil.Consolidate(mapAll, args.statementRawInfo.StatementName);
IList <FilterSpecPlanPathTripletForge> triplets = new List <FilterSpecPlanPathTripletForge>(mapAll.Triplets);
var countUnassigned = mapAll.CountUnassignedExpressions();
if (countUnassigned != 0)
{
var triplet = MakeRemainingNode(mapAll.UnassignedExpressions, args);
triplets.Add(triplet);
}
// without conditions we are done
var tripletsArray = triplets.ToArray();
if (!hasControl)
{
return(new CombPermutationTriplets(tripletsArray, null));
}
var negatingNode = ExprNodeUtilityMake.ConnectExpressionsByLogicalAndWhenNeeded(negatingPath);
var excluded = ExprNodeUtilityMake.ConnectExpressionsByLogicalAndWhenNeeded(nvPerOr);
var merged = ExprNodeUtilityMake.ConnectExpressionsByLogicalAndWhenNeeded(negatingNode, excluded);
return(new CombPermutationTriplets(tripletsArray, merged));
}
