private static FilterSpecPlanPathForge ComputePermutation( FilterSpecParaForgeMap filterParamExprMap, object[] permutation, FilterSpecParaForgeMap[][] orNodesMaps, FilterSpecCompilerArgs args) { var mapAll = new FilterSpecParaForgeMap(); mapAll.Add(filterParamExprMap); // combine for (var orNodeNum = 0; orNodeNum < permutation.Length; orNodeNum++) { var orChildNodeNum = permutation[orNodeNum].AsInt32(); var mapOrSub = orNodesMaps[orNodeNum][orChildNodeNum]; mapAll.Add(mapOrSub); } // consolidate across FilterSpecCompilerConsolidateUtil.Consolidate(mapAll, args.statementRawInfo.StatementName); IList<FilterSpecPlanPathTripletForge> filterParams = new List<FilterSpecPlanPathTripletForge>(mapAll.Triplets); var countUnassigned = mapAll.CountUnassignedExpressions(); if (countUnassigned != 0) { FilterSpecPlanPathTripletForge node = MakeRemainingNode(mapAll.UnassignedExpressions, args); filterParams.Add(node); } FilterSpecPlanPathTripletForge[] triplets = filterParams.ToArray(); return new FilterSpecPlanPathForge(triplets, null); }
private static IList<FilterSpecParamForge> ComputePermutation( FilterSpecParaForgeMap filterParamExprMap, object[] permutation, FilterSpecParaForgeMap[][] orNodesMaps, FilterSpecCompilerArgs args) { FilterSpecParaForgeMap mapAll = new FilterSpecParaForgeMap(); mapAll.Add(filterParamExprMap); // combine for (int orNodeNum = 0; orNodeNum < permutation.Length; orNodeNum++) { int orChildNodeNum = (int) permutation[orNodeNum]; FilterSpecParaForgeMap mapOrSub = orNodesMaps[orNodeNum][orChildNodeNum]; mapAll.Add(mapOrSub); } // consolidate across FilterSpecCompilerConsolidateUtil.Consolidate(mapAll, args.statementRawInfo.StatementName); IList<FilterSpecParamForge> filterParams = new List<FilterSpecParamForge>(); filterParams.AddAll(mapAll.FilterParams); int countUnassigned = mapAll.CountUnassignedExpressions(); if (countUnassigned == 0) { return filterParams; } FilterSpecParamForge node = MakeRemainingNode(mapAll.UnassignedExpressions, args); filterParams.Add(node); return filterParams; }
public OrChildNodeNVNegated( ExprNode node, FilterSpecParaForgeMap map, ExprNode control) : base(node, map) { this.Control = control; }
public OrChildNodeNV( ExprNode node, FilterSpecParaForgeMap map) { this.Node = node; this.Map = map; }
private static FilterSpecPlanForge PlanFilterParametersInternal( IList<ExprNode> validatedNodes, FilterSpecCompilerArgs args) { if (validatedNodes.IsEmpty()) { return EMPTY; } if (args.compileTimeServices.Configuration.Compiler.Execution.FilterIndexPlanning == ConfigurationCompilerExecution.FilterIndexPlanningEnum.NONE) { DecomposeCheckAggregation(validatedNodes); return BuildNoPlan(validatedNodes, args); } var performConditionPlanning = HasLevelOrHint(FilterSpecCompilerIndexPlannerHint.CONDITIONS, args.statementRawInfo, args.compileTimeServices); var filterParamExprMap = new FilterSpecParaForgeMap(); // Make filter parameter for each expression node, if it can be optimized. // Optionally receive a top-level control condition that negates var topLevelNegation = DecomposePopulateConsolidate(filterParamExprMap, performConditionPlanning, validatedNodes, args); // Use all filter parameter and unassigned expressions var countUnassigned = filterParamExprMap.CountUnassignedExpressions(); // we are done if there are no remaining nodes if (countUnassigned == 0) { return MakePlanFromTriplets(filterParamExprMap.Triplets, topLevelNegation, args); } // determine max-width var filterServiceMaxFilterWidth = args.compileTimeServices.Configuration.Compiler.Execution.FilterServiceMaxFilterWidth; var hint = HintEnum.MAX_FILTER_WIDTH.GetHint(args.statementRawInfo.Annotations); if (hint != null) { var hintValue = HintEnum.MAX_FILTER_WIDTH.GetHintAssignedValue(hint); filterServiceMaxFilterWidth = int.Parse(hintValue); } FilterSpecPlanForge plan = null; if (filterServiceMaxFilterWidth > 0) { if (performConditionPlanning) { plan = PlanRemainingNodesWithConditions(filterParamExprMap, args, filterServiceMaxFilterWidth, topLevelNegation); } else { plan = PlanRemainingNodesBasic(filterParamExprMap, args, filterServiceMaxFilterWidth); } } if (plan != null) { return plan; } // handle no-plan var triplets = new List<FilterSpecPlanPathTripletForge>(filterParamExprMap.Triplets); var unassignedExpressions = filterParamExprMap.UnassignedExpressions; var triplet = MakeRemainingNode(unassignedExpressions, args); triplets.Add(triplet); return MakePlanFromTriplets(triplets, topLevelNegation, args); }
private static IList<FilterSpecParamForge>[] PlanRemainingNodesIfFeasible( FilterSpecParaForgeMap overallExpressions, FilterSpecCompilerArgs args, int filterServiceMaxFilterWidth) { IList<ExprNode> unassigned = overallExpressions.UnassignedExpressions; IList<ExprOrNode> orNodes = new List<ExprOrNode>(unassigned.Count); foreach (ExprNode node in unassigned) { if (node is ExprOrNode) { orNodes.Add((ExprOrNode) node); } } FilterSpecParaForgeMap expressionsWithoutOr = new FilterSpecParaForgeMap(); expressionsWithoutOr.Add(overallExpressions); // first dimension: or-node index // second dimension: or child node index FilterSpecParaForgeMap[][] orNodesMaps = new FilterSpecParaForgeMap[orNodes.Count][]; int countOr = 0; int sizeFactorized = 1; int[] sizePerOr = new int[orNodes.Count]; foreach (ExprOrNode orNode in orNodes) { expressionsWithoutOr.RemoveNode(orNode); orNodesMaps[countOr] = new FilterSpecParaForgeMap[orNode.ChildNodes.Length]; int len = orNode.ChildNodes.Length; for (int i = 0; i < len; i++) { FilterSpecParaForgeMap map = new FilterSpecParaForgeMap(); orNodesMaps[countOr][i] = map; IList<ExprNode> nodes = Collections.SingletonList(orNode.ChildNodes[i]); DecomposePopulateConsolidate(map, nodes, args); } sizePerOr[countOr] = len; sizeFactorized = sizeFactorized * len; countOr++; } // we become too large if (sizeFactorized > filterServiceMaxFilterWidth) { return null; } // combine IList<FilterSpecParamForge>[] result = new IList<FilterSpecParamForge>[sizeFactorized]; IEnumerable<object[]> permutations = CombinationEnumeration.FromZeroBasedRanges(sizePerOr); int count = 0; foreach (var permutation in permutations) { result[count] = ComputePermutation(expressionsWithoutOr, permutation, orNodesMaps, args); count++; } return result; }
internal static FilterSpecPlanForge PlanRemainingNodesBasic( FilterSpecParaForgeMap overallExpressions, FilterSpecCompilerArgs args, int filterServiceMaxFilterWidth) { var unassigned = overallExpressions.UnassignedExpressions; IList<ExprOrNode> orNodes = new List<ExprOrNode>(unassigned.Count); foreach (var node in unassigned) { if (node is ExprOrNode) { orNodes.Add((ExprOrNode) node); } } var expressionsWithoutOr = new FilterSpecParaForgeMap(); expressionsWithoutOr.Add(overallExpressions); // first dimension: or-node index // second dimension: or child node index var orNodesMaps = new FilterSpecParaForgeMap[orNodes.Count][]; var countOr = 0; var sizeFactorized = 1; var sizePerOr = new int[orNodes.Count]; foreach (var orNode in orNodes) { expressionsWithoutOr.RemoveNode(orNode); orNodesMaps[countOr] = new FilterSpecParaForgeMap[orNode.ChildNodes.Length]; var len = orNode.ChildNodes.Length; for (var i = 0; i < len; i++) { var map = new FilterSpecParaForgeMap(); orNodesMaps[countOr][i] = map; var nodes = Collections.SingletonList(orNode.ChildNodes[i]); DecomposePopulateConsolidate(map, false, nodes, args); } sizePerOr[countOr] = len; sizeFactorized = sizeFactorized * len; countOr++; } // we become too large if (sizeFactorized > filterServiceMaxFilterWidth) { return null; } // combine var result = new FilterSpecPlanPathForge[sizeFactorized]; var count = 0; foreach (var permutation in CombinationEnumeration.FromZeroBasedRanges(sizePerOr)) { result[count] = ComputePermutation(expressionsWithoutOr, permutation, orNodesMaps, args); count++; } return new FilterSpecPlanForge(result, null, null, null); }
internal static ExprNode DecomposePopulateConsolidate( FilterSpecParaForgeMap filterParamExprMap, bool performConditionPlanning, IList<ExprNode> validatedNodes, FilterSpecCompilerArgs args) { var constituents = DecomposeCheckAggregation(validatedNodes); // Remove constituents that are value-expressions ExprNode topLevelControl = null; if (performConditionPlanning) { IList<ExprNode> valueOnlyConstituents = null; foreach (var node in constituents) { var visitor = new FilterSpecExprNodeVisitorValueLimitedExpr(); node.Accept(visitor); if (visitor.IsLimited) { if (valueOnlyConstituents == null) { valueOnlyConstituents = new List<ExprNode>(); } valueOnlyConstituents.Add(node); } } if (valueOnlyConstituents != null) { constituents.RemoveAll(valueOnlyConstituents); topLevelControl = ExprNodeUtilityMake.ConnectExpressionsByLogicalAndWhenNeeded(valueOnlyConstituents); } } // Make filter parameter for each expression node, if it can be optimized foreach (var constituent in constituents) { var triplet = FilterSpecCompilerIndexPlannerConstituent.MakeFilterParam( constituent, performConditionPlanning, args.taggedEventTypes, args.arrayEventTypes, args.allTagNamesOrdered, args.statementRawInfo.StatementName, args.streamTypeService, args.statementRawInfo, args.compileTimeServices); filterParamExprMap.Put(constituent, triplet); // accepts null values as the expression may not be optimized } // Consolidate entries as possible, i.e. (a != 5 and a != 6) is (a not in (5,6)) // Removes duplicates for same property and same filter operator for filter service index optimizations FilterSpecCompilerConsolidateUtil.Consolidate(filterParamExprMap, args.statementRawInfo.StatementName); return topLevelControl; }
// consolidate "val != 3 and val != 4 and val != 5" // to "val not in (3, 4, 5)" private static void HandleConsolidateNotEqual( IList<FilterSpecPlanPathTripletForge> parameters, FilterSpecParaForgeMap filterParamExprMap, string statementName) { IList<FilterSpecParamInValueForge> values = new List<FilterSpecParamInValueForge>(); ExprNode lastNotEqualsExprNode = null; foreach (FilterSpecPlanPathTripletForge triplet in parameters) { FilterSpecParamForge paramForge = triplet.Param; if (paramForge is FilterSpecParamConstantForge) { var constantParam = (FilterSpecParamConstantForge) paramForge; var constant = constantParam.FilterConstant; values.Add(new FilterForEvalConstantAnyTypeForge(constant)); } else if (paramForge is FilterSpecParamEventPropForge) { var eventProp = (FilterSpecParamEventPropForge) paramForge; values.Add( new FilterForEvalEventPropForge( eventProp.ResultEventAsName, eventProp.ResultEventProperty, eventProp.ExprIdentNodeEvaluator, eventProp.IsMustCoerce, eventProp.CoercionType.GetBoxedType())); } else if (paramForge is FilterSpecParamEventPropIndexedForge) { var eventProp = (FilterSpecParamEventPropIndexedForge) paramForge; values.Add( new FilterForEvalEventPropIndexedForge( eventProp.ResultEventAsName, eventProp.ResultEventIndex, eventProp.ResultEventProperty, eventProp.EventType, eventProp.IsMustCoerce, eventProp.CoercionType.GetBoxedType())); } else { throw new ArgumentException("Unknown filter parameter:" + paramForge); } lastNotEqualsExprNode = filterParamExprMap.RemoveEntry(triplet); } FilterSpecParamInForge param = new FilterSpecParamInForge( parameters[0].Param.Lookupable, FilterOperator.NOT_IN_LIST_OF_VALUES, values); FilterSpecPlanPathTripletForge tripletForge = new FilterSpecPlanPathTripletForge(param, null); filterParamExprMap.Put(lastNotEqualsExprNode, tripletForge); }
public static IList<FilterSpecParamForge>[] PlanFilterParameters( IList<ExprNode> validatedNodes, FilterSpecCompilerArgs args) { if (validatedNodes.IsEmpty()) { return AllocateListArray(0); } FilterSpecParaForgeMap filterParamExprMap = new FilterSpecParaForgeMap(); // Make filter parameter for each expression node, if it can be optimized DecomposePopulateConsolidate(filterParamExprMap, validatedNodes, args); // Use all filter parameter and unassigned expressions IList<FilterSpecParamForge> filterParams = new List<FilterSpecParamForge>(); filterParams.AddAll(filterParamExprMap.FilterParams); int countUnassigned = filterParamExprMap.CountUnassignedExpressions(); // we are done if there are no remaining nodes if (countUnassigned == 0) { return AllocateListArraySizeOne(filterParams); } // determine max-width int filterServiceMaxFilterWidth = args.compileTimeServices.Configuration.Compiler.Execution.FilterServiceMaxFilterWidth; var hint = HintEnum.MAX_FILTER_WIDTH.GetHint(args.statementRawInfo.Annotations); if (hint != null) { string hintValue = HintEnum.MAX_FILTER_WIDTH.GetHintAssignedValue(hint); filterServiceMaxFilterWidth = Int32.Parse(hintValue); } IList<FilterSpecParamForge>[] plan = null; if (filterServiceMaxFilterWidth > 0) { plan = PlanRemainingNodesIfFeasible(filterParamExprMap, args, filterServiceMaxFilterWidth); } if (plan != null) { return plan; } // handle no-plan FilterSpecParamForge node = MakeRemainingNode(filterParamExprMap.UnassignedExpressions, args); filterParams.Add(node); return AllocateListArraySizeOne(filterParams); }
// remove duplicate propertyName + filterOperator items making a judgement to optimize or simply remove the optimized form private static void Consolidate( IList<FilterSpecPlanPathTripletForge> items, FilterSpecParaForgeMap filterParamExprMap, string statementName) { var op = items[0].Param.FilterOperator; if (op == FilterOperator.NOT_EQUAL) { HandleConsolidateNotEqual(items, filterParamExprMap, statementName); } else { // for all others we simple remove the second optimized form (filter param with same prop name and filter op) // and thus the boolean expression that started this is included for (var i = 1; i < items.Count; i++) { filterParamExprMap.RemoveValue(items[i]); } } }
private static void DecomposePopulateConsolidate( FilterSpecParaForgeMap filterParamExprMap, IList<ExprNode> validatedNodes, FilterSpecCompilerArgs args) { IList<ExprNode> constituents = DecomposeCheckAggregation(validatedNodes); // Make filter parameter for each expression node, if it can be optimized foreach (ExprNode constituent in constituents) { FilterSpecParamForge param = FilterSpecCompilerMakeParamUtil.MakeFilterParam( constituent, args.arrayEventTypes, args.statementRawInfo.StatementName); filterParamExprMap.Put(constituent, param); // accepts null values as the expression may not be optimized } // Consolidate entries as possible, i.e. (a != 5 and a != 6) is (a not in (5,6)) // Removes duplicates for same property and same filter operator for filter service index optimizations FilterSpecCompilerConsolidateUtil.Consolidate(filterParamExprMap, args.statementRawInfo.StatementName); }
protected internal static void Consolidate( FilterSpecParaForgeMap filterParamExprMap, string statementName) { // consolidate or place in a boolean expression (by removing filter spec param from the map) // any filter parameter that feature the same property name and filter operator, // i.e. we are looking for "a!=5 and a!=6" to transform to "a not in (5,6)" which can match faster // considering that "a not in (5,6) and a not in (7,8)" is "a not in (5, 6, 7, 8)" therefore // we need to consolidate until there is no more work to do var mapOfParams = new Dictionary<Pair<ExprFilterSpecLookupableForge, FilterOperator>, IList<FilterSpecParamForge>>(); bool haveConsolidated; do { haveConsolidated = false; mapOfParams.Clear(); // sort into buckets of propertyName + filterOperator combination foreach (var currentParam in filterParamExprMap.FilterParams) { var lookupable = currentParam.Lookupable; var op = currentParam.FilterOperator; var key = new Pair<ExprFilterSpecLookupableForge, FilterOperator>(lookupable, op); var existingParam = mapOfParams.Get(key); if (existingParam == null) { existingParam = new List<FilterSpecParamForge>(); mapOfParams.Put(key, existingParam); } existingParam.Add(currentParam); } foreach (var entry in mapOfParams.Values) { if (entry.Count > 1) { haveConsolidated = true; Consolidate(entry, filterParamExprMap, statementName); } } } while (haveConsolidated); }
public void Add(FilterSpecParaForgeMap other) { exprNodes.PutAll(other.exprNodes); specParams.PutAll(other.specParams); }
internal static FilterSpecPlanForge PlanRemainingNodesWithConditions( FilterSpecParaForgeMap overallExpressions, FilterSpecCompilerArgs args, int filterServiceMaxFilterWidth, ExprNode topLevelNegator) { var unassigned = overallExpressions.UnassignedExpressions; var orNodes = new List <ExprOrNode>(unassigned.Count); foreach (var node in unassigned) { if (node is ExprOrNode) { orNodes.Add((ExprOrNode)node); } } var expressionsWithoutOr = new FilterSpecParaForgeMap(); expressionsWithoutOr.Add(overallExpressions); // first dimension: or-node index // second dimension: or child node index var countOr = 0; var sizeFactorized = 1; var sizePerOr = new int[orNodes.Count]; var orChildNodes = new OrChildNode[orNodes.Count][]; var hasControl = false; foreach (var orNode in orNodes) { expressionsWithoutOr.RemoveNode(orNode); // get value-nodes and non-value nodes var nonValueNodes = GetNonValueChildNodes(orNode); var valueNodes = new List <ExprNode>(Arrays.AsList(orNode.ChildNodes)); valueNodes.RemoveAll(nonValueNodes); ExprNode singleValueNode = ExprNodeUtilityMake.ConnectExpressionsByLogicalOrWhenNeeded(valueNodes); // get all child nodes; last one is confirm if present IList <ExprNode> allChildNodes = new List <ExprNode>(nonValueNodes); if (singleValueNode != null) { allChildNodes.Add(singleValueNode); } var len = allChildNodes.Count; orChildNodes[countOr] = new OrChildNode[len]; for (var i = 0; i < len; i++) { var child = allChildNodes[i]; if (child == singleValueNode) { hasControl = true; orChildNodes[countOr][i] = new OrChildNodeV(singleValueNode); } else { var map = new FilterSpecParaForgeMap(); var nodes = Collections.SingletonList(child); var confirm = DecomposePopulateConsolidate(map, true, nodes, args); if (confirm == null) { orChildNodes[countOr][i] = new OrChildNodeNV(child, map); } else { hasControl = true; orChildNodes[countOr][i] = new OrChildNodeNVNegated(child, map, confirm); } } } sizePerOr[countOr] = len; sizeFactorized = sizeFactorized * len; countOr++; } // compute permutations var permutations = new CombPermutationTriplets[sizeFactorized]; var combinationEnumeration = CombinationEnumeration.FromZeroBasedRanges(sizePerOr); var count = 0; foreach (var permutation in combinationEnumeration) { permutations[count] = ComputePermutation(expressionsWithoutOr, permutation, orChildNodes, hasControl, args); count++; } // Remove any permutations that only have a control-confirm var result = new List <FilterSpecPlanPathForge>(sizeFactorized); var pathControlConfirm = new List <ExprNode>(); foreach (var permutation in permutations) { if (permutation.Triplets.Length > 0) { result.Add(new FilterSpecPlanPathForge(permutation.Triplets, permutation.NegateCondition)); } else { pathControlConfirm.Add(permutation.NegateCondition); } } if (result.Count > filterServiceMaxFilterWidth) { return(null); } var pathArray = result.ToArray(); ExprNode topLevelConfirmer = ExprNodeUtilityMake.ConnectExpressionsByLogicalOrWhenNeeded(pathControlConfirm); // determine when the path-negate condition is the same as the root confirm-expression if (topLevelConfirmer != null) { var not = new ExprNotNode(); not.AddChildNode(topLevelConfirmer); foreach (var path in pathArray) { if (ExprNodeUtilityCompare.DeepEquals(not, path.PathNegate, true)) { path.PathNegate = null; } } } var convertor = new MatchedEventConvertorForge( args.taggedEventTypes, args.arrayEventTypes, args.allTagNamesOrdered, null, true); return(new FilterSpecPlanForge(pathArray, topLevelConfirmer, topLevelNegator, convertor)); }
private static CombPermutationTriplets ComputePermutation( FilterSpecParaForgeMap filterParamExprMap, object[] permutation, OrChildNode[][] orChildNodes, bool hasControl, FilterSpecCompilerArgs args) { var mapAll = new FilterSpecParaForgeMap(); mapAll.Add(filterParamExprMap); // combine IList <ExprNode> nvPerOr = new List <ExprNode>(permutation.Length); IList <ExprNode> negatingPath = new List <ExprNode>(permutation.Length); for (var orNodeNum = 0; orNodeNum < permutation.Length; orNodeNum++) { int orChildNodeNum = permutation[orNodeNum].AsInt32(); var current = orChildNodes[orNodeNum][orChildNodeNum]; if (current is OrChildNodeNV) { var nv = (OrChildNodeNV)current; mapAll.Add(nv.Map); if (current is OrChildNodeNVNegated) { negatingPath.Add(((OrChildNodeNVNegated)current).Control); } } else { var v = (OrChildNodeV)current; negatingPath.Add(v.Node); } var orChildNodesForCurrent = orChildNodes[orNodeNum]; foreach (var other in orChildNodesForCurrent) { if (current == other) { continue; } if (other is OrChildNodeV) { var v = (OrChildNodeV)other; var not = new ExprNotNode(); not.AddChildNode(v.Node); nvPerOr.Add(not); } } } // consolidate across FilterSpecCompilerConsolidateUtil.Consolidate(mapAll, args.statementRawInfo.StatementName); IList <FilterSpecPlanPathTripletForge> triplets = new List <FilterSpecPlanPathTripletForge>(mapAll.Triplets); var countUnassigned = mapAll.CountUnassignedExpressions(); if (countUnassigned != 0) { var triplet = MakeRemainingNode(mapAll.UnassignedExpressions, args); triplets.Add(triplet); } // without conditions we are done var tripletsArray = triplets.ToArray(); if (!hasControl) { return(new CombPermutationTriplets(tripletsArray, null)); } var negatingNode = ExprNodeUtilityMake.ConnectExpressionsByLogicalAndWhenNeeded(negatingPath); var excluded = ExprNodeUtilityMake.ConnectExpressionsByLogicalAndWhenNeeded(nvPerOr); var merged = ExprNodeUtilityMake.ConnectExpressionsByLogicalAndWhenNeeded(negatingNode, excluded); return(new CombPermutationTriplets(tripletsArray, merged)); }