private static IList<FilterSpecParamForge>[] PlanRemainingNodesIfFeasible( FilterSpecParaForgeMap overallExpressions, FilterSpecCompilerArgs args, int filterServiceMaxFilterWidth) { IList<ExprNode> unassigned = overallExpressions.UnassignedExpressions; IList<ExprOrNode> orNodes = new List<ExprOrNode>(unassigned.Count); foreach (ExprNode node in unassigned) { if (node is ExprOrNode) { orNodes.Add((ExprOrNode) node); } } FilterSpecParaForgeMap expressionsWithoutOr = new FilterSpecParaForgeMap(); expressionsWithoutOr.Add(overallExpressions); // first dimension: or-node index // second dimension: or child node index FilterSpecParaForgeMap[][] orNodesMaps = new FilterSpecParaForgeMap[orNodes.Count][]; int countOr = 0; int sizeFactorized = 1; int[] sizePerOr = new int[orNodes.Count]; foreach (ExprOrNode orNode in orNodes) { expressionsWithoutOr.RemoveNode(orNode); orNodesMaps[countOr] = new FilterSpecParaForgeMap[orNode.ChildNodes.Length]; int len = orNode.ChildNodes.Length; for (int i = 0; i < len; i++) { FilterSpecParaForgeMap map = new FilterSpecParaForgeMap(); orNodesMaps[countOr][i] = map; IList<ExprNode> nodes = Collections.SingletonList(orNode.ChildNodes[i]); DecomposePopulateConsolidate(map, nodes, args); } sizePerOr[countOr] = len; sizeFactorized = sizeFactorized * len; countOr++; } // we become too large if (sizeFactorized > filterServiceMaxFilterWidth) { return null; } // combine IList<FilterSpecParamForge>[] result = new IList<FilterSpecParamForge>[sizeFactorized]; IEnumerable<object[]> permutations = CombinationEnumeration.FromZeroBasedRanges(sizePerOr); int count = 0; foreach (var permutation in permutations) { result[count] = ComputePermutation(expressionsWithoutOr, permutation, orNodesMaps, args); count++; } return result; }
internal static FilterSpecPlanForge PlanRemainingNodesBasic( FilterSpecParaForgeMap overallExpressions, FilterSpecCompilerArgs args, int filterServiceMaxFilterWidth) { var unassigned = overallExpressions.UnassignedExpressions; IList<ExprOrNode> orNodes = new List<ExprOrNode>(unassigned.Count); foreach (var node in unassigned) { if (node is ExprOrNode) { orNodes.Add((ExprOrNode) node); } } var expressionsWithoutOr = new FilterSpecParaForgeMap(); expressionsWithoutOr.Add(overallExpressions); // first dimension: or-node index // second dimension: or child node index var orNodesMaps = new FilterSpecParaForgeMap[orNodes.Count][]; var countOr = 0; var sizeFactorized = 1; var sizePerOr = new int[orNodes.Count]; foreach (var orNode in orNodes) { expressionsWithoutOr.RemoveNode(orNode); orNodesMaps[countOr] = new FilterSpecParaForgeMap[orNode.ChildNodes.Length]; var len = orNode.ChildNodes.Length; for (var i = 0; i < len; i++) { var map = new FilterSpecParaForgeMap(); orNodesMaps[countOr][i] = map; var nodes = Collections.SingletonList(orNode.ChildNodes[i]); DecomposePopulateConsolidate(map, false, nodes, args); } sizePerOr[countOr] = len; sizeFactorized = sizeFactorized * len; countOr++; } // we become too large if (sizeFactorized > filterServiceMaxFilterWidth) { return null; } // combine var result = new FilterSpecPlanPathForge[sizeFactorized]; var count = 0; foreach (var permutation in CombinationEnumeration.FromZeroBasedRanges(sizePerOr)) { result[count] = ComputePermutation(expressionsWithoutOr, permutation, orNodesMaps, args); count++; } return new FilterSpecPlanForge(result, null, null, null); }
public static GroupByClauseExpressions GetGroupByRollupExpressions( IList<GroupByClauseElement> groupByElements, SelectClauseSpecRaw selectClauseSpec, ExprNode optionalHavingNode, IList<OrderByItem> orderByList, ExpressionCopier expressionCopier) { if (groupByElements == null || groupByElements.Count == 0) { return null; } // walk group-by-elements, determine group-by expressions and rollup nodes var groupByExpressionInfo = GroupByToRollupNodes(groupByElements); // obtain expression nodes, collect unique nodes and assign index IList<ExprNode> distinctGroupByExpressions = new List<ExprNode>(); IDictionary<ExprNode, int> expressionToIndex = new Dictionary<ExprNode, int>(); foreach (var exprNode in groupByExpressionInfo.Expressions) { var found = false; for (var i = 0; i < distinctGroupByExpressions.Count; i++) { ExprNode other = distinctGroupByExpressions[i]; // find same expression if (ExprNodeUtilityCompare.DeepEquals(exprNode, other, false)) { expressionToIndex.Put(exprNode, i); found = true; break; } } // not seen before if (!found) { expressionToIndex.Put(exprNode, distinctGroupByExpressions.Count); distinctGroupByExpressions.Add(exprNode); } } // determine rollup, validate it is either (not both) var hasGroupingSet = false; var hasRollup = false; foreach (var element in groupByElements) { if (element is GroupByClauseElementGroupingSet) { hasGroupingSet = true; } if (element is GroupByClauseElementRollupOrCube) { hasRollup = true; } } // no-rollup or grouping-sets means simply validate ExprNode[] groupByExpressions = distinctGroupByExpressions.ToArray(); if (!hasRollup && !hasGroupingSet) { return new GroupByClauseExpressions(groupByExpressions); } // evaluate rollup node roots var nodes = groupByExpressionInfo.Nodes; var perNodeCombinations = new object[nodes.Count][]; var context = new GroupByRollupEvalContext(expressionToIndex); try { for (var i = 0; i < nodes.Count; i++) { GroupByRollupNodeBase node = nodes[i]; var combinations = node.Evaluate(context); perNodeCombinations[i] = new object[combinations.Count]; for (var j = 0; j < combinations.Count; j++) { perNodeCombinations[i][j] = combinations[j]; } } } catch (GroupByRollupDuplicateException ex) { if (ex.Indexes.Length == 0) { throw new ExprValidationException( "Failed to validate the group-by clause, found duplicate specification of the overall grouping '()'"); } var writer = new StringWriter(); var delimiter = ""; for (var i = 0; i < ex.Indexes.Length; i++) { writer.Write(delimiter); writer.Write( ExprNodeUtilityPrint.ToExpressionStringMinPrecedenceSafe(groupByExpressions[ex.Indexes[i]])); delimiter = ", "; } throw new ExprValidationException( "Failed to validate the group-by clause, found duplicate specification of expressions (" + writer + ")"); } // enumerate combinations building an index list var combinationEnumeration = new CombinationEnumeration(perNodeCombinations); var combination = new SortedSet<int>(); var indexList = new LinkedHashSet<MultiKeyArrayInt>(); while (combinationEnumeration.MoveNext()) { combination.Clear(); object[] combinationOA = combinationEnumeration.Current; foreach (var indexes in combinationOA) { var indexarr = (int[]) indexes; foreach (var anIndex in indexarr) { combination.Add(anIndex); } } var indexArr = CollectionUtil.IntArray(combination); indexList.Add(new MultiKeyArrayInt(indexArr)); } // obtain rollup levels var rollupLevels = new int[indexList.Count][]; var count = 0; foreach (var mk in indexList) { rollupLevels[count++] = mk.Keys; } var numberOfLevels = rollupLevels.Length; if (numberOfLevels == 1 && rollupLevels[0].Length == 0) { throw new ExprValidationException( "Failed to validate the group-by clause, the overall grouping '()' cannot be the only grouping"); } // obtain select-expression copies for rewrite var expressions = selectClauseSpec.SelectExprList; var selects = new ExprNode[numberOfLevels][]; for (var i = 0; i < numberOfLevels; i++) { selects[i] = new ExprNode[expressions.Count]; for (var j = 0; j < expressions.Count; j++) { SelectClauseElementRaw selectRaw = expressions[j]; if (!(selectRaw is SelectClauseExprRawSpec)) { throw new ExprValidationException( "Group-by with rollup requires that the select-clause does not use wildcard"); } var compiled = (SelectClauseExprRawSpec) selectRaw; selects[i][j] = CopyVisitExpression(compiled.SelectExpression, expressionCopier); } } // obtain having-expression copies for rewrite ExprNode[] optHavingNodeCopy = null; if (optionalHavingNode != null) { optHavingNodeCopy = new ExprNode[numberOfLevels]; for (var i = 0; i < numberOfLevels; i++) { optHavingNodeCopy[i] = CopyVisitExpression(optionalHavingNode, expressionCopier); } } // obtain orderby-expression copies for rewrite ExprNode[][] optOrderByCopy = null; if (orderByList != null && orderByList.Count > 0) { optOrderByCopy = new ExprNode[numberOfLevels][]; for (var i = 0; i < numberOfLevels; i++) { optOrderByCopy[i] = new ExprNode[orderByList.Count]; for (var j = 0; j < orderByList.Count; j++) { OrderByItem element = orderByList[j]; optOrderByCopy[i][j] = CopyVisitExpression(element.ExprNode, expressionCopier); } } } return new GroupByClauseExpressions( groupByExpressions, rollupLevels, selects, optHavingNodeCopy, optOrderByCopy); }
public static ICollection <EventBean> Snapshot( FilterSpecCompiled optionalFilter, Attribute[] annotations, VirtualDWView virtualDataWindow, EventTableIndexRepository indexRepository, bool queryPlanLogging, ILog queryPlanLogDestination, string objectName, AgentInstanceContext agentInstanceContext) { if (optionalFilter == null || optionalFilter.Parameters.Length == 0) { if (virtualDataWindow != null) { var pair = virtualDataWindow.GetFireAndForgetDesc(Collections.GetEmptySet <string>(), Collections.GetEmptySet <string>()); return(virtualDataWindow.GetFireAndForgetData(pair.Second, new object[0], new RangeIndexLookupValue[0], annotations)); } return(null); } // Determine what straight-equals keys and which ranges are available. // Widening/Coercion is part of filter spec compile. ISet <string> keysAvailable = new HashSet <string>(); ISet <string> rangesAvailable = new HashSet <string>(); if (optionalFilter.Parameters.Length == 1) { foreach (FilterSpecParam param in optionalFilter.Parameters[0]) { if (!(param is FilterSpecParamConstant || param is FilterSpecParamRange || param is FilterSpecParamIn)) { continue; } if (param.FilterOperator == FilterOperator.EQUAL || param.FilterOperator == FilterOperator.IS || param.FilterOperator == FilterOperator.IN_LIST_OF_VALUES) { keysAvailable.Add(param.Lookupable.Expression); } else if (param.FilterOperator.IsRangeOperator() || param.FilterOperator.IsInvertedRangeOperator() || param.FilterOperator.IsComparisonOperator()) { rangesAvailable.Add(param.Lookupable.Expression); } else if (param.FilterOperator.IsRangeOperator()) { rangesAvailable.Add(param.Lookupable.Expression); } } } // Find an index that matches the needs Pair <IndexMultiKey, EventTableAndNamePair> tablePair; if (virtualDataWindow != null) { var tablePairNoName = virtualDataWindow.GetFireAndForgetDesc(keysAvailable, rangesAvailable); tablePair = new Pair <IndexMultiKey, EventTableAndNamePair>(tablePairNoName.First, new EventTableAndNamePair(tablePairNoName.Second, null)); } else { var indexHint = IndexHint.GetIndexHint(annotations); IList <IndexHintInstruction> optionalIndexHintInstructions = null; if (indexHint != null) { optionalIndexHintInstructions = indexHint.InstructionsFireAndForget; } tablePair = indexRepository.FindTable(keysAvailable, rangesAvailable, optionalIndexHintInstructions); } var hook = QueryPlanIndexHookUtil.GetHook(annotations); if (queryPlanLogging && (queryPlanLogDestination.IsInfoEnabled || hook != null)) { var prefix = "Fire-and-forget from " + objectName + " "; var indexName = tablePair != null && tablePair.Second != null ? tablePair.Second.IndexName : null; var indexText = indexName != null ? "index " + indexName + " " : "full table scan "; indexText += "(snapshot only, for join see separate query plan)"; if (tablePair == null) { queryPlanLogDestination.Info(prefix + indexText); } else { queryPlanLogDestination.Info(prefix + indexText + tablePair.Second.EventTable.ToQueryPlan()); } if (hook != null) { hook.FireAndForget(new QueryPlanIndexDescFAF( new IndexNameAndDescPair[] { new IndexNameAndDescPair(indexName, tablePair != null ? tablePair.Second.EventTable.ProviderClass.Name : null) })); } } if (tablePair == null) { return(null); // indicates table scan } // Compile key sets which contain key index lookup values var keyIndexProps = IndexedPropDesc.GetIndexProperties(tablePair.First.HashIndexedProps); var hasKeyWithInClause = false; var keyValues = new object[keyIndexProps.Length]; for (var keyIndex = 0; keyIndex < keyIndexProps.Length; keyIndex++) { foreach (var param in optionalFilter.Parameters[0]) { if (param.Lookupable.Expression.Equals(keyIndexProps[keyIndex])) { if (param.FilterOperator == FilterOperator.IN_LIST_OF_VALUES) { var keyValuesList = ((MultiKeyUntyped)param.GetFilterValue(null, agentInstanceContext)).Keys; if (keyValuesList.Length == 0) { continue; } else if (keyValuesList.Length == 1) { keyValues[keyIndex] = keyValuesList[0]; } else { keyValues[keyIndex] = keyValuesList; hasKeyWithInClause = true; } } else { keyValues[keyIndex] = param.GetFilterValue(null, agentInstanceContext); } break; } } } // Analyze ranges - these may include key lookup value (EQUALS semantics) var rangeIndexProps = IndexedPropDesc.GetIndexProperties(tablePair.First.RangeIndexedProps); RangeIndexLookupValue[] rangeValues; if (rangeIndexProps.Length > 0) { rangeValues = CompileRangeLookupValues(rangeIndexProps, optionalFilter.Parameters[0], agentInstanceContext); } else { rangeValues = new RangeIndexLookupValue[0]; } var eventTable = tablePair.Second.EventTable; var indexMultiKey = tablePair.First; // table lookup without in-clause if (!hasKeyWithInClause) { return(FafTableLookup(virtualDataWindow, indexMultiKey, eventTable, keyValues, rangeValues, annotations)); } // table lookup with in-clause: determine combinations var combinations = new object[keyIndexProps.Length][]; for (var i = 0; i < keyValues.Length; i++) { if (keyValues[i] is object[]) { combinations[i] = (object[])keyValues[i]; } else { combinations[i] = new object[] { keyValues[i] }; } } // enumerate combinations var enumeration = new CombinationEnumeration(combinations); var events = new HashSet <EventBean>(); for (; enumeration.MoveNext();) { object[] keys = enumeration.Current; var result = FafTableLookup(virtualDataWindow, indexMultiKey, eventTable, keys, rangeValues, annotations); events.AddAll(result); } return(events); }
/// <summary> /// The SnapshotInKeyword /// </summary> /// <param name="queryGraphValue">The <see cref="QueryGraphValue"/></param> /// <param name="indexRepository">The <see cref="EventTableIndexRepository"/></param> /// <param name="virtualDataWindow">The <see cref="VirtualDWView"/></param> /// <param name="attributes">The <see cref="Attribute" /> array</param> /// <param name="agentInstanceContext">The <see cref="AgentInstanceContext"/></param> /// <param name="queryPlanLogging">The <see cref="bool"/></param> /// <param name="queryPlanLogDestination">The <see cref="ILog"/></param> /// <param name="objectName">The <see cref="string"/></param> /// <returns>The collection of event beans</returns> private static NullableObject <ICollection <EventBean> > SnapshotInKeyword( QueryGraphValue queryGraphValue, EventTableIndexRepository indexRepository, VirtualDWView virtualDataWindow, Attribute[] attributes, AgentInstanceContext agentInstanceContext, bool queryPlanLogging, ILog queryPlanLogDestination, string objectName) { var inkwSingles = queryGraphValue.InKeywordSingles; if (inkwSingles.Indexed.Length == 0) { return(null); } var tablePair = FindIndex( new HashSet <string>(inkwSingles.Indexed), Collections.GetEmptySet <string>(), indexRepository, virtualDataWindow, attributes); if (tablePair == null) { return(null); } queryPlanReport(tablePair.Second.IndexName, tablePair.Second.EventTable, attributes, agentInstanceContext, queryPlanLogging, queryPlanLogDestination, objectName); var evaluateParamsTrue = new EvaluateParams(null, true, agentInstanceContext); // table lookup with in-clause: determine combinations var tableHashProps = tablePair.First.HashIndexedProps; var combinations = new Object[tableHashProps.Length][]; for (var tableHashPropNum = 0; tableHashPropNum < tableHashProps.Length; tableHashPropNum++) { for (var i = 0; i < inkwSingles.Indexed.Length; i++) { if (inkwSingles.Indexed[i].Equals(tableHashProps[tableHashPropNum].IndexPropName)) { var keysExpressions = inkwSingles.Key[i]; var values = new Object[keysExpressions.KeyExprs.Count]; combinations[tableHashPropNum] = values; for (var j = 0; j < keysExpressions.KeyExprs.Count; j++) { values[j] = keysExpressions.KeyExprs[j].ExprEvaluator.Evaluate(evaluateParamsTrue); } } } } // enumerate combinations var enumeration = CombinationEnumeration.New(combinations); var events = new HashSet <EventBean>(); foreach (Object[] keys in enumeration) { var result = FafTableLookup(virtualDataWindow, tablePair.First, tablePair.Second.EventTable, keys, null, attributes); events.AddAll(result); } return(new NullableObject <ICollection <EventBean> >(events)); }
private static NullableObject<ICollection<EventBean>> SnapshotInKeyword( QueryGraphValue queryGraphValue, EventTableIndexRepository indexRepository, VirtualDWView virtualDataWindow, Attribute[] annotations, AgentInstanceContext agentInstanceContext, string objectName) { var inkwSingles = queryGraphValue.InKeywordSingles; if (inkwSingles.Indexed.Length == 0) { return null; } Pair<IndexMultiKey, EventTableAndNamePair> tablePair = FindIndex( new HashSet<string>(inkwSingles.Indexed), new EmptySet<string>(), indexRepository, virtualDataWindow, annotations); if (tablePair == null) { return null; } QueryPlanReport( tablePair.Second.IndexName, tablePair.Second.EventTable, annotations, agentInstanceContext, objectName); // table lookup with in-clause: determine combinations var tableHashProps = tablePair.First.HashIndexedProps; var combinations = new object[tableHashProps.Length][]; for (var tableHashPropNum = 0; tableHashPropNum < tableHashProps.Length; tableHashPropNum++) { for (var i = 0; i < inkwSingles.Indexed.Length; i++) { if (inkwSingles.Indexed[i].Equals(tableHashProps[tableHashPropNum].IndexPropName)) { QueryGraphValueEntryInKeywordSingleIdx keysExpressions = inkwSingles.Key[i]; var values = new object[keysExpressions.KeyExprs.Length]; combinations[tableHashPropNum] = values; for (var j = 0; j < keysExpressions.KeyExprs.Length; j++) { values[j] = keysExpressions.KeyExprs[j].Evaluate(null, true, agentInstanceContext); } } } } // enumerate combinations var enumeration = new CombinationEnumeration(combinations); var events = new HashSet<EventBean>(); while (enumeration.MoveNext()) { object[] keys = enumeration.Current; var result = FafTableLookup( virtualDataWindow, tablePair.First, tablePair.Second.EventTable, keys, null, annotations, agentInstanceContext); events.AddAll(result); } return new NullableObject<ICollection<EventBean>>(events); }
internal static FilterSpecPlanForge PlanRemainingNodesWithConditions( FilterSpecParaForgeMap overallExpressions, FilterSpecCompilerArgs args, int filterServiceMaxFilterWidth, ExprNode topLevelNegator) { var unassigned = overallExpressions.UnassignedExpressions; var orNodes = new List <ExprOrNode>(unassigned.Count); foreach (var node in unassigned) { if (node is ExprOrNode) { orNodes.Add((ExprOrNode)node); } } var expressionsWithoutOr = new FilterSpecParaForgeMap(); expressionsWithoutOr.Add(overallExpressions); // first dimension: or-node index // second dimension: or child node index var countOr = 0; var sizeFactorized = 1; var sizePerOr = new int[orNodes.Count]; var orChildNodes = new OrChildNode[orNodes.Count][]; var hasControl = false; foreach (var orNode in orNodes) { expressionsWithoutOr.RemoveNode(orNode); // get value-nodes and non-value nodes var nonValueNodes = GetNonValueChildNodes(orNode); var valueNodes = new List <ExprNode>(Arrays.AsList(orNode.ChildNodes)); valueNodes.RemoveAll(nonValueNodes); ExprNode singleValueNode = ExprNodeUtilityMake.ConnectExpressionsByLogicalOrWhenNeeded(valueNodes); // get all child nodes; last one is confirm if present IList <ExprNode> allChildNodes = new List <ExprNode>(nonValueNodes); if (singleValueNode != null) { allChildNodes.Add(singleValueNode); } var len = allChildNodes.Count; orChildNodes[countOr] = new OrChildNode[len]; for (var i = 0; i < len; i++) { var child = allChildNodes[i]; if (child == singleValueNode) { hasControl = true; orChildNodes[countOr][i] = new OrChildNodeV(singleValueNode); } else { var map = new FilterSpecParaForgeMap(); var nodes = Collections.SingletonList(child); var confirm = DecomposePopulateConsolidate(map, true, nodes, args); if (confirm == null) { orChildNodes[countOr][i] = new OrChildNodeNV(child, map); } else { hasControl = true; orChildNodes[countOr][i] = new OrChildNodeNVNegated(child, map, confirm); } } } sizePerOr[countOr] = len; sizeFactorized = sizeFactorized * len; countOr++; } // compute permutations var permutations = new CombPermutationTriplets[sizeFactorized]; var combinationEnumeration = CombinationEnumeration.FromZeroBasedRanges(sizePerOr); var count = 0; foreach (var permutation in combinationEnumeration) { permutations[count] = ComputePermutation(expressionsWithoutOr, permutation, orChildNodes, hasControl, args); count++; } // Remove any permutations that only have a control-confirm var result = new List <FilterSpecPlanPathForge>(sizeFactorized); var pathControlConfirm = new List <ExprNode>(); foreach (var permutation in permutations) { if (permutation.Triplets.Length > 0) { result.Add(new FilterSpecPlanPathForge(permutation.Triplets, permutation.NegateCondition)); } else { pathControlConfirm.Add(permutation.NegateCondition); } } if (result.Count > filterServiceMaxFilterWidth) { return(null); } var pathArray = result.ToArray(); ExprNode topLevelConfirmer = ExprNodeUtilityMake.ConnectExpressionsByLogicalOrWhenNeeded(pathControlConfirm); // determine when the path-negate condition is the same as the root confirm-expression if (topLevelConfirmer != null) { var not = new ExprNotNode(); not.AddChildNode(topLevelConfirmer); foreach (var path in pathArray) { if (ExprNodeUtilityCompare.DeepEquals(not, path.PathNegate, true)) { path.PathNegate = null; } } } var convertor = new MatchedEventConvertorForge( args.taggedEventTypes, args.arrayEventTypes, args.allTagNamesOrdered, null, true); return(new FilterSpecPlanForge(pathArray, topLevelConfirmer, topLevelNegator, convertor)); }