public void TestCreateStreamPlan() { QueryPlanIndex[] indexes = QueryPlanIndexBuilder.BuildIndexSpec(_queryGraph, _typesPerStream, new string[_queryGraph.NumStreams][][]); for (int i = 0; i < indexes.Length; i++) { Log.Debug(".testCreateStreamPlan index " + i + " = " + indexes[i]); } QueryPlanNode plan = NStreamQueryPlanBuilder.CreateStreamPlan( 0, new int[] { 2, 4, 3, 1 }, _queryGraph, indexes, _typesPerStream, new bool[5], null, new TableMetadata[_queryGraph.NumStreams]); Log.Debug(".testCreateStreamPlan plan=" + plan); Assert.IsTrue(plan is NestedIterationNode); var nested = (NestedIterationNode)plan; var tableLookupSpec = (TableLookupNode)nested.ChildNodes[0]; // Check lookup strategy for first lookup var lookupStrategySpec = (IndexedTableLookupPlanSingle)tableLookupSpec.LookupStrategySpec; Assert.AreEqual("P01", ((QueryGraphValueEntryHashKeyedProp)lookupStrategySpec.HashKey).KeyProperty); Assert.AreEqual(0, lookupStrategySpec.LookupStream); Assert.AreEqual(2, lookupStrategySpec.IndexedStream); Assert.NotNull(lookupStrategySpec.IndexNum); // Check lookup strategy for last lookup tableLookupSpec = (TableLookupNode)nested.ChildNodes[3]; var unkeyedSpecScan = (FullTableScanLookupPlan)tableLookupSpec.LookupStrategySpec; Assert.AreEqual(1, unkeyedSpecScan.IndexedStream); Assert.NotNull(unkeyedSpecScan.IndexNum); }
/// <summary> /// Build query plan. /// </summary> /// <param name="typesPerStream">event types for each stream</param> /// <param name="queryGraph">navigability info</param> /// <param name="optionalOuterJoinType">outer join type, null if not an outer join</param> /// <param name="uniqueIndexProps">The unique index props.</param> /// <param name="tablesPerStream">The tables per stream.</param> /// <returns> /// query plan /// </returns> public static QueryPlan Build(EventType[] typesPerStream, QueryGraph queryGraph, OuterJoinType?optionalOuterJoinType, string[][][] uniqueIndexProps, TableMetadata[] tablesPerStream) { var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec(queryGraph, typesPerStream, uniqueIndexProps); var execNodeSpecs = new QueryPlanNode[2]; var lookupPlans = new TableLookupPlan[2]; // plan lookup from 1 to zero lookupPlans[1] = NStreamQueryPlanBuilder.CreateLookupPlan(queryGraph, 1, 0, indexSpecs[0], typesPerStream, tablesPerStream[0]); // plan lookup from zero to 1 lookupPlans[0] = NStreamQueryPlanBuilder.CreateLookupPlan(queryGraph, 0, 1, indexSpecs[1], typesPerStream, tablesPerStream[1]); execNodeSpecs[0] = new TableLookupNode(lookupPlans[0]); execNodeSpecs[1] = new TableLookupNode(lookupPlans[1]); if (optionalOuterJoinType != null) { if ((optionalOuterJoinType.Equals(OuterJoinType.LEFT)) || (optionalOuterJoinType.Equals(OuterJoinType.FULL))) { execNodeSpecs[0] = new TableOuterLookupNode(lookupPlans[0]); } if ((optionalOuterJoinType.Equals(OuterJoinType.RIGHT)) || (optionalOuterJoinType.Equals(OuterJoinType.FULL))) { execNodeSpecs[1] = new TableOuterLookupNode(lookupPlans[1]); } } return(new QueryPlan(indexSpecs, execNodeSpecs)); }
private static QueryPlanNode BuildPlanNode( int numStreams, int streamNo, string[] streamNames, QueryGraph queryGraph, OuterInnerDirectionalGraph outerInnerGraph, OuterJoinDesc[] outerJoinDescList, InnerJoinGraph innerJoinGraph, QueryPlanIndex[] indexSpecs, EventType[] typesPerStream, bool[] ishistorical, DependencyGraph dependencyGraph, HistoricalStreamIndexList[] historicalStreamIndexLists, ExprEvaluatorContext exprEvaluatorContext, TableMetadata[] tablesPerStream) { // For each stream build an array of substreams, considering required streams (inner joins) first // The order is relevant therefore preserving order via a LinkedHashMap. var substreamsPerStream = new LinkedHashMap <int, int[]>(); var requiredPerStream = new bool[numStreams]; // Recursive populating the required (outer) and optional (inner) relationships // of this stream and the substream ISet <int> completedStreams = new HashSet <int>(); // keep track of tree path as only those stream events are always available to historical streams var streamCallStack = new Stack <int>(); streamCallStack.Push(streamNo); // For all inner-joins, the algorithm is slightly different if (innerJoinGraph.IsAllInnerJoin) { requiredPerStream.Fill(true); RecursiveBuildInnerJoin(streamNo, streamCallStack, queryGraph, completedStreams, substreamsPerStream, dependencyGraph); // compute a best chain to see if all streams are handled and add the remaining var bestChain = NStreamQueryPlanBuilder.ComputeBestPath(streamNo, queryGraph, dependencyGraph); AddNotYetNavigated(streamNo, numStreams, substreamsPerStream, bestChain); } else { RecursiveBuild(streamNo, streamCallStack, queryGraph, outerInnerGraph, innerJoinGraph, completedStreams, substreamsPerStream, requiredPerStream, dependencyGraph); } // verify the substreamsPerStream, all streams must exists and be linked VerifyJoinedPerStream(streamNo, substreamsPerStream); // build list of instructions for lookup var lookupInstructions = BuildLookupInstructions(streamNo, substreamsPerStream, requiredPerStream, streamNames, queryGraph, indexSpecs, typesPerStream, outerJoinDescList, ishistorical, historicalStreamIndexLists, exprEvaluatorContext, tablesPerStream); // build strategy tree for putting the result back together var assemblyTopNodeFactory = AssemblyStrategyTreeBuilder.Build(streamNo, substreamsPerStream, requiredPerStream); var assemblyInstructionFactories = BaseAssemblyNodeFactory.GetDescendentNodesBottomUp(assemblyTopNodeFactory); return(new LookupInstructionQueryPlanNode(streamNo, streamNames[streamNo], numStreams, requiredPerStream, lookupInstructions, assemblyInstructionFactories)); }
public void TestBuildDefaultNestingOrder() { int[] result = NStreamQueryPlanBuilder.BuildDefaultNestingOrder(4, 0); Assert.IsTrue(Collections.AreEqual(result, new int[] { 1, 2, 3 })); result = NStreamQueryPlanBuilder.BuildDefaultNestingOrder(4, 1); Assert.IsTrue(Collections.AreEqual(result, new int[] { 0, 2, 3 })); result = NStreamQueryPlanBuilder.BuildDefaultNestingOrder(4, 2); Assert.IsTrue(Collections.AreEqual(result, new int[] { 0, 1, 3 })); result = NStreamQueryPlanBuilder.BuildDefaultNestingOrder(4, 3); Assert.IsTrue(Collections.AreEqual(result, new int[] { 0, 1, 2 })); }
public void TestComputeBestPath() { NStreamQueryPlanBuilder.BestChainResult bestChain = NStreamQueryPlanBuilder.ComputeBestPath( 0, _queryGraph, _dependencyGraph); Assert.AreEqual(3, bestChain.Depth); Assert.IsTrue(Collections.AreEqual(bestChain.Chain, new int[] { 2, 4, 3, 1 })); bestChain = NStreamQueryPlanBuilder.ComputeBestPath(3, _queryGraph, _dependencyGraph); Assert.AreEqual(4, bestChain.Depth); Assert.IsTrue(Collections.AreEqual(bestChain.Chain, new int[] { 4, 2, 0, 1 })); // try a stream that is not connected in any way _queryGraph = new QueryGraph(6, null, false); bestChain = NStreamQueryPlanBuilder.ComputeBestPath(5, _queryGraph, _dependencyGraph); Log.Debug(".testComputeBestPath bestChain=" + bestChain); Assert.AreEqual(0, bestChain.Depth); Assert.IsTrue(Collections.AreEqual(bestChain.Chain, new int[] { 0, 1, 2, 3, 4 })); }
public void TestComputeNavigableDepth() { _queryGraph.AddStrictEquals(3, "P30", null, 2, "P20", null); _queryGraph.AddStrictEquals(2, "P30", null, 1, "P20", null); int depth = NStreamQueryPlanBuilder.ComputeNavigableDepth(0, new int[] { 1, 2, 3, 4 }, _queryGraph); Assert.AreEqual(4, depth); depth = NStreamQueryPlanBuilder.ComputeNavigableDepth(0, new int[] { 4, 2, 3, 1 }, _queryGraph); Assert.AreEqual(0, depth); depth = NStreamQueryPlanBuilder.ComputeNavigableDepth(4, new int[] { 3, 2, 1, 0 }, _queryGraph); Assert.AreEqual(4, depth); depth = NStreamQueryPlanBuilder.ComputeNavigableDepth(1, new int[] { 0, 3, 4, 2 }, _queryGraph); Assert.AreEqual(1, depth); }
public void TestIsDependencySatisfied() { var graph = new DependencyGraph(3, false); graph.AddDependency(1, 0); graph.AddDependency(2, 0); Assert.IsTrue(NStreamQueryPlanBuilder.IsDependencySatisfied(0, new int[] { 1, 2 }, graph)); Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(1, new int[] { 0, 2 }, graph)); Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(2, new int[] { 0, 1 }, graph)); graph = new DependencyGraph(5, false); graph.AddDependency(4, 1); graph.AddDependency(4, 2); graph.AddDependency(2, 0); Assert.IsTrue(NStreamQueryPlanBuilder.IsDependencySatisfied(0, new int[] { 1, 2, 3, 4 }, graph)); Assert.IsTrue(NStreamQueryPlanBuilder.IsDependencySatisfied(1, new int[] { 0, 2, 3, 4 }, graph)); Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(1, new int[] { 2, 0, 3, 4 }, graph)); Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(1, new int[] { 4, 0, 3, 2 }, graph)); Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(3, new int[] { 4, 0, 1, 2 }, graph)); Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(2, new int[] { 3, 1, 4, 0 }, graph)); Assert.IsTrue(NStreamQueryPlanBuilder.IsDependencySatisfied(3, new int[] { 1, 0, 2, 4 }, graph)); }
/// <summary> /// Build query plan using the filter. /// </summary> /// <param name="typesPerStream">event types for each stream</param> /// <param name="outerJoinDescList">list of outer join criteria, or null if there are no outer joins</param> /// <param name="queryGraph">relationships between streams based on filter expressions and outer-join on-criteria</param> /// <param name="streamNames">names of streams</param> /// <param name="historicalViewableDesc">The historical viewable desc.</param> /// <param name="dependencyGraph">dependencies between historical streams</param> /// <param name="historicalStreamIndexLists">index management, populated for the query plan</param> /// <param name="streamJoinAnalysisResult">The stream join analysis result.</param> /// <param name="isQueryPlanLogging">if set to <c>true</c> [is query plan logging].</param> /// <param name="annotations">The annotations.</param> /// <param name="exprEvaluatorContext">The expr evaluator context.</param> /// <returns> /// query plan /// </returns> /// <exception cref="System.ArgumentException"> /// Number of join stream types is less then 2 /// or /// Too many outer join descriptors found /// </exception> /// <throws>ExprValidationException if the query plan fails</throws> public static QueryPlan GetPlan(EventType[] typesPerStream, OuterJoinDesc[] outerJoinDescList, QueryGraph queryGraph, string[] streamNames, HistoricalViewableDesc historicalViewableDesc, DependencyGraph dependencyGraph, HistoricalStreamIndexList[] historicalStreamIndexLists, StreamJoinAnalysisResult streamJoinAnalysisResult, bool isQueryPlanLogging, Attribute[] annotations, ExprEvaluatorContext exprEvaluatorContext) { string methodName = ".getPlan "; int numStreams = typesPerStream.Length; if (numStreams < 2) { throw new ArgumentException("Number of join stream types is less then 2"); } if (outerJoinDescList.Length >= numStreams) { throw new ArgumentException("Too many outer join descriptors found"); } if (numStreams == 2) { OuterJoinType?outerJoinType = null; if (outerJoinDescList.Length > 0) { outerJoinType = outerJoinDescList[0].OuterJoinType; } QueryPlan queryPlanX = TwoStreamQueryPlanBuilder.Build(typesPerStream, queryGraph, outerJoinType, streamJoinAnalysisResult.UniqueKeys, streamJoinAnalysisResult.TablesPerStream); RemoveUnidirectionalAndTable(queryPlanX, streamJoinAnalysisResult); if (log.IsDebugEnabled) { log.Debug(methodName + "2-Stream queryPlan=" + queryPlanX); } return(queryPlanX); } bool hasPreferMergeJoin = HintEnum.PREFER_MERGE_JOIN.GetHint(annotations) != null; bool hasForceNestedIter = HintEnum.FORCE_NESTED_ITER.GetHint(annotations) != null; bool isAllInnerJoins = outerJoinDescList.Length == 0 || OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList); if (isAllInnerJoins && !hasPreferMergeJoin) { QueryPlan queryPlanX = NStreamQueryPlanBuilder.Build(queryGraph, typesPerStream, historicalViewableDesc, dependencyGraph, historicalStreamIndexLists, hasForceNestedIter, streamJoinAnalysisResult.UniqueKeys, streamJoinAnalysisResult.TablesPerStream); if (queryPlanX != null) { RemoveUnidirectionalAndTable(queryPlanX, streamJoinAnalysisResult); if (log.IsDebugEnabled) { log.Debug(methodName + "Count-Stream inner-join queryPlan=" + queryPlanX); } return(queryPlanX); } if (isQueryPlanLogging && queryPlanLog.IsInfoEnabled) { log.Info("Switching to Outer-NStream algorithm for query plan"); } } QueryPlan queryPlan = NStreamOuterQueryPlanBuilder.Build(queryGraph, outerJoinDescList, streamNames, typesPerStream, historicalViewableDesc, dependencyGraph, historicalStreamIndexLists, exprEvaluatorContext, streamJoinAnalysisResult.UniqueKeys, streamJoinAnalysisResult.TablesPerStream); RemoveUnidirectionalAndTable(queryPlan, streamJoinAnalysisResult); return(queryPlan); }
public void TestBuild() { QueryPlan plan = NStreamQueryPlanBuilder.Build(_queryGraph, _typesPerStream, new HistoricalViewableDesc(6), _dependencyGraph, null, false, new string[_queryGraph.NumStreams][][], new TableMetadata[_queryGraph.NumStreams]); Log.Debug(".testBuild plan=" + plan); }
private static IList <LookupInstructionPlan> BuildLookupInstructions( int rootStreamNum, LinkedHashMap <int, int[]> substreamsPerStream, bool[] requiredPerStream, string[] streamNames, QueryGraph queryGraph, QueryPlanIndex[] indexSpecs, EventType[] typesPerStream, OuterJoinDesc[] outerJoinDescList, bool[] isHistorical, HistoricalStreamIndexList[] historicalStreamIndexLists, ExprEvaluatorContext exprEvaluatorContext, TableMetadata[] tablesPerStream) { IList <LookupInstructionPlan> result = new List <LookupInstructionPlan>(); foreach (int fromStream in substreamsPerStream.Keys) { var substreams = substreamsPerStream.Get(fromStream); // for streams with no substreams we don't need to look up if (substreams.Length == 0) { continue; } var plans = new TableLookupPlan[substreams.Length]; var historicalPlans = new HistoricalDataPlanNode[substreams.Length]; for (var i = 0; i < substreams.Length; i++) { var toStream = substreams[i]; if (isHistorical[toStream]) { // There may not be an outer-join descriptor, use if provided to build the associated expression ExprNode outerJoinExpr = null; if (outerJoinDescList.Length > 0) { OuterJoinDesc outerJoinDesc; if (toStream == 0) { outerJoinDesc = outerJoinDescList[0]; } else { outerJoinDesc = outerJoinDescList[toStream - 1]; } outerJoinExpr = outerJoinDesc.MakeExprNode(exprEvaluatorContext); } if (historicalStreamIndexLists[toStream] == null) { historicalStreamIndexLists[toStream] = new HistoricalStreamIndexList(toStream, typesPerStream, queryGraph); } historicalStreamIndexLists[toStream].AddIndex(fromStream); historicalPlans[i] = new HistoricalDataPlanNode(toStream, rootStreamNum, fromStream, typesPerStream.Length, outerJoinExpr); } else { plans[i] = NStreamQueryPlanBuilder.CreateLookupPlan(queryGraph, fromStream, toStream, indexSpecs[toStream], typesPerStream, tablesPerStream[toStream]); } } var fromStreamName = streamNames[fromStream]; var instruction = new LookupInstructionPlan(fromStream, fromStreamName, substreams, plans, historicalPlans, requiredPerStream); result.Add(instruction); } return(result); }