private static ISet<int> GetInnerStreams( int fromStream, ISet<int> toStreams, OuterInnerDirectionalGraph outerInnerGraph, InnerJoinGraph innerJoinGraph, ISet<int> completedStreams) { ISet<int> innerStreams = new HashSet<int>(); foreach (var toStream in toStreams) { if (outerInnerGraph.IsInner(fromStream, toStream)) { // if the to-stream, recursively, has an inner join itself, it becomes a required stream and not optional var hasInnerJoin = false; if (!innerJoinGraph.IsEmpty()) { var doNotUseStreams = new HashSet<int>(completedStreams); completedStreams.Add(fromStream); hasInnerJoin = RecursiveHasInnerJoin( toStream, outerInnerGraph, innerJoinGraph, doNotUseStreams); } if (!hasInnerJoin) { innerStreams.Add(toStream); } } } return innerStreams; }
private static bool RecursiveHasInnerJoin( int toStream, OuterInnerDirectionalGraph outerInnerGraph, InnerJoinGraph innerJoinGraph, ISet<int> completedStreams) { // Check if the to-stream is in any of the inner joins var hasInnerJoin = innerJoinGraph.HasInnerJoin(toStream); if (hasInnerJoin) { return true; } var innerToToStream = outerInnerGraph.GetInner(toStream); if (innerToToStream != null) { foreach (var nextStream in innerToToStream) { if (completedStreams.Contains(nextStream)) { continue; } var notConsider = new HashSet<int>(completedStreams); notConsider.Add(toStream); var result = RecursiveHasInnerJoin(nextStream, outerInnerGraph, innerJoinGraph, notConsider); if (result) { return true; } } } var outerToToStream = outerInnerGraph.GetOuter(toStream); if (outerToToStream != null) { foreach (var nextStream in outerToToStream) { if (completedStreams.Contains(nextStream)) { continue; } var notConsider = new HashSet<int>(completedStreams); notConsider.Add(toStream); var result = RecursiveHasInnerJoin(nextStream, outerInnerGraph, innerJoinGraph, notConsider); if (result) { return true; } } } return false; }
public void TestRecursiveBuild() { var streamNum = 2; var queryGraph = new QueryGraphForge(6, null, false); var outerInnerGraph = new OuterInnerDirectionalGraph(6); var completedStreams = new HashSet <int>(); var substreamsPerStream = new LinkedHashMap <int, int[]>(); var requiredPerStream = new bool[6]; outerInnerGraph.Add(3, 2).Add(2, 1).Add(4, 3).Add(1, 0).Add(3, 5); var fake = supportExprNodeFactory.MakeIdentNode("TheString", "s0"); queryGraph.AddStrictEquals(2, "", fake, 3, "", fake); queryGraph.AddStrictEquals(3, "", fake, 4, "", fake); queryGraph.AddStrictEquals(3, "", fake, 5, "", fake); queryGraph.AddStrictEquals(2, "", fake, 1, "", fake); queryGraph.AddStrictEquals(1, "", fake, 0, "", fake); ISet <InterchangeablePair <int, int> > innerJoins = new HashSet <InterchangeablePair <int, int> >(); var innerJoinGraph = new InnerJoinGraph(6, innerJoins); var streamStack = new Stack <int>(); NStreamOuterQueryPlanBuilder.RecursiveBuild( streamNum, streamStack, queryGraph, outerInnerGraph, innerJoinGraph, completedStreams, substreamsPerStream, requiredPerStream, new DependencyGraph(6, false)); Assert.AreEqual(6, substreamsPerStream.Count); EPAssertionUtil.AssertEqualsExactOrder(substreamsPerStream.Get(2), new[] { 3, 1 }); EPAssertionUtil.AssertEqualsExactOrder(substreamsPerStream.Get(3), new[] { 4, 5 }); EPAssertionUtil.AssertEqualsExactOrder(substreamsPerStream.Get(1), new[] { 0 }); EPAssertionUtil.AssertEqualsExactOrder(substreamsPerStream.Get(4), new int[] { }); EPAssertionUtil.AssertEqualsExactOrder(substreamsPerStream.Get(5), new int[] { }); EPAssertionUtil.AssertEqualsExactOrder(substreamsPerStream.Get(0), new int[] { }); NStreamOuterQueryPlanBuilder.VerifyJoinedPerStream(2, substreamsPerStream); EPAssertionUtil.AssertEqualsExactOrder( requiredPerStream, new[] { false, false, false, true, true, false } ); }
/// <summary> /// Recusivly builds a substream-per-stream ordered tree graph using the /// join information supplied for outer joins and from the query graph (where clause). /// <para /> /// Required streams are considered first and their lookup is placed first in the list /// to gain performance. /// </summary> /// <param name="streamNum">is the root stream number that supplies the incoming event to build the tree for</param> /// <param name="queryGraph">contains where-clause stream relationship info</param> /// <param name="outerInnerGraph">contains the outer join stream relationship info</param> /// <param name="completedStreams">is a temporary holder for streams already considered</param> /// <param name="substreamsPerStream">is the ordered, tree-like structure to be filled</param> /// <param name="requiredPerStream">indicates which streams are required and which are optional</param> /// <param name="streamCallStack">the query plan call stack of streams available via cursor</param> /// <param name="dependencyGraph">dependencies between historical streams</param> /// <param name="innerJoinGraph">inner join graph</param> /// <throws>ExprValidationException if the query planning failed</throws> public static void RecursiveBuild( int streamNum, Stack<int> streamCallStack, QueryGraphForge queryGraph, OuterInnerDirectionalGraph outerInnerGraph, InnerJoinGraph innerJoinGraph, ISet<int> completedStreams, IDictionary<int, int[]> substreamsPerStream, bool[] requiredPerStream, DependencyGraph dependencyGraph ) { // add this stream to the set of completed streams completedStreams.Add(streamNum); // check if the dependencies have been satisfied if (dependencyGraph.HasDependency(streamNum)) { var dependencies = dependencyGraph.GetDependenciesForStream(streamNum); foreach (var dependentStream in dependencies) { if (!streamCallStack.Contains(dependentStream)) { throw new ExprValidationException( "Historical stream " + streamNum + " parameter dependency originating in stream " + dependentStream + " cannot or may not be satisfied by the join"); } } } // Determine the streams we can navigate to from this stream var navigableStreams = queryGraph.GetNavigableStreams(streamNum); // add unqualified navigable streams (since on-expressions in outer joins are optional) var unqualifiedNavigable = outerInnerGraph.UnqualifiedNavigableStreams.Get(streamNum); if (unqualifiedNavigable != null) { navigableStreams.AddAll(unqualifiedNavigable); } // remove those already done navigableStreams.RemoveAll(completedStreams); // Which streams are inner streams to this stream (optional), which ones are outer to the stream (required) var requiredStreams = GetOuterStreams(streamNum, navigableStreams, outerInnerGraph); // Add inner joins, if any, unless already completed for this stream innerJoinGraph.AddRequiredStreams(streamNum, requiredStreams, completedStreams); var optionalStreams = GetInnerStreams( streamNum, navigableStreams, outerInnerGraph, innerJoinGraph, completedStreams); // Remove from the required streams the optional streams which places 'full' joined streams // into the optional stream category requiredStreams.RemoveAll(optionalStreams); // if we are a leaf node, we are done if (navigableStreams.IsEmpty()) { substreamsPerStream.Put(streamNum, new int[0]); return; } // First the outer (required) streams to this stream, then the inner (optional) streams var substreams = new int[requiredStreams.Count + optionalStreams.Count]; substreamsPerStream.Put(streamNum, substreams); var count = 0; foreach (var stream in requiredStreams) { substreams[count++] = stream; requiredPerStream[stream] = true; } foreach (var stream in optionalStreams) { substreams[count++] = stream; } // next we look at all the required streams and add their dependent streams foreach (var stream in requiredStreams) { completedStreams.Add(stream); } foreach (var stream in requiredStreams) { streamCallStack.Push(stream); RecursiveBuild( stream, streamCallStack, queryGraph, outerInnerGraph, innerJoinGraph, completedStreams, substreamsPerStream, requiredPerStream, dependencyGraph); streamCallStack.Pop(); } // look at all the optional streams and add their dependent streams foreach (var stream in optionalStreams) { streamCallStack.Push(stream); RecursiveBuild( stream, streamCallStack, queryGraph, outerInnerGraph, innerJoinGraph, completedStreams, substreamsPerStream, requiredPerStream, dependencyGraph); streamCallStack.Pop(); } }
internal static QueryPlanForgeDesc Build( QueryGraphForge queryGraph, OuterJoinDesc[] outerJoinDescList, string[] streamNames, EventType[] typesPerStream, HistoricalViewableDesc historicalViewableDesc, DependencyGraph dependencyGraph, HistoricalStreamIndexListForge[] historicalStreamIndexLists, string[][][] indexedStreamsUniqueProps, TableMetaData[] tablesPerStream, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, StatementRawInfo statementRawInfo, StatementCompileTimeServices services) { if (Log.IsDebugEnabled) { Log.Debug(".build filterQueryGraph=" + queryGraph); } var numStreams = queryGraph.NumStreams; var planNodeSpecs = new QueryPlanNodeForge[numStreams]; var additionalForgeables = new List<StmtClassForgeableFactory>(); // Build index specifications var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec( queryGraph, typesPerStream, indexedStreamsUniqueProps); // any historical streams don't get indexes, the lookup strategy accounts for cached indexes if (historicalViewableDesc.IsHistorical) { for (var i = 0; i < historicalViewableDesc.Historical.Length; i++) { if (historicalViewableDesc.Historical[i]) { indexSpecs[i] = null; } } } // Build graph of the outer join to inner table relationships. // Build a map of inner joins. OuterInnerDirectionalGraph outerInnerGraph; InnerJoinGraph innerJoinGraph; if (outerJoinDescList.Length > 0) { outerInnerGraph = GraphOuterJoins(numStreams, outerJoinDescList); innerJoinGraph = InnerJoinGraph.GraphInnerJoins(numStreams, outerJoinDescList); } else { // all inner joins - thereby no (or empty) directional graph outerInnerGraph = new OuterInnerDirectionalGraph(numStreams); innerJoinGraph = new InnerJoinGraph(numStreams, true); } if (Log.IsDebugEnabled) { Log.Debug(".build directional graph=" + outerInnerGraph.Print()); } // For each stream determine the query plan for (var streamNo = 0; streamNo < numStreams; streamNo++) { // no plan for historical streams that are dependent upon other streams if (historicalViewableDesc.Historical[streamNo] && dependencyGraph.HasDependency(streamNo)) { planNodeSpecs[streamNo] = new QueryPlanNodeNoOpForge(); continue; } QueryPlanNodeForgeDesc desc = BuildPlanNode( numStreams, streamNo, streamNames, queryGraph, outerInnerGraph, outerJoinDescList, innerJoinGraph, indexSpecs, typesPerStream, historicalViewableDesc.Historical, dependencyGraph, historicalStreamIndexLists, tablesPerStream, streamJoinAnalysisResult, statementRawInfo, services); QueryPlanNodeForge queryPlanNode = desc.Forge; additionalForgeables.AddAll(desc.AdditionalForgeables); if (Log.IsDebugEnabled) { Log.Debug( ".build spec for stream '" + streamNames[streamNo] + "' number " + streamNo + " is " + queryPlanNode); } planNodeSpecs[streamNo] = queryPlanNode; } var queryPlan = new QueryPlanForge(indexSpecs, planNodeSpecs); if (Log.IsDebugEnabled) { Log.Debug(".build query plan=" + queryPlan); } return new QueryPlanForgeDesc(queryPlan, additionalForgeables); }
private static QueryPlanNodeForgeDesc BuildPlanNode( int numStreams, int streamNo, string[] streamNames, QueryGraphForge queryGraph, OuterInnerDirectionalGraph outerInnerGraph, OuterJoinDesc[] outerJoinDescList, InnerJoinGraph innerJoinGraph, QueryPlanIndexForge[] indexSpecs, EventType[] typesPerStream, bool[] isHistorical, DependencyGraph dependencyGraph, HistoricalStreamIndexListForge[] historicalStreamIndexLists, TableMetaData[] tablesPerStream, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, StatementRawInfo statementRawInfo, StatementCompileTimeServices services) { // For each stream build an array of substreams, considering required streams (inner joins) first // The order is relevant therefore preserving order via a LinkedHashMap. var substreamsPerStream = new LinkedHashMap<int, int[]>(); var requiredPerStream = new bool[numStreams]; var additionalForgeables = new List<StmtClassForgeableFactory>(); // Recursive populating the required (outer) and optional (inner) relationships // of this stream and the substream ISet<int> completedStreams = new HashSet<int>(); // keep track of tree path as only those stream events are always available to historical streams var streamCallStack = new Stack<int>(); streamCallStack.Push(streamNo); // For all inner-joins, the algorithm is slightly different if (innerJoinGraph.IsAllInnerJoin) { requiredPerStream.Fill(true); RecursiveBuildInnerJoin( streamNo, streamCallStack, queryGraph, completedStreams, substreamsPerStream, dependencyGraph); // compute a best chain to see if all streams are handled and add the remaining var bestChain = NStreamQueryPlanBuilder.ComputeBestPath(streamNo, queryGraph, dependencyGraph); AddNotYetNavigated(streamNo, numStreams, substreamsPerStream, bestChain); } else { RecursiveBuild( streamNo, streamCallStack, queryGraph, outerInnerGraph, innerJoinGraph, completedStreams, substreamsPerStream, requiredPerStream, dependencyGraph); } // verify the substreamsPerStream, all streams must exists and be linked VerifyJoinedPerStream(streamNo, substreamsPerStream); // build list of instructions for lookup LookupInstructionPlanDesc lookupDesc = BuildLookupInstructions( streamNo, substreamsPerStream, requiredPerStream, streamNames, queryGraph, indexSpecs, typesPerStream, outerJoinDescList, isHistorical, historicalStreamIndexLists, tablesPerStream, streamJoinAnalysisResult, statementRawInfo, services); var lookupInstructions = lookupDesc.Forges; additionalForgeables.AddAll(lookupDesc.AdditionalForgeables); // build historical index and lookup strategies foreach (var lookups in lookupInstructions) { foreach (var historical in lookups.HistoricalPlans) { if (historical == null) { continue; } JoinSetComposerPrototypeHistoricalDesc desc = historicalStreamIndexLists[historical.StreamNum] .GetStrategy(historical.LookupStreamNum, statementRawInfo, services.SerdeResolver); historical.HistoricalIndexLookupStrategy = desc.LookupForge; historical.PollResultIndexingStrategy = desc.IndexingForge; additionalForgeables.AddAll(desc.AdditionalForgeables); } } // build strategy tree for putting the result back together var assemblyTopNodeFactory = AssemblyStrategyTreeBuilder.Build( streamNo, substreamsPerStream, requiredPerStream); var assemblyInstructionFactories = BaseAssemblyNodeFactory.GetDescendentNodesBottomUp(assemblyTopNodeFactory); var forge = new LookupInstructionQueryPlanNodeForge( streamNo, streamNames[streamNo], numStreams, requiredPerStream, lookupInstructions, assemblyInstructionFactories); return new QueryPlanNodeForgeDesc(forge, additionalForgeables); }