/// <summary> /// Get the strategies to use for polling from a given stream. /// </summary> /// <param name="streamViewStreamNum">the stream providing the polling events</param> /// <returns>looking and indexing strategy</returns> public JoinSetComposerPrototypeHistoricalDesc GetStrategy( int streamViewStreamNum, StatementRawInfo raw, SerdeCompileTimeResolver serdeResolver) { // If there is only a single polling stream, then build a single index if (_pollingStreams.Count == 1) { return JoinSetComposerPrototypeForgeFactory.DetermineIndexing( _queryGraph, _typesPerStream[_historicalStreamNum], _typesPerStream[streamViewStreamNum], _historicalStreamNum, streamViewStreamNum, raw, serdeResolver); } // If there are multiple polling streams, determine if a single index is appropriate. // An index can be reused if: // (a) indexed property names are the same // (b) indexed property types are the same // (c) key property types are the same (because of coercion) // A index lookup strategy is always specific to the providing stream. var additionalForgeables = new List<StmtClassForgeableFactory>(); if (_indexesUsedByStreams == null) { _indexesUsedByStreams = new LinkedHashMap<HistoricalStreamIndexDesc, IList<int>>(); foreach (var pollingStream in _pollingStreams) { var queryGraphValue = _queryGraph.GetGraphValue(pollingStream, _historicalStreamNum); var hashKeyProps = queryGraphValue.HashKeyProps; var indexProperties = hashKeyProps.Indexed; var keyTypes = GetPropertyTypes(hashKeyProps.Keys); var indexTypes = GetPropertyTypes(_typesPerStream[_historicalStreamNum], indexProperties); var desc = new HistoricalStreamIndexDesc( indexProperties, indexTypes, keyTypes); var usedByStreams = _indexesUsedByStreams.Get(desc); if (usedByStreams == null) { usedByStreams = new List<int>(); _indexesUsedByStreams.Put(desc, usedByStreams); } usedByStreams.Add(pollingStream); } // There are multiple indexes required: // Build a master indexing strategy that forms multiple indexes and numbers each. if (_indexesUsedByStreams.Count > 1) { var numIndexes = _indexesUsedByStreams.Count; var indexingStrategies = new PollResultIndexingStrategyForge[numIndexes]; // create an indexing strategy for each index var count = 0; foreach (var desc in _indexesUsedByStreams) { var sampleStreamViewStreamNum = desc.Value[0]; var indexingX = JoinSetComposerPrototypeForgeFactory.DetermineIndexing( _queryGraph, _typesPerStream[_historicalStreamNum], _typesPerStream[sampleStreamViewStreamNum], _historicalStreamNum, sampleStreamViewStreamNum, raw, serdeResolver); indexingStrategies[count] = indexingX.IndexingForge; additionalForgeables.AddAll(indexingX.AdditionalForgeables); count++; } // create a master indexing strategy that utilizes each indexing strategy to create a set of indexes _masterIndexingStrategy = new PollResultIndexingStrategyMultiForge( streamViewStreamNum, indexingStrategies); } } // there is one type of index if (_indexesUsedByStreams.Count == 1) { return JoinSetComposerPrototypeForgeFactory.DetermineIndexing( _queryGraph, _typesPerStream[_historicalStreamNum], _typesPerStream[streamViewStreamNum], _historicalStreamNum, streamViewStreamNum, raw, serdeResolver); } // determine which index number the polling stream must use var indexUsed = 0; var found = false; foreach (var desc in _indexesUsedByStreams.Values) { if (desc.Contains(streamViewStreamNum)) { found = true; break; } indexUsed++; } if (!found) { throw new IllegalStateException("Index not found for use by stream " + streamViewStreamNum); } // Use one of the indexes built by the master index and a lookup strategy JoinSetComposerPrototypeHistoricalDesc indexing = JoinSetComposerPrototypeForgeFactory.DetermineIndexing( _queryGraph, _typesPerStream[_historicalStreamNum], _typesPerStream[streamViewStreamNum], _historicalStreamNum, streamViewStreamNum, raw, serdeResolver); HistoricalIndexLookupStrategyForge innerLookupStrategy = indexing.LookupForge; HistoricalIndexLookupStrategyForge lookupStrategy = new HistoricalIndexLookupStrategyMultiForge(indexUsed, innerLookupStrategy); additionalForgeables.AddAll(indexing.AdditionalForgeables); return new JoinSetComposerPrototypeHistoricalDesc(lookupStrategy, _masterIndexingStrategy, additionalForgeables); }
public static QueryPlanForgeDesc Build( QueryGraphForge queryGraph, EventType[] typesPerStream, HistoricalViewableDesc historicalViewableDesc, DependencyGraph dependencyGraph, HistoricalStreamIndexListForge[] historicalStreamIndexLists, bool hasForceNestedIter, string[][][] indexedStreamsUniqueProps, TableMetaData[] tablesPerStream, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, StatementRawInfo raw, SerdeCompileTimeResolver serdeResolver) { if (Log.IsDebugEnabled) { Log.Debug(".build filterQueryGraph=" + queryGraph); } var numStreams = queryGraph.NumStreams; var additionalForgeables = new List<StmtClassForgeableFactory>(); var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec( queryGraph, typesPerStream, indexedStreamsUniqueProps); if (Log.IsDebugEnabled) { Log.Debug(".build Index build completed, indexes=" + QueryPlanIndexForge.Print(indexSpecs)); } // any historical streams don't get indexes, the lookup strategy accounts for cached indexes if (historicalViewableDesc.IsHistorical) { for (var i = 0; i < historicalViewableDesc.Historical.Length; i++) { if (historicalViewableDesc.Historical[i]) { indexSpecs[i] = null; } } } var planNodeSpecs = new QueryPlanNodeForge[numStreams]; var worstDepth = int.MaxValue; for (var streamNo = 0; streamNo < numStreams; streamNo++) { // no plan for historical streams that are dependent upon other streams if (historicalViewableDesc.Historical[streamNo] && dependencyGraph.HasDependency(streamNo)) { planNodeSpecs[streamNo] = new QueryPlanNodeNoOpForge(); continue; } var bestChainResult = ComputeBestPath(streamNo, queryGraph, dependencyGraph); var bestChain = bestChainResult.Chain; if (Log.IsDebugEnabled) { Log.Debug(".build For stream " + streamNo + " bestChain=" + bestChain.RenderAny()); } if (bestChainResult.Depth < worstDepth) { worstDepth = bestChainResult.Depth; } var planDesc = CreateStreamPlan( streamNo, bestChain, queryGraph, indexSpecs, typesPerStream, historicalViewableDesc.Historical, historicalStreamIndexLists, tablesPerStream, streamJoinAnalysisResult, raw, serdeResolver); planNodeSpecs[streamNo] = planDesc.Forge; additionalForgeables.AddAll(planDesc.AdditionalForgeables); if (Log.IsDebugEnabled) { Log.Debug(".build spec=" + planNodeSpecs[streamNo]); } } // We use the merge/nested (outer) join algorithm instead. if (worstDepth < numStreams - 1 && !hasForceNestedIter) { return null; } // build historical index and lookup strategies for (var i = 0; i < numStreams; i++) { var plan = planNodeSpecs[i]; QueryPlanNodeForgeVisitor visitor = new ProxyQueryPlanNodeForgeVisitor { ProcVisit = node => { if (node is HistoricalDataPlanNodeForge) { var historical = (HistoricalDataPlanNodeForge) node; JoinSetComposerPrototypeHistoricalDesc desc = historicalStreamIndexLists[historical.StreamNum].GetStrategy( historical.LookupStreamNum, raw, serdeResolver); historical.PollResultIndexingStrategy = desc.IndexingForge; historical.HistoricalIndexLookupStrategy = desc.LookupForge; additionalForgeables.AddAll(desc.AdditionalForgeables); } } }; plan.Accept(visitor); } var forge = new QueryPlanForge(indexSpecs, planNodeSpecs); return new QueryPlanForgeDesc(forge, additionalForgeables); }
private static QueryPlanNodeForgeDesc BuildPlanNode( int numStreams, int streamNo, string[] streamNames, QueryGraphForge queryGraph, OuterInnerDirectionalGraph outerInnerGraph, OuterJoinDesc[] outerJoinDescList, InnerJoinGraph innerJoinGraph, QueryPlanIndexForge[] indexSpecs, EventType[] typesPerStream, bool[] isHistorical, DependencyGraph dependencyGraph, HistoricalStreamIndexListForge[] historicalStreamIndexLists, TableMetaData[] tablesPerStream, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, StatementRawInfo statementRawInfo, StatementCompileTimeServices services) { // For each stream build an array of substreams, considering required streams (inner joins) first // The order is relevant therefore preserving order via a LinkedHashMap. var substreamsPerStream = new LinkedHashMap<int, int[]>(); var requiredPerStream = new bool[numStreams]; var additionalForgeables = new List<StmtClassForgeableFactory>(); // Recursive populating the required (outer) and optional (inner) relationships // of this stream and the substream ISet<int> completedStreams = new HashSet<int>(); // keep track of tree path as only those stream events are always available to historical streams var streamCallStack = new Stack<int>(); streamCallStack.Push(streamNo); // For all inner-joins, the algorithm is slightly different if (innerJoinGraph.IsAllInnerJoin) { requiredPerStream.Fill(true); RecursiveBuildInnerJoin( streamNo, streamCallStack, queryGraph, completedStreams, substreamsPerStream, dependencyGraph); // compute a best chain to see if all streams are handled and add the remaining var bestChain = NStreamQueryPlanBuilder.ComputeBestPath(streamNo, queryGraph, dependencyGraph); AddNotYetNavigated(streamNo, numStreams, substreamsPerStream, bestChain); } else { RecursiveBuild( streamNo, streamCallStack, queryGraph, outerInnerGraph, innerJoinGraph, completedStreams, substreamsPerStream, requiredPerStream, dependencyGraph); } // verify the substreamsPerStream, all streams must exists and be linked VerifyJoinedPerStream(streamNo, substreamsPerStream); // build list of instructions for lookup LookupInstructionPlanDesc lookupDesc = BuildLookupInstructions( streamNo, substreamsPerStream, requiredPerStream, streamNames, queryGraph, indexSpecs, typesPerStream, outerJoinDescList, isHistorical, historicalStreamIndexLists, tablesPerStream, streamJoinAnalysisResult, statementRawInfo, services); var lookupInstructions = lookupDesc.Forges; additionalForgeables.AddAll(lookupDesc.AdditionalForgeables); // build historical index and lookup strategies foreach (var lookups in lookupInstructions) { foreach (var historical in lookups.HistoricalPlans) { if (historical == null) { continue; } JoinSetComposerPrototypeHistoricalDesc desc = historicalStreamIndexLists[historical.StreamNum] .GetStrategy(historical.LookupStreamNum, statementRawInfo, services.SerdeResolver); historical.HistoricalIndexLookupStrategy = desc.LookupForge; historical.PollResultIndexingStrategy = desc.IndexingForge; additionalForgeables.AddAll(desc.AdditionalForgeables); } } // build strategy tree for putting the result back together var assemblyTopNodeFactory = AssemblyStrategyTreeBuilder.Build( streamNo, substreamsPerStream, requiredPerStream); var assemblyInstructionFactories = BaseAssemblyNodeFactory.GetDescendentNodesBottomUp(assemblyTopNodeFactory); var forge = new LookupInstructionQueryPlanNodeForge( streamNo, streamNames[streamNo], numStreams, requiredPerStream, lookupInstructions, assemblyInstructionFactories); return new QueryPlanNodeForgeDesc(forge, additionalForgeables); }