/// <summary> /// Walks the chain of lookups and constructs lookup strategy and plan specification based /// on the index specifications. /// </summary> /// <param name="lookupStream">the stream to construct the query plan for</param> /// <param name="bestChain">the chain that the lookup follows to make best use of indexes</param> /// <param name="queryGraph">the repository for key properties to indexes</param> /// <param name="indexSpecsPerStream">specifications of indexes</param> /// <param name="typesPerStream">event types for each stream</param> /// <param name="isHistorical">indicator for each stream if it is a historical streams or not</param> /// <param name="historicalStreamIndexLists">index management, populated for the query plan</param> /// <returns>NestedIterationNode with lookups attached underneath</returns> internal static QueryPlanNode CreateStreamPlan( int lookupStream, int[] bestChain, QueryGraph queryGraph, QueryPlanIndex[] indexSpecsPerStream, EventType[] typesPerStream, bool[] isHistorical, HistoricalStreamIndexList[] historicalStreamIndexLists, TableMetadata[] tablesPerStream) { var nestedIterNode = new NestedIterationNode(bestChain); var currentLookupStream = lookupStream; // Walk through each successive lookup for (var i = 0; i < bestChain.Length; i++) { var indexedStream = bestChain[i]; QueryPlanNode node; if (isHistorical[indexedStream]) { if (historicalStreamIndexLists[indexedStream] == null) { historicalStreamIndexLists[indexedStream] = new HistoricalStreamIndexList(indexedStream, typesPerStream, queryGraph); } historicalStreamIndexLists[indexedStream].AddIndex(currentLookupStream); node = new HistoricalDataPlanNode(indexedStream, lookupStream, currentLookupStream, typesPerStream.Length, null); } else { var tableLookupPlan = CreateLookupPlan(queryGraph, currentLookupStream, indexedStream, indexSpecsPerStream[indexedStream], typesPerStream, tablesPerStream[indexedStream]); node = new TableLookupNode(tableLookupPlan); } nestedIterNode.AddChildNode(node); currentLookupStream = bestChain[i]; } return(nestedIterNode); }
/// <summary> /// Builds join tuple composer. /// </summary> /// <param name="statementName">Name of the statement.</param> /// <param name="statementId">The statement identifier.</param> /// <param name="outerJoinDescList">list of descriptors for outer join criteria</param> /// <param name="optionalFilterNode">filter tree for analysis to build indexes for fast access</param> /// <param name="streamTypes">types of streams</param> /// <param name="streamNames">names of streams</param> /// <param name="streamJoinAnalysisResult">The stream join analysis result.</param> /// <param name="queryPlanLogging">if set to <c>true</c> [query plan logging].</param> /// <param name="statementContext">The statement context.</param> /// <param name="historicalViewableDesc">The historical viewable desc.</param> /// <param name="exprEvaluatorContext">The expr evaluator context.</param> /// <param name="selectsRemoveStream">if set to <c>true</c> [selects remove stream].</param> /// <param name="hasAggregations">if set to <c>true</c> [has aggregations].</param> /// <param name="tableService">The table service.</param> /// <param name="isOnDemandQuery">if set to <c>true</c> [is on demand query].</param> /// <param name="allowIndexInit">if set to <c>true</c> [allow index initialize].</param> /// <returns> /// composer implementation /// </returns> /// <throws>com.espertech.esper.epl.expression.core.ExprValidationException is thrown to indicate thatvalidation of view use in joins failed. /// {D255958A-8513-4226-94B9-080D98F904A1}</throws> public static JoinSetComposerPrototype MakeComposerPrototype(string statementName, int statementId, OuterJoinDesc[] outerJoinDescList, ExprNode optionalFilterNode, EventType[] streamTypes, string[] streamNames, StreamJoinAnalysisResult streamJoinAnalysisResult, bool queryPlanLogging, StatementContext statementContext, HistoricalViewableDesc historicalViewableDesc, ExprEvaluatorContext exprEvaluatorContext, bool selectsRemoveStream, bool hasAggregations, TableService tableService, bool isOnDemandQuery, bool allowIndexInit) { // Determine if there is a historical stream, and what dependencies exist var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false); for (var i = 0; i < streamTypes.Length; i++) { if (historicalViewableDesc.Historical[i]) { var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i]; historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn); } } if (log.IsDebugEnabled) { log.Debug("Dependency graph: " + historicalDependencyGraph); } // Handle a join with a database or other historical data source for 2 streams if ((historicalViewableDesc.HasHistorical) && (streamTypes.Length == 2)) { return(MakeComposerHistorical2Stream(outerJoinDescList, optionalFilterNode, streamTypes, historicalViewableDesc, queryPlanLogging, exprEvaluatorContext, statementContext, streamNames, allowIndexInit)); } var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList); // Query graph for graph relationships between streams/historicals // For outer joins the query graph will just contain outer join relationships var hint = ExcludePlanHint.GetHint(streamNames, statementContext); var queryGraph = new QueryGraph(streamTypes.Length, hint, false); if (outerJoinDescList.Length > 0) { OuterJoinAnalyzer.Analyze(outerJoinDescList, queryGraph); if (log.IsDebugEnabled) { log.Debug(".makeComposer After outer join queryGraph=\n" + queryGraph); } } // Let the query graph reflect the where-clause if (optionalFilterNode != null) { // Analyze relationships between streams using the optional filter expression. // Relationships are properties in AND and EQUALS nodes of joins. FilterExprAnalyzer.Analyze(optionalFilterNode, queryGraph, isOuterJoins); if (log.IsDebugEnabled) { log.Debug(".makeComposer After filter expression queryGraph=\n" + queryGraph); } // Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c) QueryGraph.FillEquivalentNav(streamTypes, queryGraph); if (log.IsDebugEnabled) { log.Debug(".makeComposer After fill equiv. nav. queryGraph=\n" + queryGraph); } } // Historical index lists var historicalStreamIndexLists = new HistoricalStreamIndexList[streamTypes.Length]; var queryPlan = QueryPlanBuilder.GetPlan(streamTypes, outerJoinDescList, queryGraph, streamNames, historicalViewableDesc, historicalDependencyGraph, historicalStreamIndexLists, streamJoinAnalysisResult, queryPlanLogging, statementContext.Annotations, exprEvaluatorContext); // remove unused indexes - consider all streams or all unidirectional var usedIndexes = new HashSet <TableLookupIndexReqKey>(); var indexSpecs = queryPlan.IndexSpecs; for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) { var planNode = queryPlan.ExecNodeSpecs[streamNum]; if (planNode != null) { planNode.AddIndexes(usedIndexes); } } foreach (var indexSpec in indexSpecs) { if (indexSpec == null) { continue; } var items = indexSpec.Items; var indexNames = items.Keys.ToArray(); foreach (var indexName in indexNames) { if (!usedIndexes.Contains(indexName)) { items.Remove(indexName); } } } var hook = QueryPlanIndexHookUtil.GetHook(statementContext.Annotations); if (queryPlanLogging && (QueryPlanLog.IsInfoEnabled || hook != null)) { QueryPlanLog.Info("Query plan: " + queryPlan.ToQueryPlan()); if (hook != null) { hook.Join(queryPlan); } } // register index-use references for tables if (!isOnDemandQuery) { foreach (var usedIndex in usedIndexes) { if (usedIndex.TableName != null) { tableService.GetTableMetadata(usedIndex.TableName).AddIndexReference(usedIndex.Name, statementName); } } } var joinRemoveStream = selectsRemoveStream || hasAggregations; return(new JoinSetComposerPrototypeImpl( statementName, statementId, outerJoinDescList, optionalFilterNode, streamTypes, streamNames, streamJoinAnalysisResult, statementContext.Annotations, historicalViewableDesc, exprEvaluatorContext, indexSpecs, queryPlan, historicalStreamIndexLists, joinRemoveStream, isOuterJoins, tableService, statementContext.EventTableIndexService)); }
private static IList <LookupInstructionPlan> BuildLookupInstructions( int rootStreamNum, LinkedHashMap <int, int[]> substreamsPerStream, bool[] requiredPerStream, string[] streamNames, QueryGraph queryGraph, QueryPlanIndex[] indexSpecs, EventType[] typesPerStream, OuterJoinDesc[] outerJoinDescList, bool[] isHistorical, HistoricalStreamIndexList[] historicalStreamIndexLists, ExprEvaluatorContext exprEvaluatorContext, TableMetadata[] tablesPerStream) { IList <LookupInstructionPlan> result = new List <LookupInstructionPlan>(); foreach (int fromStream in substreamsPerStream.Keys) { var substreams = substreamsPerStream.Get(fromStream); // for streams with no substreams we don't need to look up if (substreams.Length == 0) { continue; } var plans = new TableLookupPlan[substreams.Length]; var historicalPlans = new HistoricalDataPlanNode[substreams.Length]; for (var i = 0; i < substreams.Length; i++) { var toStream = substreams[i]; if (isHistorical[toStream]) { // There may not be an outer-join descriptor, use if provided to build the associated expression ExprNode outerJoinExpr = null; if (outerJoinDescList.Length > 0) { OuterJoinDesc outerJoinDesc; if (toStream == 0) { outerJoinDesc = outerJoinDescList[0]; } else { outerJoinDesc = outerJoinDescList[toStream - 1]; } outerJoinExpr = outerJoinDesc.MakeExprNode(exprEvaluatorContext); } if (historicalStreamIndexLists[toStream] == null) { historicalStreamIndexLists[toStream] = new HistoricalStreamIndexList(toStream, typesPerStream, queryGraph); } historicalStreamIndexLists[toStream].AddIndex(fromStream); historicalPlans[i] = new HistoricalDataPlanNode(toStream, rootStreamNum, fromStream, typesPerStream.Length, outerJoinExpr); } else { plans[i] = NStreamQueryPlanBuilder.CreateLookupPlan(queryGraph, fromStream, toStream, indexSpecs[toStream], typesPerStream, tablesPerStream[toStream]); } } var fromStreamName = streamNames[fromStream]; var instruction = new LookupInstructionPlan(fromStream, fromStreamName, substreams, plans, historicalPlans, requiredPerStream); result.Add(instruction); } return(result); }