/// <summary> /// Walks the chain of lookups and constructs lookup strategy and plan specification based /// on the index specifications. /// </summary> /// <param name="lookupStream">the stream to construct the query plan for</param> /// <param name="bestChain">the chain that the lookup follows to make best use of indexes</param> /// <param name="queryGraph">the repository for key properties to indexes</param> /// <param name="indexSpecsPerStream">specifications of indexes</param> /// <param name="typesPerStream">event types for each stream</param> /// <param name="isHistorical">indicator for each stream if it is a historical streams or not</param> /// <param name="historicalStreamIndexLists">index management, populated for the query plan</param> /// <param name="tablesPerStream">tables</param> /// <param name="streamJoinAnalysisResult">stream join analysis</param> /// <param name="raw">raw statement information</param> /// <param name="serdeResolver">serde resolver</param> /// <returns>NestedIterationNode with lookups attached underneath</returns> public static QueryPlanNodeForgeDesc CreateStreamPlan( int lookupStream, int[] bestChain, QueryGraphForge queryGraph, QueryPlanIndexForge[] indexSpecsPerStream, EventType[] typesPerStream, bool[] isHistorical, HistoricalStreamIndexListForge[] historicalStreamIndexLists, TableMetaData[] tablesPerStream, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, StatementRawInfo raw, SerdeCompileTimeResolver serdeResolver) { var nestedIterNode = new NestedIterationNodeForge(bestChain); var currentLookupStream = lookupStream; var additionalForgeables = new List<StmtClassForgeableFactory>(); // Walk through each successive lookup for (var i = 0; i < bestChain.Length; i++) { var indexedStream = bestChain[i]; QueryPlanNodeForge node; if (isHistorical[indexedStream]) { if (historicalStreamIndexLists[indexedStream] == null) { historicalStreamIndexLists[indexedStream] = new HistoricalStreamIndexListForge( indexedStream, typesPerStream, queryGraph); } historicalStreamIndexLists[indexedStream].AddIndex(currentLookupStream); node = new HistoricalDataPlanNodeForge( indexedStream, lookupStream, currentLookupStream, typesPerStream.Length, null); } else { var tableLookupPlan = CreateLookupPlan( queryGraph, currentLookupStream, indexedStream, streamJoinAnalysisResult.IsVirtualDW(indexedStream), indexSpecsPerStream[indexedStream], typesPerStream, tablesPerStream[indexedStream], raw, serdeResolver); node = new TableLookupNodeForge(tableLookupPlan.Forge); additionalForgeables.AddAll(tableLookupPlan.AdditionalForgeables); } nestedIterNode.AddChildNode(node); currentLookupStream = bestChain[i]; } return new QueryPlanNodeForgeDesc(nestedIterNode, additionalForgeables); }
private static LookupInstructionPlanDesc BuildLookupInstructions( int rootStreamNum, IDictionary<int, int[]> substreamsPerStream, bool[] requiredPerStream, string[] streamNames, QueryGraphForge queryGraph, QueryPlanIndexForge[] indexSpecs, EventType[] typesPerStream, OuterJoinDesc[] outerJoinDescList, bool[] isHistorical, HistoricalStreamIndexListForge[] historicalStreamIndexLists, TableMetaData[] tablesPerStream, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, StatementRawInfo statementRawInfo, StatementCompileTimeServices services) { var result = new List<LookupInstructionPlanForge>(); var additionalForgeables = new List<StmtClassForgeableFactory>(); foreach (var fromStream in substreamsPerStream.Keys) { var substreams = substreamsPerStream.Get(fromStream); // for streams with no substreams we don't need to look up if (substreams.Length == 0) { continue; } var plans = new TableLookupPlanForge[substreams.Length]; var historicalPlans = new HistoricalDataPlanNodeForge[substreams.Length]; for (var i = 0; i < substreams.Length; i++) { var toStream = substreams[i]; if (isHistorical[toStream]) { // There may not be an outer-join descriptor, use if provided to build the associated expression ExprNode outerJoinExpr = null; if (outerJoinDescList.Length > 0) { OuterJoinDesc outerJoinDesc; if (toStream == 0) { outerJoinDesc = outerJoinDescList[0]; } else { outerJoinDesc = outerJoinDescList[toStream - 1]; } outerJoinExpr = outerJoinDesc.MakeExprNode(statementRawInfo, services); } if (historicalStreamIndexLists[toStream] == null) { historicalStreamIndexLists[toStream] = new HistoricalStreamIndexListForge( toStream, typesPerStream, queryGraph); } historicalStreamIndexLists[toStream].AddIndex(fromStream); historicalPlans[i] = new HistoricalDataPlanNodeForge( toStream, rootStreamNum, fromStream, typesPerStream.Length, outerJoinExpr == null ? null : outerJoinExpr.Forge); } else { TableLookupPlanDesc planDesc = NStreamQueryPlanBuilder.CreateLookupPlan( queryGraph, fromStream, toStream, streamJoinAnalysisResult.IsVirtualDW(toStream), indexSpecs[toStream], typesPerStream, tablesPerStream[toStream], statementRawInfo, SerdeCompileTimeResolverNonHA.INSTANCE); plans[i] = planDesc.Forge; additionalForgeables.AddAll(planDesc.AdditionalForgeables); } } var fromStreamName = streamNames[fromStream]; var instruction = new LookupInstructionPlanForge( fromStream, fromStreamName, substreams, plans, historicalPlans, requiredPerStream); result.Add(instruction); } return new LookupInstructionPlanDesc(result, additionalForgeables); }
public static JoinSetComposerPrototypeDesc MakeComposerPrototype( StatementSpecCompiled spec, StreamJoinAnalysisResultCompileTime joinAnalysisResult, StreamTypeService typeService, HistoricalViewableDesc historicalViewableDesc, bool isOnDemandQuery, bool hasAggregations, StatementRawInfo statementRawInfo, StatementCompileTimeServices compileTimeServices) { var streamTypes = typeService.EventTypes; var streamNames = typeService.StreamNames; var whereClause = spec.Raw.WhereClause; var queryPlanLogging = compileTimeServices.Configuration.Common.Logging.IsEnableQueryPlan; var additionalForgeables = new List<StmtClassForgeableFactory>(); // Determine if there is a historical stream, and what dependencies exist var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false); for (var i = 0; i < streamTypes.Length; i++) { if (historicalViewableDesc.Historical[i]) { var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i]; historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn); } } if (Log.IsDebugEnabled) { Log.Debug("Dependency graph: " + historicalDependencyGraph); } // Handle a join with a database or other historical data source for 2 streams var outerJoinDescs = OuterJoinDesc.ToArray(spec.Raw.OuterJoinDescList); if (historicalViewableDesc.IsHistorical && streamTypes.Length == 2) { var desc = MakeComposerHistorical2Stream( outerJoinDescs, whereClause, streamTypes, streamNames, historicalViewableDesc, queryPlanLogging, statementRawInfo, compileTimeServices); return new JoinSetComposerPrototypeDesc(desc.Forge, desc.AdditionalForgeables); } var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescs); // Query graph for graph relationships between streams/historicals // For outer joins the query graph will just contain outer join relationships var hint = ExcludePlanHint.GetHint( typeService.StreamNames, statementRawInfo, compileTimeServices); var queryGraph = new QueryGraphForge(streamTypes.Length, hint, false); if (outerJoinDescs.Length > 0) { OuterJoinAnalyzer.Analyze(outerJoinDescs, queryGraph); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After outer join filterQueryGraph=\n" + queryGraph); } } // Let the query graph reflect the where-clause if (whereClause != null) { // Analyze relationships between streams using the optional filter expression. // Relationships are properties in AND and EQUALS nodes of joins. FilterExprAnalyzer.Analyze(whereClause, queryGraph, isOuterJoins); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After filter expression filterQueryGraph=\n" + queryGraph); } // Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c) QueryGraphForge.FillEquivalentNav(streamTypes, queryGraph); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After fill equiv. nav. filterQueryGraph=\n" + queryGraph); } } // Historical index lists var historicalStreamIndexLists = new HistoricalStreamIndexListForge[streamTypes.Length]; var queryPlanDesc = QueryPlanBuilder.GetPlan( streamTypes, outerJoinDescs, queryGraph, typeService.StreamNames, historicalViewableDesc, historicalDependencyGraph, historicalStreamIndexLists, joinAnalysisResult, queryPlanLogging, statementRawInfo, compileTimeServices); QueryPlanForge queryPlan = queryPlanDesc.Forge; additionalForgeables.AddAll(queryPlanDesc.AdditionalForgeables); // remove unused indexes - consider all streams or all unidirectional var usedIndexes = new HashSet<TableLookupIndexReqKey>(); var indexSpecs = queryPlan.IndexSpecs; for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) { var planNode = queryPlan.ExecNodeSpecs[streamNum]; planNode?.AddIndexes(usedIndexes); } foreach (var indexSpec in indexSpecs) { if (indexSpec == null) { continue; } var items = indexSpec.Items; var indexNames = items.Keys.ToArray(); foreach (var indexName in indexNames) { if (!usedIndexes.Contains(indexName)) { items.Remove(indexName); } } } // plan multikeys IList<StmtClassForgeableFactory> multikeyForgeables = PlanMultikeys( indexSpecs, statementRawInfo, compileTimeServices); additionalForgeables.AddAll(multikeyForgeables); QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook( spec.Annotations, compileTimeServices.ImportServiceCompileTime); if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) { QUERY_PLAN_LOG.Info("Query plan: " + queryPlan.ToQueryPlan()); hook?.Join(queryPlan); } var selectsRemoveStream = spec.Raw.SelectStreamSelectorEnum.IsSelectsRStream() || spec.Raw.OutputLimitSpec != null; var joinRemoveStream = selectsRemoveStream || hasAggregations; ExprNode postJoinEvaluator; if (JoinSetComposerUtil.IsNonUnidirectionalNonSelf( isOuterJoins, joinAnalysisResult.IsUnidirectional, joinAnalysisResult.IsPureSelfJoin)) { postJoinEvaluator = GetFilterExpressionInclOnClause( spec.Raw.WhereClause, outerJoinDescs, statementRawInfo, compileTimeServices); } else { postJoinEvaluator = spec.Raw.WhereClause; } JoinSetComposerPrototypeGeneralForge forge = new JoinSetComposerPrototypeGeneralForge( typeService.EventTypes, postJoinEvaluator, outerJoinDescs.Length > 0, queryPlan, joinAnalysisResult, typeService.StreamNames, joinRemoveStream, historicalViewableDesc.IsHistorical); return new JoinSetComposerPrototypeDesc(forge, additionalForgeables); }