/// <summary> /// Walks the chain of lookups and constructs lookup strategy and plan specification based /// on the index specifications. /// </summary> /// <param name="lookupStream">the stream to construct the query plan for</param> /// <param name="bestChain">the chain that the lookup follows to make best use of indexes</param> /// <param name="queryGraph">the repository for key properties to indexes</param> /// <param name="indexSpecsPerStream">specifications of indexes</param> /// <param name="typesPerStream">event types for each stream</param> /// <param name="isHistorical">indicator for each stream if it is a historical streams or not</param> /// <param name="historicalStreamIndexLists">index management, populated for the query plan</param> /// <param name="tablesPerStream">tables</param> /// <param name="streamJoinAnalysisResult">stream join analysis</param> /// <param name="raw">raw statement information</param> /// <param name="serdeResolver">serde resolver</param> /// <returns>NestedIterationNode with lookups attached underneath</returns> public static QueryPlanNodeForgeDesc CreateStreamPlan( int lookupStream, int[] bestChain, QueryGraphForge queryGraph, QueryPlanIndexForge[] indexSpecsPerStream, EventType[] typesPerStream, bool[] isHistorical, HistoricalStreamIndexListForge[] historicalStreamIndexLists, TableMetaData[] tablesPerStream, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, StatementRawInfo raw, SerdeCompileTimeResolver serdeResolver) { var nestedIterNode = new NestedIterationNodeForge(bestChain); var currentLookupStream = lookupStream; var additionalForgeables = new List<StmtClassForgeableFactory>(); // Walk through each successive lookup for (var i = 0; i < bestChain.Length; i++) { var indexedStream = bestChain[i]; QueryPlanNodeForge node; if (isHistorical[indexedStream]) { if (historicalStreamIndexLists[indexedStream] == null) { historicalStreamIndexLists[indexedStream] = new HistoricalStreamIndexListForge( indexedStream, typesPerStream, queryGraph); } historicalStreamIndexLists[indexedStream].AddIndex(currentLookupStream); node = new HistoricalDataPlanNodeForge( indexedStream, lookupStream, currentLookupStream, typesPerStream.Length, null); } else { var tableLookupPlan = CreateLookupPlan( queryGraph, currentLookupStream, indexedStream, streamJoinAnalysisResult.IsVirtualDW(indexedStream), indexSpecsPerStream[indexedStream], typesPerStream, tablesPerStream[indexedStream], raw, serdeResolver); node = new TableLookupNodeForge(tableLookupPlan.Forge); additionalForgeables.AddAll(tableLookupPlan.AdditionalForgeables); } nestedIterNode.AddChildNode(node); currentLookupStream = bestChain[i]; } return new QueryPlanNodeForgeDesc(nestedIterNode, additionalForgeables); }
public JoinSetComposerPrototypeGeneralForge( EventType[] streamTypes, ExprNode postJoinEvaluator, bool outerJoins, QueryPlanForge queryPlan, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, string[] streamNames, bool joinRemoveStream, bool hasHistorical) : base(streamTypes, postJoinEvaluator, outerJoins) { this.queryPlan = queryPlan; this.streamJoinAnalysisResult = streamJoinAnalysisResult; this.streamNames = streamNames; this.joinRemoveStream = joinRemoveStream; this.hasHistorical = hasHistorical; }
// Remove plans for non-unidirectional streams private static void RemoveUnidirectionalAndTable( QueryPlanForge queryPlan, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult) { bool allUnidirectional = streamJoinAnalysisResult.IsUnidirectionalAll; for (int streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) { if (allUnidirectional) { queryPlan.ExecNodeSpecs[streamNum] = new QueryPlanNodeForgeAllUnidirectionalOuter(streamNum); } else { bool unidirectional = streamJoinAnalysisResult.IsUnidirectional && !streamJoinAnalysisResult.UnidirectionalInd[streamNum]; bool table = streamJoinAnalysisResult.TablesPerStream[streamNum] != null; if (unidirectional || table) { queryPlan.ExecNodeSpecs[streamNum] = QueryPlanNodeNoOpForge.INSTANCE; } } } }
public static QueryPlanForgeDesc Build( QueryGraphForge queryGraph, EventType[] typesPerStream, HistoricalViewableDesc historicalViewableDesc, DependencyGraph dependencyGraph, HistoricalStreamIndexListForge[] historicalStreamIndexLists, bool hasForceNestedIter, string[][][] indexedStreamsUniqueProps, TableMetaData[] tablesPerStream, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, StatementRawInfo raw, SerdeCompileTimeResolver serdeResolver) { if (Log.IsDebugEnabled) { Log.Debug(".build filterQueryGraph=" + queryGraph); } var numStreams = queryGraph.NumStreams; var additionalForgeables = new List<StmtClassForgeableFactory>(); var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec( queryGraph, typesPerStream, indexedStreamsUniqueProps); if (Log.IsDebugEnabled) { Log.Debug(".build Index build completed, indexes=" + QueryPlanIndexForge.Print(indexSpecs)); } // any historical streams don't get indexes, the lookup strategy accounts for cached indexes if (historicalViewableDesc.IsHistorical) { for (var i = 0; i < historicalViewableDesc.Historical.Length; i++) { if (historicalViewableDesc.Historical[i]) { indexSpecs[i] = null; } } } var planNodeSpecs = new QueryPlanNodeForge[numStreams]; var worstDepth = int.MaxValue; for (var streamNo = 0; streamNo < numStreams; streamNo++) { // no plan for historical streams that are dependent upon other streams if (historicalViewableDesc.Historical[streamNo] && dependencyGraph.HasDependency(streamNo)) { planNodeSpecs[streamNo] = new QueryPlanNodeNoOpForge(); continue; } var bestChainResult = ComputeBestPath(streamNo, queryGraph, dependencyGraph); var bestChain = bestChainResult.Chain; if (Log.IsDebugEnabled) { Log.Debug(".build For stream " + streamNo + " bestChain=" + bestChain.RenderAny()); } if (bestChainResult.Depth < worstDepth) { worstDepth = bestChainResult.Depth; } var planDesc = CreateStreamPlan( streamNo, bestChain, queryGraph, indexSpecs, typesPerStream, historicalViewableDesc.Historical, historicalStreamIndexLists, tablesPerStream, streamJoinAnalysisResult, raw, serdeResolver); planNodeSpecs[streamNo] = planDesc.Forge; additionalForgeables.AddAll(planDesc.AdditionalForgeables); if (Log.IsDebugEnabled) { Log.Debug(".build spec=" + planNodeSpecs[streamNo]); } } // We use the merge/nested (outer) join algorithm instead. if (worstDepth < numStreams - 1 && !hasForceNestedIter) { return null; } // build historical index and lookup strategies for (var i = 0; i < numStreams; i++) { var plan = planNodeSpecs[i]; QueryPlanNodeForgeVisitor visitor = new ProxyQueryPlanNodeForgeVisitor { ProcVisit = node => { if (node is HistoricalDataPlanNodeForge) { var historical = (HistoricalDataPlanNodeForge) node; JoinSetComposerPrototypeHistoricalDesc desc = historicalStreamIndexLists[historical.StreamNum].GetStrategy( historical.LookupStreamNum, raw, serdeResolver); historical.PollResultIndexingStrategy = desc.IndexingForge; historical.HistoricalIndexLookupStrategy = desc.LookupForge; additionalForgeables.AddAll(desc.AdditionalForgeables); } } }; plan.Accept(visitor); } var forge = new QueryPlanForge(indexSpecs, planNodeSpecs); return new QueryPlanForgeDesc(forge, additionalForgeables); }
/// <summary> /// Build query plan. /// </summary> /// <param name="queryGraph">navigability info</param> /// <param name="optionalOuterJoinType">outer join type, null if not an outer join</param> /// <param name="typesPerStream">event types for each stream</param> /// <param name="streamJoinAnalysisResult">stream join analysis</param> /// <returns>query plan</returns> public static QueryPlanForgeDesc Build( EventType[] typesPerStream, QueryGraphForge queryGraph, OuterJoinType? optionalOuterJoinType, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, StatementRawInfo raw) { var uniqueIndexProps = streamJoinAnalysisResult.UniqueKeys; var tablesPerStream = streamJoinAnalysisResult.TablesPerStream; var additionalForgeable = new List<StmtClassForgeableFactory>(); var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec( queryGraph, typesPerStream, uniqueIndexProps); var execNodeSpecs = new QueryPlanNodeForge[2]; var lookupPlans = new TableLookupPlanForge[2]; // plan lookup from 1 to zero TableLookupPlanDesc plan1to0 = NStreamQueryPlanBuilder.CreateLookupPlan( queryGraph, 1, 0, streamJoinAnalysisResult.IsVirtualDW(0), indexSpecs[0], typesPerStream, tablesPerStream[0], raw, SerdeCompileTimeResolverNonHA.INSTANCE); lookupPlans[1] = plan1to0.Forge; additionalForgeable.AddAll(plan1to0.AdditionalForgeables); // plan lookup from zero to 1 TableLookupPlanDesc plan0to1 = NStreamQueryPlanBuilder.CreateLookupPlan( queryGraph, 0, 1, streamJoinAnalysisResult.IsVirtualDW(1), indexSpecs[1], typesPerStream, tablesPerStream[1], raw, SerdeCompileTimeResolverNonHA.INSTANCE); lookupPlans[0] = plan0to1.Forge; additionalForgeable.AddAll(plan0to1.AdditionalForgeables); execNodeSpecs[0] = new TableLookupNodeForge(lookupPlans[0]); execNodeSpecs[1] = new TableLookupNodeForge(lookupPlans[1]); if (optionalOuterJoinType != null) { if ((optionalOuterJoinType.Equals(OuterJoinType.LEFT)) || (optionalOuterJoinType.Equals(OuterJoinType.FULL))) { execNodeSpecs[0] = new TableOuterLookupNodeForge(lookupPlans[0]); } if ((optionalOuterJoinType.Equals(OuterJoinType.RIGHT)) || (optionalOuterJoinType.Equals(OuterJoinType.FULL))) { execNodeSpecs[1] = new TableOuterLookupNodeForge(lookupPlans[1]); } } var forge = new QueryPlanForge(indexSpecs, execNodeSpecs); return new QueryPlanForgeDesc(forge, additionalForgeable); }
public static QueryPlanForgeDesc GetPlan( EventType[] typesPerStream, OuterJoinDesc[] outerJoinDescList, QueryGraphForge queryGraph, string[] streamNames, HistoricalViewableDesc historicalViewableDesc, DependencyGraph dependencyGraph, HistoricalStreamIndexListForge[] historicalStreamIndexLists, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, bool isQueryPlanLogging, StatementRawInfo statementRawInfo, StatementCompileTimeServices services) { string methodName = ".getPlan "; int numStreams = typesPerStream.Length; if (numStreams < 2) { throw new ArgumentException("Number of join stream types is less then 2"); } if (outerJoinDescList.Length >= numStreams) { throw new ArgumentException("Too many outer join descriptors found"); } if (numStreams == 2) { OuterJoinType? outerJoinType = null; if (outerJoinDescList.Length > 0) { outerJoinType = outerJoinDescList[0].OuterJoinType; } QueryPlanForgeDesc queryPlan = TwoStreamQueryPlanBuilder.Build( typesPerStream, queryGraph, outerJoinType, streamJoinAnalysisResult, statementRawInfo); RemoveUnidirectionalAndTable(queryPlan.Forge, streamJoinAnalysisResult); if (Log.IsDebugEnabled) { Log.Debug(methodName + "2-Stream queryPlan=" + queryPlan); } return queryPlan; } bool hasPreferMergeJoin = HintEnum.PREFER_MERGE_JOIN.GetHint(statementRawInfo.Annotations) != null; bool hasForceNestedIter = HintEnum.FORCE_NESTED_ITER.GetHint(statementRawInfo.Annotations) != null; bool isAllInnerJoins = outerJoinDescList.Length == 0 || OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList); if (isAllInnerJoins && !hasPreferMergeJoin) { QueryPlanForgeDesc queryPlan = NStreamQueryPlanBuilder.Build( queryGraph, typesPerStream, historicalViewableDesc, dependencyGraph, historicalStreamIndexLists, hasForceNestedIter, streamJoinAnalysisResult.UniqueKeys, streamJoinAnalysisResult.TablesPerStream, streamJoinAnalysisResult, statementRawInfo, services.SerdeResolver); if (queryPlan != null) { RemoveUnidirectionalAndTable(queryPlan.Forge, streamJoinAnalysisResult); if (Log.IsDebugEnabled) { Log.Debug(methodName + "N-Stream inner-join queryPlan=" + queryPlan); } return queryPlan; } if (isQueryPlanLogging && QUERY_PLAN_LOG.IsInfoEnabled) { Log.Info("Switching to Outer-NStream algorithm for query plan"); } } QueryPlanForgeDesc queryPlanX = NStreamOuterQueryPlanBuilder.Build( queryGraph, outerJoinDescList, streamNames, typesPerStream, historicalViewableDesc, dependencyGraph, historicalStreamIndexLists, streamJoinAnalysisResult.UniqueKeys, streamJoinAnalysisResult.TablesPerStream, streamJoinAnalysisResult, statementRawInfo, services); RemoveUnidirectionalAndTable(queryPlanX.Forge, streamJoinAnalysisResult); return queryPlanX; }
private StmtForgeMethodResult Build( string @namespace, string classPostfix, StatementCompileTimeServices services) { var compileResult = CreateWindowUtil.HandleCreateWindow(@base, services); var namedWindowType = compileResult.FilterSpecCompiled.FilterForEventType; // view must be non-empty list var createWindowDesc = @base.StatementSpec.Raw.CreateWindowDesc; if (createWindowDesc.ViewSpecs.IsEmpty()) { throw new ExprValidationException(NamedWindowManagementServiceConstants.ERROR_MSG_DATAWINDOWS); } if (services.NamedWindowCompileTimeResolver.Resolve(createWindowDesc.WindowName) != null) { throw new ExprValidationException( "Named window named '" + createWindowDesc.WindowName + "' has already been declared"); } // build forge var activator = new ViewableActivatorFilterForge(compileResult.FilterSpecCompiled, false, 0, false, -1); var viewSpecs = createWindowDesc.ViewSpecs; var viewArgs = new ViewFactoryForgeArgs( 0, false, -1, createWindowDesc.StreamSpecOptions, createWindowDesc.WindowName, @base.StatementRawInfo, services); var viewForges = ViewFactoryForgeUtil.CreateForges(viewSpecs.ToArray(), viewArgs, namedWindowType); IList<ScheduleHandleCallbackProvider> schedules = new List<ScheduleHandleCallbackProvider>(); ViewFactoryForgeUtil.DetermineViewSchedules(viewForges, schedules); VerifyDataWindowViewFactoryChain(viewForges); var optionalUniqueKeyProps = StreamJoinAnalysisResultCompileTime.GetUniqueCandidateProperties( viewForges, @base.StatementSpec.Annotations); var uniqueKeyProArray = optionalUniqueKeyProps == null ? null : optionalUniqueKeyProps.ToArray(); NamedWindowMetaData insertFromNamedWindow = null; ExprNode insertFromFilter = null; if (createWindowDesc.IsInsert || createWindowDesc.InsertFilter != null) { var name = createWindowDesc.AsEventTypeName; insertFromNamedWindow = services.NamedWindowCompileTimeResolver.Resolve(name); if (insertFromNamedWindow == null) { throw new ExprValidationException( "A named window by name '" + name + "' could not be located, the insert-keyword requires an existing named window"); } insertFromFilter = createWindowDesc.InsertFilter; if (insertFromFilter != null) { var checkMinimal = ExprNodeUtilityValidate.IsMinimalExpression(insertFromFilter); if (checkMinimal != null) { throw new ExprValidationException("Create window where-clause may not have " + checkMinimal); } StreamTypeService streamTypeService = new StreamTypeServiceImpl( insertFromNamedWindow.EventType, name, true); var validationContext = new ExprValidationContextBuilder( streamTypeService, @base.StatementRawInfo, services).Build(); insertFromFilter = ExprNodeUtilityValidate.GetValidatedSubtree( ExprNodeOrigin.CREATEWINDOWFILTER, insertFromFilter, validationContext); } } // handle output format var defaultSelectAllSpec = new StatementSpecCompiled(); defaultSelectAllSpec.SelectClauseCompiled.WithSelectExprList(new SelectClauseElementWildcard()); defaultSelectAllSpec.Raw.SelectStreamDirEnum = SelectClauseStreamSelectorEnum.RSTREAM_ISTREAM_BOTH; StreamTypeService typeService = new StreamTypeServiceImpl( new[] {namedWindowType}, new[] {createWindowDesc.WindowName}, new[] {true}, false, false); var resultSetProcessor = ResultSetProcessorFactoryFactory.GetProcessorPrototype( new ResultSetSpec(defaultSelectAllSpec), typeService, null, new bool[1], false, @base.ContextPropertyRegistry, false, false, @base.StatementRawInfo, services); var classNameRSP = CodeGenerationIDGenerator.GenerateClassNameSimple( typeof(ResultSetProcessorFactoryProvider), classPostfix); var selectSubscriberDescriptor = resultSetProcessor.SelectSubscriberDescriptor; var forge = new StatementAgentInstanceFactoryCreateNWForge( activator, createWindowDesc.WindowName, viewForges, insertFromNamedWindow, insertFromFilter, compileResult.AsEventType, classNameRSP); // add named window var isBatchingDataWindow = DetermineBatchingDataWindow(viewForges); var virtualDataWindow = viewForges[0] is VirtualDWViewFactoryForge; var isEnableIndexShare = virtualDataWindow || HintEnum.ENABLE_WINDOW_SUBQUERY_INDEXSHARE.GetHint( @base.StatementSpec.Annotations) != null; var metaData = new NamedWindowMetaData( namedWindowType, @base.ModuleName, @base.ContextName, uniqueKeyProArray, isBatchingDataWindow, isEnableIndexShare, compileResult.AsEventType, virtualDataWindow); services.NamedWindowCompileTimeRegistry.NewNamedWindow(metaData); // build forge list IList<StmtClassForgable> forgables = new List<StmtClassForgable>(2); var statementFieldsClassName = CodeGenerationIDGenerator.GenerateClassNameSimple(typeof(StatementFields), classPostfix); var packageScope = new CodegenNamespaceScope( @namespace, statementFieldsClassName, services.IsInstrumented); forgables.Add( new StmtClassForgableRSPFactoryProvider( classNameRSP, resultSetProcessor, packageScope, @base.StatementRawInfo)); var aiFactoryProviderClassName = CodeGenerationIDGenerator.GenerateClassNameSimple( typeof(StatementAIFactoryProvider), classPostfix); var aiFactoryForgable = new StmtClassForgableAIFactoryProviderCreateNW( aiFactoryProviderClassName, packageScope, forge, createWindowDesc.WindowName); forgables.Add(aiFactoryForgable); var statementProviderClassName = CodeGenerationIDGenerator.GenerateClassNameSimple(typeof(StatementProvider), classPostfix); var informationals = StatementInformationalsUtil.GetInformationals( @base, Collections.SingletonList(compileResult.FilterSpecCompiled), schedules, new EmptyList<NamedWindowConsumerStreamSpec>(), true, selectSubscriberDescriptor, packageScope, services); forgables.Add( new StmtClassForgableStmtProvider( aiFactoryProviderClassName, statementProviderClassName, informationals, packageScope)); forgables.Add(new StmtClassForgableStmtFields(statementFieldsClassName, packageScope, 1)); return new StmtForgeMethodResult( forgables, Collections.SingletonList(compileResult.FilterSpecCompiled), schedules, new EmptyList<NamedWindowConsumerStreamSpec>(), new EmptyList<FilterSpecParamExprNodeForge>()); }
internal static QueryPlanForgeDesc Build( QueryGraphForge queryGraph, OuterJoinDesc[] outerJoinDescList, string[] streamNames, EventType[] typesPerStream, HistoricalViewableDesc historicalViewableDesc, DependencyGraph dependencyGraph, HistoricalStreamIndexListForge[] historicalStreamIndexLists, string[][][] indexedStreamsUniqueProps, TableMetaData[] tablesPerStream, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, StatementRawInfo statementRawInfo, StatementCompileTimeServices services) { if (Log.IsDebugEnabled) { Log.Debug(".build filterQueryGraph=" + queryGraph); } var numStreams = queryGraph.NumStreams; var planNodeSpecs = new QueryPlanNodeForge[numStreams]; var additionalForgeables = new List<StmtClassForgeableFactory>(); // Build index specifications var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec( queryGraph, typesPerStream, indexedStreamsUniqueProps); // any historical streams don't get indexes, the lookup strategy accounts for cached indexes if (historicalViewableDesc.IsHistorical) { for (var i = 0; i < historicalViewableDesc.Historical.Length; i++) { if (historicalViewableDesc.Historical[i]) { indexSpecs[i] = null; } } } // Build graph of the outer join to inner table relationships. // Build a map of inner joins. OuterInnerDirectionalGraph outerInnerGraph; InnerJoinGraph innerJoinGraph; if (outerJoinDescList.Length > 0) { outerInnerGraph = GraphOuterJoins(numStreams, outerJoinDescList); innerJoinGraph = InnerJoinGraph.GraphInnerJoins(numStreams, outerJoinDescList); } else { // all inner joins - thereby no (or empty) directional graph outerInnerGraph = new OuterInnerDirectionalGraph(numStreams); innerJoinGraph = new InnerJoinGraph(numStreams, true); } if (Log.IsDebugEnabled) { Log.Debug(".build directional graph=" + outerInnerGraph.Print()); } // For each stream determine the query plan for (var streamNo = 0; streamNo < numStreams; streamNo++) { // no plan for historical streams that are dependent upon other streams if (historicalViewableDesc.Historical[streamNo] && dependencyGraph.HasDependency(streamNo)) { planNodeSpecs[streamNo] = new QueryPlanNodeNoOpForge(); continue; } QueryPlanNodeForgeDesc desc = BuildPlanNode( numStreams, streamNo, streamNames, queryGraph, outerInnerGraph, outerJoinDescList, innerJoinGraph, indexSpecs, typesPerStream, historicalViewableDesc.Historical, dependencyGraph, historicalStreamIndexLists, tablesPerStream, streamJoinAnalysisResult, statementRawInfo, services); QueryPlanNodeForge queryPlanNode = desc.Forge; additionalForgeables.AddAll(desc.AdditionalForgeables); if (Log.IsDebugEnabled) { Log.Debug( ".build spec for stream '" + streamNames[streamNo] + "' number " + streamNo + " is " + queryPlanNode); } planNodeSpecs[streamNo] = queryPlanNode; } var queryPlan = new QueryPlanForge(indexSpecs, planNodeSpecs); if (Log.IsDebugEnabled) { Log.Debug(".build query plan=" + queryPlan); } return new QueryPlanForgeDesc(queryPlan, additionalForgeables); }
private static LookupInstructionPlanDesc BuildLookupInstructions( int rootStreamNum, IDictionary<int, int[]> substreamsPerStream, bool[] requiredPerStream, string[] streamNames, QueryGraphForge queryGraph, QueryPlanIndexForge[] indexSpecs, EventType[] typesPerStream, OuterJoinDesc[] outerJoinDescList, bool[] isHistorical, HistoricalStreamIndexListForge[] historicalStreamIndexLists, TableMetaData[] tablesPerStream, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, StatementRawInfo statementRawInfo, StatementCompileTimeServices services) { var result = new List<LookupInstructionPlanForge>(); var additionalForgeables = new List<StmtClassForgeableFactory>(); foreach (var fromStream in substreamsPerStream.Keys) { var substreams = substreamsPerStream.Get(fromStream); // for streams with no substreams we don't need to look up if (substreams.Length == 0) { continue; } var plans = new TableLookupPlanForge[substreams.Length]; var historicalPlans = new HistoricalDataPlanNodeForge[substreams.Length]; for (var i = 0; i < substreams.Length; i++) { var toStream = substreams[i]; if (isHistorical[toStream]) { // There may not be an outer-join descriptor, use if provided to build the associated expression ExprNode outerJoinExpr = null; if (outerJoinDescList.Length > 0) { OuterJoinDesc outerJoinDesc; if (toStream == 0) { outerJoinDesc = outerJoinDescList[0]; } else { outerJoinDesc = outerJoinDescList[toStream - 1]; } outerJoinExpr = outerJoinDesc.MakeExprNode(statementRawInfo, services); } if (historicalStreamIndexLists[toStream] == null) { historicalStreamIndexLists[toStream] = new HistoricalStreamIndexListForge( toStream, typesPerStream, queryGraph); } historicalStreamIndexLists[toStream].AddIndex(fromStream); historicalPlans[i] = new HistoricalDataPlanNodeForge( toStream, rootStreamNum, fromStream, typesPerStream.Length, outerJoinExpr == null ? null : outerJoinExpr.Forge); } else { TableLookupPlanDesc planDesc = NStreamQueryPlanBuilder.CreateLookupPlan( queryGraph, fromStream, toStream, streamJoinAnalysisResult.IsVirtualDW(toStream), indexSpecs[toStream], typesPerStream, tablesPerStream[toStream], statementRawInfo, SerdeCompileTimeResolverNonHA.INSTANCE); plans[i] = planDesc.Forge; additionalForgeables.AddAll(planDesc.AdditionalForgeables); } } var fromStreamName = streamNames[fromStream]; var instruction = new LookupInstructionPlanForge( fromStream, fromStreamName, substreams, plans, historicalPlans, requiredPerStream); result.Add(instruction); } return new LookupInstructionPlanDesc(result, additionalForgeables); }
private static QueryPlanNodeForgeDesc BuildPlanNode( int numStreams, int streamNo, string[] streamNames, QueryGraphForge queryGraph, OuterInnerDirectionalGraph outerInnerGraph, OuterJoinDesc[] outerJoinDescList, InnerJoinGraph innerJoinGraph, QueryPlanIndexForge[] indexSpecs, EventType[] typesPerStream, bool[] isHistorical, DependencyGraph dependencyGraph, HistoricalStreamIndexListForge[] historicalStreamIndexLists, TableMetaData[] tablesPerStream, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, StatementRawInfo statementRawInfo, StatementCompileTimeServices services) { // For each stream build an array of substreams, considering required streams (inner joins) first // The order is relevant therefore preserving order via a LinkedHashMap. var substreamsPerStream = new LinkedHashMap<int, int[]>(); var requiredPerStream = new bool[numStreams]; var additionalForgeables = new List<StmtClassForgeableFactory>(); // Recursive populating the required (outer) and optional (inner) relationships // of this stream and the substream ISet<int> completedStreams = new HashSet<int>(); // keep track of tree path as only those stream events are always available to historical streams var streamCallStack = new Stack<int>(); streamCallStack.Push(streamNo); // For all inner-joins, the algorithm is slightly different if (innerJoinGraph.IsAllInnerJoin) { requiredPerStream.Fill(true); RecursiveBuildInnerJoin( streamNo, streamCallStack, queryGraph, completedStreams, substreamsPerStream, dependencyGraph); // compute a best chain to see if all streams are handled and add the remaining var bestChain = NStreamQueryPlanBuilder.ComputeBestPath(streamNo, queryGraph, dependencyGraph); AddNotYetNavigated(streamNo, numStreams, substreamsPerStream, bestChain); } else { RecursiveBuild( streamNo, streamCallStack, queryGraph, outerInnerGraph, innerJoinGraph, completedStreams, substreamsPerStream, requiredPerStream, dependencyGraph); } // verify the substreamsPerStream, all streams must exists and be linked VerifyJoinedPerStream(streamNo, substreamsPerStream); // build list of instructions for lookup LookupInstructionPlanDesc lookupDesc = BuildLookupInstructions( streamNo, substreamsPerStream, requiredPerStream, streamNames, queryGraph, indexSpecs, typesPerStream, outerJoinDescList, isHistorical, historicalStreamIndexLists, tablesPerStream, streamJoinAnalysisResult, statementRawInfo, services); var lookupInstructions = lookupDesc.Forges; additionalForgeables.AddAll(lookupDesc.AdditionalForgeables); // build historical index and lookup strategies foreach (var lookups in lookupInstructions) { foreach (var historical in lookups.HistoricalPlans) { if (historical == null) { continue; } JoinSetComposerPrototypeHistoricalDesc desc = historicalStreamIndexLists[historical.StreamNum] .GetStrategy(historical.LookupStreamNum, statementRawInfo, services.SerdeResolver); historical.HistoricalIndexLookupStrategy = desc.LookupForge; historical.PollResultIndexingStrategy = desc.IndexingForge; additionalForgeables.AddAll(desc.AdditionalForgeables); } } // build strategy tree for putting the result back together var assemblyTopNodeFactory = AssemblyStrategyTreeBuilder.Build( streamNo, substreamsPerStream, requiredPerStream); var assemblyInstructionFactories = BaseAssemblyNodeFactory.GetDescendentNodesBottomUp(assemblyTopNodeFactory); var forge = new LookupInstructionQueryPlanNodeForge( streamNo, streamNames[streamNo], numStreams, requiredPerStream, lookupInstructions, assemblyInstructionFactories); return new QueryPlanNodeForgeDesc(forge, additionalForgeables); }
public static JoinSetComposerPrototypeDesc MakeComposerPrototype( StatementSpecCompiled spec, StreamJoinAnalysisResultCompileTime joinAnalysisResult, StreamTypeService typeService, HistoricalViewableDesc historicalViewableDesc, bool isOnDemandQuery, bool hasAggregations, StatementRawInfo statementRawInfo, StatementCompileTimeServices compileTimeServices) { var streamTypes = typeService.EventTypes; var streamNames = typeService.StreamNames; var whereClause = spec.Raw.WhereClause; var queryPlanLogging = compileTimeServices.Configuration.Common.Logging.IsEnableQueryPlan; var additionalForgeables = new List<StmtClassForgeableFactory>(); // Determine if there is a historical stream, and what dependencies exist var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false); for (var i = 0; i < streamTypes.Length; i++) { if (historicalViewableDesc.Historical[i]) { var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i]; historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn); } } if (Log.IsDebugEnabled) { Log.Debug("Dependency graph: " + historicalDependencyGraph); } // Handle a join with a database or other historical data source for 2 streams var outerJoinDescs = OuterJoinDesc.ToArray(spec.Raw.OuterJoinDescList); if (historicalViewableDesc.IsHistorical && streamTypes.Length == 2) { var desc = MakeComposerHistorical2Stream( outerJoinDescs, whereClause, streamTypes, streamNames, historicalViewableDesc, queryPlanLogging, statementRawInfo, compileTimeServices); return new JoinSetComposerPrototypeDesc(desc.Forge, desc.AdditionalForgeables); } var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescs); // Query graph for graph relationships between streams/historicals // For outer joins the query graph will just contain outer join relationships var hint = ExcludePlanHint.GetHint( typeService.StreamNames, statementRawInfo, compileTimeServices); var queryGraph = new QueryGraphForge(streamTypes.Length, hint, false); if (outerJoinDescs.Length > 0) { OuterJoinAnalyzer.Analyze(outerJoinDescs, queryGraph); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After outer join filterQueryGraph=\n" + queryGraph); } } // Let the query graph reflect the where-clause if (whereClause != null) { // Analyze relationships between streams using the optional filter expression. // Relationships are properties in AND and EQUALS nodes of joins. FilterExprAnalyzer.Analyze(whereClause, queryGraph, isOuterJoins); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After filter expression filterQueryGraph=\n" + queryGraph); } // Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c) QueryGraphForge.FillEquivalentNav(streamTypes, queryGraph); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After fill equiv. nav. filterQueryGraph=\n" + queryGraph); } } // Historical index lists var historicalStreamIndexLists = new HistoricalStreamIndexListForge[streamTypes.Length]; var queryPlanDesc = QueryPlanBuilder.GetPlan( streamTypes, outerJoinDescs, queryGraph, typeService.StreamNames, historicalViewableDesc, historicalDependencyGraph, historicalStreamIndexLists, joinAnalysisResult, queryPlanLogging, statementRawInfo, compileTimeServices); QueryPlanForge queryPlan = queryPlanDesc.Forge; additionalForgeables.AddAll(queryPlanDesc.AdditionalForgeables); // remove unused indexes - consider all streams or all unidirectional var usedIndexes = new HashSet<TableLookupIndexReqKey>(); var indexSpecs = queryPlan.IndexSpecs; for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) { var planNode = queryPlan.ExecNodeSpecs[streamNum]; planNode?.AddIndexes(usedIndexes); } foreach (var indexSpec in indexSpecs) { if (indexSpec == null) { continue; } var items = indexSpec.Items; var indexNames = items.Keys.ToArray(); foreach (var indexName in indexNames) { if (!usedIndexes.Contains(indexName)) { items.Remove(indexName); } } } // plan multikeys IList<StmtClassForgeableFactory> multikeyForgeables = PlanMultikeys( indexSpecs, statementRawInfo, compileTimeServices); additionalForgeables.AddAll(multikeyForgeables); QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook( spec.Annotations, compileTimeServices.ImportServiceCompileTime); if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) { QUERY_PLAN_LOG.Info("Query plan: " + queryPlan.ToQueryPlan()); hook?.Join(queryPlan); } var selectsRemoveStream = spec.Raw.SelectStreamSelectorEnum.IsSelectsRStream() || spec.Raw.OutputLimitSpec != null; var joinRemoveStream = selectsRemoveStream || hasAggregations; ExprNode postJoinEvaluator; if (JoinSetComposerUtil.IsNonUnidirectionalNonSelf( isOuterJoins, joinAnalysisResult.IsUnidirectional, joinAnalysisResult.IsPureSelfJoin)) { postJoinEvaluator = GetFilterExpressionInclOnClause( spec.Raw.WhereClause, outerJoinDescs, statementRawInfo, compileTimeServices); } else { postJoinEvaluator = spec.Raw.WhereClause; } JoinSetComposerPrototypeGeneralForge forge = new JoinSetComposerPrototypeGeneralForge( typeService.EventTypes, postJoinEvaluator, outerJoinDescs.Length > 0, queryPlan, joinAnalysisResult, typeService.StreamNames, joinRemoveStream, historicalViewableDesc.IsHistorical); return new JoinSetComposerPrototypeDesc(forge, additionalForgeables); }
public FAFQueryMethodSelectDesc( StatementSpecCompiled statementSpec, Compilable compilable, StatementRawInfo statementRawInfo, StatementCompileTimeServices services) { Annotations = statementSpec.Annotations; ContextName = statementSpec.Raw.OptionalContextName; var queryPlanLogging = services.Configuration.Common.Logging.IsEnableQueryPlan; if (queryPlanLogging) { QUERY_PLAN_LOG.Info("Query plans for Fire-and-forget query '" + compilable.ToEPL() + "'"); } HasTableAccess = statementSpec.TableAccessNodes != null && statementSpec.TableAccessNodes.Count > 0; foreach (var streamSpec in statementSpec.StreamSpecs) { HasTableAccess |= streamSpec is TableQueryStreamSpec; } HasTableAccess |= StatementLifecycleSvcUtil.IsSubqueryWithTable( statementSpec.SubselectNodes, services.TableCompileTimeResolver); IsDistinct = statementSpec.SelectClauseCompiled.IsDistinct; FAFQueryMethodHelper.ValidateFAFQuery(statementSpec); var numStreams = statementSpec.StreamSpecs.Length; var typesPerStream = new EventType[numStreams]; var namesPerStream = new string[numStreams]; var eventTypeNames = new string[numStreams]; Processors = new FireAndForgetProcessorForge[numStreams]; ConsumerFilters = new ExprNode[numStreams]; // check context partition use if (statementSpec.Raw.OptionalContextName != null) { if (numStreams > 1) { throw new ExprValidationException( "Joins in runtime queries for context partitions are not supported"); } } // resolve types and processors for (var i = 0; i < numStreams; i++) { var streamSpec = statementSpec.StreamSpecs[i]; Processors[i] = FireAndForgetProcessorForgeFactory.ValidateResolveProcessor(streamSpec); if (numStreams > 1 && Processors[i].ContextName != null) { throw new ExprValidationException( "Joins against named windows that are under context are not supported"); } var streamName = Processors[i].NamedWindowOrTableName; if (streamSpec.OptionalStreamName != null) { streamName = streamSpec.OptionalStreamName; } namesPerStream[i] = streamName; typesPerStream[i] = Processors[i].EventTypeRspInputEvents; eventTypeNames[i] = typesPerStream[i].Name; IList<ExprNode> consumerFilterExprs; if (streamSpec is NamedWindowConsumerStreamSpec) { var namedSpec = (NamedWindowConsumerStreamSpec) streamSpec; consumerFilterExprs = namedSpec.FilterExpressions; } else { var tableSpec = (TableQueryStreamSpec) streamSpec; consumerFilterExprs = tableSpec.FilterExpressions; } ConsumerFilters[i] = ExprNodeUtilityMake.ConnectExpressionsByLogicalAndWhenNeeded(consumerFilterExprs); } // compile filter to optimize access to named window var optionalStreamsIfAny = OuterJoinAnalyzer.OptionalStreamsIfAny(statementSpec.Raw.OuterJoinDescList); var types = new StreamTypeServiceImpl( typesPerStream, namesPerStream, new bool[numStreams], false, optionalStreamsIfAny); var excludePlanHint = ExcludePlanHint.GetHint(types.StreamNames, statementRawInfo, services); QueryGraph = new QueryGraphForge(numStreams, excludePlanHint, false); if (statementSpec.Raw.WhereClause != null) { for (var i = 0; i < numStreams; i++) { try { var validationContext = new ExprValidationContextBuilder(types, statementRawInfo, services) .WithAllowBindingConsumption(true) .WithIsFilterExpression(true) .Build(); var validated = ExprNodeUtilityValidate.GetValidatedSubtree( ExprNodeOrigin.FILTER, statementSpec.Raw.WhereClause, validationContext); FilterExprAnalyzer.Analyze(validated, QueryGraph, false); } catch (Exception ex) { Log.Warn("Unexpected exception analyzing filter paths: " + ex.Message, ex); } } } // handle subselects // first we create streams for subselects, if there are any var @base = new StatementBaseInfo(compilable, statementSpec, null, statementRawInfo, null); var subqueryNamedWindowConsumers = new List<NamedWindowConsumerStreamSpec>(); SubSelectActivationDesc subSelectActivationDesc = SubSelectHelperActivations.CreateSubSelectActivation( EmptyList<FilterSpecCompiled>.Instance, subqueryNamedWindowConsumers, @base, services); IDictionary<ExprSubselectNode, SubSelectActivationPlan> subselectActivation = subSelectActivationDesc.Subselects; AdditionalForgeables.AddAll(subSelectActivationDesc.AdditionalForgeables); SubSelectHelperForgePlan subSelectForgePlan = SubSelectHelperForgePlanner.PlanSubSelect( @base, subselectActivation, namesPerStream, typesPerStream, eventTypeNames, services); SubselectForges = subSelectForgePlan.Subselects; AdditionalForgeables.AddAll(subSelectForgePlan.AdditionalForgeables); // obtain result set processor var isIStreamOnly = new bool[namesPerStream.Length]; isIStreamOnly.Fill(true); StreamTypeService typeService = new StreamTypeServiceImpl( typesPerStream, namesPerStream, isIStreamOnly, true, optionalStreamsIfAny); WhereClause = EPStatementStartMethodHelperValidate.ValidateNodes( statementSpec.Raw, typeService, null, statementRawInfo, services); var resultSetSpec = new ResultSetSpec(statementSpec); ResultSetProcessor = ResultSetProcessorFactoryFactory.GetProcessorPrototype( resultSetSpec, typeService, null, new bool[0], true, null, true, false, statementRawInfo, services); AdditionalForgeables.AddAll(ResultSetProcessor.AdditionalForgeables); // plan table access TableAccessForges = ExprTableEvalHelperPlan.PlanTableAccess(statementSpec.Raw.TableExpressions); // plan joins or simple queries if (numStreams > 1) { var streamJoinAnalysisResult = new StreamJoinAnalysisResultCompileTime(numStreams); CompatExtensions.Fill(streamJoinAnalysisResult.NamedWindowsPerStream, (NamedWindowMetaData) null); for (var i = 0; i < numStreams; i++) { var uniqueIndexes = Processors[i].UniqueIndexes; streamJoinAnalysisResult.UniqueKeys[i] = uniqueIndexes; } var hasAggregations = ResultSetProcessor.ResultSetProcessorType.IsAggregated(); var desc = JoinSetComposerPrototypeForgeFactory.MakeComposerPrototype( statementSpec, streamJoinAnalysisResult, types, new HistoricalViewableDesc(numStreams), true, hasAggregations, statementRawInfo, services); AdditionalForgeables.AddAll(desc.AdditionalForgeables); Joins = desc.Forge; } else { Joins = null; } var multiKeyPlan = MultiKeyPlanner.PlanMultiKeyDistinct( IsDistinct, ResultSetProcessor.ResultEventType, statementRawInfo, SerdeCompileTimeResolverNonHA.INSTANCE); AdditionalForgeables.AddAll(multiKeyPlan.MultiKeyForgeables); DistinctMultiKey = multiKeyPlan.ClassRef; }