private static void HandleIndexDependencies( QueryPlanForge queryPlan, StatementCompileTimeServices services) { if (queryPlan == null) { return; } var indexes = new HashSet<TableLookupIndexReqKey>(); for (var streamnum = 0; streamnum < queryPlan.ExecNodeSpecs.Length; streamnum++) { var node = queryPlan.ExecNodeSpecs[streamnum]; indexes.Clear(); node.AddIndexes(indexes); foreach (var index in indexes) { if (index.TableName != null) { var tableMeta = services.TableCompileTimeResolver.Resolve(index.TableName); if (tableMeta.TableVisibility == NameAccessModifier.PUBLIC) { services.ModuleDependenciesCompileTime.AddPathIndex( false, index.TableName, tableMeta.TableModuleName, index.IndexName, index.IndexModuleName, services.NamedWindowCompileTimeRegistry, services.TableCompileTimeRegistry); } } } } }
public static void CompareQueryPlans( QueryPlanForge expectedPlan, QueryPlanForge actualPlan) { IDictionary<TableLookupIndexReqKey, TableLookupIndexReqKey> indexNameMapping = new Dictionary<TableLookupIndexReqKey, TableLookupIndexReqKey>(); CompareIndexes(expectedPlan.IndexSpecs, actualPlan.IndexSpecs, indexNameMapping); CompareExecNodeSpecs(expectedPlan.ExecNodeSpecs, actualPlan.ExecNodeSpecs, indexNameMapping); }
private SupportQueryPlanBuilder(int numStreams) { var indexes = new QueryPlanIndexForge[numStreams]; for (var i = 0; i < indexes.Length; i++) { indexes[i] = new QueryPlanIndexForge(new LinkedHashMap<TableLookupIndexReqKey, QueryPlanIndexItemForge>()); } queryPlan = new QueryPlanForge(indexes, new QueryPlanNodeForge[numStreams]); }
private static void TryAssertion( RegressionEnvironment env, string epl, QueryPlanForge expectedPlan) { SupportQueryPlanIndexHook.Reset(); epl = INDEX_CALLBACK_HOOK + epl; env.CompileDeploy(epl); var actualPlan = SupportQueryPlanIndexHook.AssertJoinAndReset(); SupportQueryPlanIndexHelper.CompareQueryPlans(expectedPlan, actualPlan); env.UndeployAll(); }
public JoinSetComposerPrototypeGeneralForge( EventType[] streamTypes, ExprNode postJoinEvaluator, bool outerJoins, QueryPlanForge queryPlan, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, string[] streamNames, bool joinRemoveStream, bool hasHistorical) : base(streamTypes, postJoinEvaluator, outerJoins) { this.queryPlan = queryPlan; this.streamJoinAnalysisResult = streamJoinAnalysisResult; this.streamNames = streamNames; this.joinRemoveStream = joinRemoveStream; this.hasHistorical = hasHistorical; }
// Remove plans for non-unidirectional streams private static void RemoveUnidirectionalAndTable( QueryPlanForge queryPlan, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult) { bool allUnidirectional = streamJoinAnalysisResult.IsUnidirectionalAll; for (int streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) { if (allUnidirectional) { queryPlan.ExecNodeSpecs[streamNum] = new QueryPlanNodeForgeAllUnidirectionalOuter(streamNum); } else { bool unidirectional = streamJoinAnalysisResult.IsUnidirectional && !streamJoinAnalysisResult.UnidirectionalInd[streamNum]; bool table = streamJoinAnalysisResult.TablesPerStream[streamNum] != null; if (unidirectional || table) { queryPlan.ExecNodeSpecs[streamNum] = QueryPlanNodeNoOpForge.INSTANCE; } } } }
private void AssertPlan(QueryPlanForge plan) { Assert.AreEqual(2, plan.ExecNodeSpecs.Length); Assert.AreEqual(2, plan.ExecNodeSpecs.Length); }
public static SupportQueryPlanBuilder Start(QueryPlanForge existing) { return new SupportQueryPlanBuilder(existing); }
public SupportQueryPlanBuilder(QueryPlanForge queryPlan) { this.queryPlan = queryPlan; }
public static QueryPlanForgeDesc Build( QueryGraphForge queryGraph, EventType[] typesPerStream, HistoricalViewableDesc historicalViewableDesc, DependencyGraph dependencyGraph, HistoricalStreamIndexListForge[] historicalStreamIndexLists, bool hasForceNestedIter, string[][][] indexedStreamsUniqueProps, TableMetaData[] tablesPerStream, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, StatementRawInfo raw, SerdeCompileTimeResolver serdeResolver) { if (Log.IsDebugEnabled) { Log.Debug(".build filterQueryGraph=" + queryGraph); } var numStreams = queryGraph.NumStreams; var additionalForgeables = new List<StmtClassForgeableFactory>(); var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec( queryGraph, typesPerStream, indexedStreamsUniqueProps); if (Log.IsDebugEnabled) { Log.Debug(".build Index build completed, indexes=" + QueryPlanIndexForge.Print(indexSpecs)); } // any historical streams don't get indexes, the lookup strategy accounts for cached indexes if (historicalViewableDesc.IsHistorical) { for (var i = 0; i < historicalViewableDesc.Historical.Length; i++) { if (historicalViewableDesc.Historical[i]) { indexSpecs[i] = null; } } } var planNodeSpecs = new QueryPlanNodeForge[numStreams]; var worstDepth = int.MaxValue; for (var streamNo = 0; streamNo < numStreams; streamNo++) { // no plan for historical streams that are dependent upon other streams if (historicalViewableDesc.Historical[streamNo] && dependencyGraph.HasDependency(streamNo)) { planNodeSpecs[streamNo] = new QueryPlanNodeNoOpForge(); continue; } var bestChainResult = ComputeBestPath(streamNo, queryGraph, dependencyGraph); var bestChain = bestChainResult.Chain; if (Log.IsDebugEnabled) { Log.Debug(".build For stream " + streamNo + " bestChain=" + bestChain.RenderAny()); } if (bestChainResult.Depth < worstDepth) { worstDepth = bestChainResult.Depth; } var planDesc = CreateStreamPlan( streamNo, bestChain, queryGraph, indexSpecs, typesPerStream, historicalViewableDesc.Historical, historicalStreamIndexLists, tablesPerStream, streamJoinAnalysisResult, raw, serdeResolver); planNodeSpecs[streamNo] = planDesc.Forge; additionalForgeables.AddAll(planDesc.AdditionalForgeables); if (Log.IsDebugEnabled) { Log.Debug(".build spec=" + planNodeSpecs[streamNo]); } } // We use the merge/nested (outer) join algorithm instead. if (worstDepth < numStreams - 1 && !hasForceNestedIter) { return null; } // build historical index and lookup strategies for (var i = 0; i < numStreams; i++) { var plan = planNodeSpecs[i]; QueryPlanNodeForgeVisitor visitor = new ProxyQueryPlanNodeForgeVisitor { ProcVisit = node => { if (node is HistoricalDataPlanNodeForge) { var historical = (HistoricalDataPlanNodeForge) node; JoinSetComposerPrototypeHistoricalDesc desc = historicalStreamIndexLists[historical.StreamNum].GetStrategy( historical.LookupStreamNum, raw, serdeResolver); historical.PollResultIndexingStrategy = desc.IndexingForge; historical.HistoricalIndexLookupStrategy = desc.LookupForge; additionalForgeables.AddAll(desc.AdditionalForgeables); } } }; plan.Accept(visitor); } var forge = new QueryPlanForge(indexSpecs, planNodeSpecs); return new QueryPlanForgeDesc(forge, additionalForgeables); }
/// <summary> /// Build query plan. /// </summary> /// <param name="queryGraph">navigability info</param> /// <param name="optionalOuterJoinType">outer join type, null if not an outer join</param> /// <param name="typesPerStream">event types for each stream</param> /// <param name="streamJoinAnalysisResult">stream join analysis</param> /// <returns>query plan</returns> public static QueryPlanForgeDesc Build( EventType[] typesPerStream, QueryGraphForge queryGraph, OuterJoinType? optionalOuterJoinType, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, StatementRawInfo raw) { var uniqueIndexProps = streamJoinAnalysisResult.UniqueKeys; var tablesPerStream = streamJoinAnalysisResult.TablesPerStream; var additionalForgeable = new List<StmtClassForgeableFactory>(); var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec( queryGraph, typesPerStream, uniqueIndexProps); var execNodeSpecs = new QueryPlanNodeForge[2]; var lookupPlans = new TableLookupPlanForge[2]; // plan lookup from 1 to zero TableLookupPlanDesc plan1to0 = NStreamQueryPlanBuilder.CreateLookupPlan( queryGraph, 1, 0, streamJoinAnalysisResult.IsVirtualDW(0), indexSpecs[0], typesPerStream, tablesPerStream[0], raw, SerdeCompileTimeResolverNonHA.INSTANCE); lookupPlans[1] = plan1to0.Forge; additionalForgeable.AddAll(plan1to0.AdditionalForgeables); // plan lookup from zero to 1 TableLookupPlanDesc plan0to1 = NStreamQueryPlanBuilder.CreateLookupPlan( queryGraph, 0, 1, streamJoinAnalysisResult.IsVirtualDW(1), indexSpecs[1], typesPerStream, tablesPerStream[1], raw, SerdeCompileTimeResolverNonHA.INSTANCE); lookupPlans[0] = plan0to1.Forge; additionalForgeable.AddAll(plan0to1.AdditionalForgeables); execNodeSpecs[0] = new TableLookupNodeForge(lookupPlans[0]); execNodeSpecs[1] = new TableLookupNodeForge(lookupPlans[1]); if (optionalOuterJoinType != null) { if ((optionalOuterJoinType.Equals(OuterJoinType.LEFT)) || (optionalOuterJoinType.Equals(OuterJoinType.FULL))) { execNodeSpecs[0] = new TableOuterLookupNodeForge(lookupPlans[0]); } if ((optionalOuterJoinType.Equals(OuterJoinType.RIGHT)) || (optionalOuterJoinType.Equals(OuterJoinType.FULL))) { execNodeSpecs[1] = new TableOuterLookupNodeForge(lookupPlans[1]); } } var forge = new QueryPlanForge(indexSpecs, execNodeSpecs); return new QueryPlanForgeDesc(forge, additionalForgeable); }
public void Join(QueryPlanForge join) { JOINS.Add(join); }
public void TestBuild() { QueryPlanForge plan = NStreamQueryPlanBuilder.Build(queryGraph, typesPerStream, new HistoricalViewableDesc(6), dependencyGraph, null, false, new string[queryGraph.NumStreams][][], new TableMetaData[queryGraph.NumStreams], new StreamJoinAnalysisResultCompileTime(5)); log.Debug(".testBuild plan=" + plan); }
internal static QueryPlanForgeDesc Build( QueryGraphForge queryGraph, OuterJoinDesc[] outerJoinDescList, string[] streamNames, EventType[] typesPerStream, HistoricalViewableDesc historicalViewableDesc, DependencyGraph dependencyGraph, HistoricalStreamIndexListForge[] historicalStreamIndexLists, string[][][] indexedStreamsUniqueProps, TableMetaData[] tablesPerStream, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, StatementRawInfo statementRawInfo, StatementCompileTimeServices services) { if (Log.IsDebugEnabled) { Log.Debug(".build filterQueryGraph=" + queryGraph); } var numStreams = queryGraph.NumStreams; var planNodeSpecs = new QueryPlanNodeForge[numStreams]; var additionalForgeables = new List<StmtClassForgeableFactory>(); // Build index specifications var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec( queryGraph, typesPerStream, indexedStreamsUniqueProps); // any historical streams don't get indexes, the lookup strategy accounts for cached indexes if (historicalViewableDesc.IsHistorical) { for (var i = 0; i < historicalViewableDesc.Historical.Length; i++) { if (historicalViewableDesc.Historical[i]) { indexSpecs[i] = null; } } } // Build graph of the outer join to inner table relationships. // Build a map of inner joins. OuterInnerDirectionalGraph outerInnerGraph; InnerJoinGraph innerJoinGraph; if (outerJoinDescList.Length > 0) { outerInnerGraph = GraphOuterJoins(numStreams, outerJoinDescList); innerJoinGraph = InnerJoinGraph.GraphInnerJoins(numStreams, outerJoinDescList); } else { // all inner joins - thereby no (or empty) directional graph outerInnerGraph = new OuterInnerDirectionalGraph(numStreams); innerJoinGraph = new InnerJoinGraph(numStreams, true); } if (Log.IsDebugEnabled) { Log.Debug(".build directional graph=" + outerInnerGraph.Print()); } // For each stream determine the query plan for (var streamNo = 0; streamNo < numStreams; streamNo++) { // no plan for historical streams that are dependent upon other streams if (historicalViewableDesc.Historical[streamNo] && dependencyGraph.HasDependency(streamNo)) { planNodeSpecs[streamNo] = new QueryPlanNodeNoOpForge(); continue; } QueryPlanNodeForgeDesc desc = BuildPlanNode( numStreams, streamNo, streamNames, queryGraph, outerInnerGraph, outerJoinDescList, innerJoinGraph, indexSpecs, typesPerStream, historicalViewableDesc.Historical, dependencyGraph, historicalStreamIndexLists, tablesPerStream, streamJoinAnalysisResult, statementRawInfo, services); QueryPlanNodeForge queryPlanNode = desc.Forge; additionalForgeables.AddAll(desc.AdditionalForgeables); if (Log.IsDebugEnabled) { Log.Debug( ".build spec for stream '" + streamNames[streamNo] + "' number " + streamNo + " is " + queryPlanNode); } planNodeSpecs[streamNo] = queryPlanNode; } var queryPlan = new QueryPlanForge(indexSpecs, planNodeSpecs); if (Log.IsDebugEnabled) { Log.Debug(".build query plan=" + queryPlan); } return new QueryPlanForgeDesc(queryPlan, additionalForgeables); }
public static JoinSetComposerPrototypeDesc MakeComposerPrototype( StatementSpecCompiled spec, StreamJoinAnalysisResultCompileTime joinAnalysisResult, StreamTypeService typeService, HistoricalViewableDesc historicalViewableDesc, bool isOnDemandQuery, bool hasAggregations, StatementRawInfo statementRawInfo, StatementCompileTimeServices compileTimeServices) { var streamTypes = typeService.EventTypes; var streamNames = typeService.StreamNames; var whereClause = spec.Raw.WhereClause; var queryPlanLogging = compileTimeServices.Configuration.Common.Logging.IsEnableQueryPlan; var additionalForgeables = new List<StmtClassForgeableFactory>(); // Determine if there is a historical stream, and what dependencies exist var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false); for (var i = 0; i < streamTypes.Length; i++) { if (historicalViewableDesc.Historical[i]) { var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i]; historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn); } } if (Log.IsDebugEnabled) { Log.Debug("Dependency graph: " + historicalDependencyGraph); } // Handle a join with a database or other historical data source for 2 streams var outerJoinDescs = OuterJoinDesc.ToArray(spec.Raw.OuterJoinDescList); if (historicalViewableDesc.IsHistorical && streamTypes.Length == 2) { var desc = MakeComposerHistorical2Stream( outerJoinDescs, whereClause, streamTypes, streamNames, historicalViewableDesc, queryPlanLogging, statementRawInfo, compileTimeServices); return new JoinSetComposerPrototypeDesc(desc.Forge, desc.AdditionalForgeables); } var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescs); // Query graph for graph relationships between streams/historicals // For outer joins the query graph will just contain outer join relationships var hint = ExcludePlanHint.GetHint( typeService.StreamNames, statementRawInfo, compileTimeServices); var queryGraph = new QueryGraphForge(streamTypes.Length, hint, false); if (outerJoinDescs.Length > 0) { OuterJoinAnalyzer.Analyze(outerJoinDescs, queryGraph); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After outer join filterQueryGraph=\n" + queryGraph); } } // Let the query graph reflect the where-clause if (whereClause != null) { // Analyze relationships between streams using the optional filter expression. // Relationships are properties in AND and EQUALS nodes of joins. FilterExprAnalyzer.Analyze(whereClause, queryGraph, isOuterJoins); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After filter expression filterQueryGraph=\n" + queryGraph); } // Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c) QueryGraphForge.FillEquivalentNav(streamTypes, queryGraph); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After fill equiv. nav. filterQueryGraph=\n" + queryGraph); } } // Historical index lists var historicalStreamIndexLists = new HistoricalStreamIndexListForge[streamTypes.Length]; var queryPlanDesc = QueryPlanBuilder.GetPlan( streamTypes, outerJoinDescs, queryGraph, typeService.StreamNames, historicalViewableDesc, historicalDependencyGraph, historicalStreamIndexLists, joinAnalysisResult, queryPlanLogging, statementRawInfo, compileTimeServices); QueryPlanForge queryPlan = queryPlanDesc.Forge; additionalForgeables.AddAll(queryPlanDesc.AdditionalForgeables); // remove unused indexes - consider all streams or all unidirectional var usedIndexes = new HashSet<TableLookupIndexReqKey>(); var indexSpecs = queryPlan.IndexSpecs; for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) { var planNode = queryPlan.ExecNodeSpecs[streamNum]; planNode?.AddIndexes(usedIndexes); } foreach (var indexSpec in indexSpecs) { if (indexSpec == null) { continue; } var items = indexSpec.Items; var indexNames = items.Keys.ToArray(); foreach (var indexName in indexNames) { if (!usedIndexes.Contains(indexName)) { items.Remove(indexName); } } } // plan multikeys IList<StmtClassForgeableFactory> multikeyForgeables = PlanMultikeys( indexSpecs, statementRawInfo, compileTimeServices); additionalForgeables.AddAll(multikeyForgeables); QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook( spec.Annotations, compileTimeServices.ImportServiceCompileTime); if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) { QUERY_PLAN_LOG.Info("Query plan: " + queryPlan.ToQueryPlan()); hook?.Join(queryPlan); } var selectsRemoveStream = spec.Raw.SelectStreamSelectorEnum.IsSelectsRStream() || spec.Raw.OutputLimitSpec != null; var joinRemoveStream = selectsRemoveStream || hasAggregations; ExprNode postJoinEvaluator; if (JoinSetComposerUtil.IsNonUnidirectionalNonSelf( isOuterJoins, joinAnalysisResult.IsUnidirectional, joinAnalysisResult.IsPureSelfJoin)) { postJoinEvaluator = GetFilterExpressionInclOnClause( spec.Raw.WhereClause, outerJoinDescs, statementRawInfo, compileTimeServices); } else { postJoinEvaluator = spec.Raw.WhereClause; } JoinSetComposerPrototypeGeneralForge forge = new JoinSetComposerPrototypeGeneralForge( typeService.EventTypes, postJoinEvaluator, outerJoinDescs.Length > 0, queryPlan, joinAnalysisResult, typeService.StreamNames, joinRemoveStream, historicalViewableDesc.IsHistorical); return new JoinSetComposerPrototypeDesc(forge, additionalForgeables); }