public static void QueryPlanLogOnSubq( bool queryPlanLogging, ILog queryPlanLog, SubordinateQueryPlanDescForge plan, int subqueryNum, Attribute[] annotations, ImportService importService) { QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook(annotations, importService); if (queryPlanLogging && (queryPlanLog.IsInfoEnabled || hook != null)) { var prefix = "Subquery " + subqueryNum + " "; var strategy = plan == null || plan.LookupStrategyFactory == null ? "table scan" : plan.LookupStrategyFactory.ToQueryPlan(); queryPlanLog.Info(prefix + "strategy " + strategy); if (plan?.IndexDescs != null) { for (var i = 0; i < plan.IndexDescs.Length; i++) { var indexName = plan.IndexDescs[i].IndexName; var indexText = indexName != null ? "index " + indexName + " " : "(implicit) "; queryPlanLog.Info(prefix + "shared index"); queryPlanLog.Info(prefix + indexText); } } if (hook != null) { var pairs = plan == null ? new IndexNameAndDescPair[0] : GetPairs(plan.IndexDescs); var factory = plan?.LookupStrategyFactory.GetType().GetSimpleName(); hook.Subquery(new QueryPlanIndexDescSubquery(pairs, subqueryNum, factory)); } } }
public static void QueryPlanLogOnExpr( bool queryPlanLogging, ILog queryPlanLog, SubordinateWMatchExprQueryPlanForge strategy, Attribute[] annotations, ImportService importService) { QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook(annotations, importService); if (queryPlanLogging && (queryPlanLog.IsInfoEnabled || hook != null)) { var prefix = "On-Expr "; queryPlanLog.Info(prefix + "strategy " + strategy.Strategy.ToQueryPlan()); if (strategy.Indexes == null) { queryPlanLog.Info(prefix + "full table scan"); } else { for (var i = 0; i < strategy.Indexes.Length; i++) { string indexName = strategy.Indexes[i].IndexName; var indexText = indexName != null ? "index " + indexName + " " : "(implicit) (" + i + ")"; queryPlanLog.Info(prefix + indexText); } } if (hook != null) { var pairs = GetPairs(strategy.Indexes); SubordTableLookupStrategyFactoryForge inner = strategy.Strategy.OptionalInnerStrategy; hook.InfraOnExpr( new QueryPlanIndexDescOnExpr( pairs, strategy.Strategy.GetType().GetSimpleName(), inner == null ? null : inner.GetType().GetSimpleName())); } } }
public static JoinSetComposerPrototypeDesc MakeComposerPrototype( StatementSpecCompiled spec, StreamJoinAnalysisResultCompileTime joinAnalysisResult, StreamTypeService typeService, HistoricalViewableDesc historicalViewableDesc, bool isOnDemandQuery, bool hasAggregations, StatementRawInfo statementRawInfo, StatementCompileTimeServices compileTimeServices) { var streamTypes = typeService.EventTypes; var streamNames = typeService.StreamNames; var whereClause = spec.Raw.WhereClause; var queryPlanLogging = compileTimeServices.Configuration.Common.Logging.IsEnableQueryPlan; var additionalForgeables = new List<StmtClassForgeableFactory>(); // Determine if there is a historical stream, and what dependencies exist var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false); for (var i = 0; i < streamTypes.Length; i++) { if (historicalViewableDesc.Historical[i]) { var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i]; historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn); } } if (Log.IsDebugEnabled) { Log.Debug("Dependency graph: " + historicalDependencyGraph); } // Handle a join with a database or other historical data source for 2 streams var outerJoinDescs = OuterJoinDesc.ToArray(spec.Raw.OuterJoinDescList); if (historicalViewableDesc.IsHistorical && streamTypes.Length == 2) { var desc = MakeComposerHistorical2Stream( outerJoinDescs, whereClause, streamTypes, streamNames, historicalViewableDesc, queryPlanLogging, statementRawInfo, compileTimeServices); return new JoinSetComposerPrototypeDesc(desc.Forge, desc.AdditionalForgeables); } var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescs); // Query graph for graph relationships between streams/historicals // For outer joins the query graph will just contain outer join relationships var hint = ExcludePlanHint.GetHint( typeService.StreamNames, statementRawInfo, compileTimeServices); var queryGraph = new QueryGraphForge(streamTypes.Length, hint, false); if (outerJoinDescs.Length > 0) { OuterJoinAnalyzer.Analyze(outerJoinDescs, queryGraph); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After outer join filterQueryGraph=\n" + queryGraph); } } // Let the query graph reflect the where-clause if (whereClause != null) { // Analyze relationships between streams using the optional filter expression. // Relationships are properties in AND and EQUALS nodes of joins. FilterExprAnalyzer.Analyze(whereClause, queryGraph, isOuterJoins); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After filter expression filterQueryGraph=\n" + queryGraph); } // Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c) QueryGraphForge.FillEquivalentNav(streamTypes, queryGraph); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After fill equiv. nav. filterQueryGraph=\n" + queryGraph); } } // Historical index lists var historicalStreamIndexLists = new HistoricalStreamIndexListForge[streamTypes.Length]; var queryPlanDesc = QueryPlanBuilder.GetPlan( streamTypes, outerJoinDescs, queryGraph, typeService.StreamNames, historicalViewableDesc, historicalDependencyGraph, historicalStreamIndexLists, joinAnalysisResult, queryPlanLogging, statementRawInfo, compileTimeServices); QueryPlanForge queryPlan = queryPlanDesc.Forge; additionalForgeables.AddAll(queryPlanDesc.AdditionalForgeables); // remove unused indexes - consider all streams or all unidirectional var usedIndexes = new HashSet<TableLookupIndexReqKey>(); var indexSpecs = queryPlan.IndexSpecs; for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) { var planNode = queryPlan.ExecNodeSpecs[streamNum]; planNode?.AddIndexes(usedIndexes); } foreach (var indexSpec in indexSpecs) { if (indexSpec == null) { continue; } var items = indexSpec.Items; var indexNames = items.Keys.ToArray(); foreach (var indexName in indexNames) { if (!usedIndexes.Contains(indexName)) { items.Remove(indexName); } } } // plan multikeys IList<StmtClassForgeableFactory> multikeyForgeables = PlanMultikeys( indexSpecs, statementRawInfo, compileTimeServices); additionalForgeables.AddAll(multikeyForgeables); QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook( spec.Annotations, compileTimeServices.ImportServiceCompileTime); if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) { QUERY_PLAN_LOG.Info("Query plan: " + queryPlan.ToQueryPlan()); hook?.Join(queryPlan); } var selectsRemoveStream = spec.Raw.SelectStreamSelectorEnum.IsSelectsRStream() || spec.Raw.OutputLimitSpec != null; var joinRemoveStream = selectsRemoveStream || hasAggregations; ExprNode postJoinEvaluator; if (JoinSetComposerUtil.IsNonUnidirectionalNonSelf( isOuterJoins, joinAnalysisResult.IsUnidirectional, joinAnalysisResult.IsPureSelfJoin)) { postJoinEvaluator = GetFilterExpressionInclOnClause( spec.Raw.WhereClause, outerJoinDescs, statementRawInfo, compileTimeServices); } else { postJoinEvaluator = spec.Raw.WhereClause; } JoinSetComposerPrototypeGeneralForge forge = new JoinSetComposerPrototypeGeneralForge( typeService.EventTypes, postJoinEvaluator, outerJoinDescs.Length > 0, queryPlan, joinAnalysisResult, typeService.StreamNames, joinRemoveStream, historicalViewableDesc.IsHistorical); return new JoinSetComposerPrototypeDesc(forge, additionalForgeables); }
private static JoinSetComposerPrototypeHistorical2StreamDesc MakeComposerHistorical2Stream( OuterJoinDesc[] outerJoinDescs, ExprNode whereClause, EventType[] streamTypes, string[] streamNames, HistoricalViewableDesc historicalViewableDesc, bool queryPlanLogging, StatementRawInfo statementRawInfo, StatementCompileTimeServices services) { var polledViewNum = 0; var streamViewNum = 1; if (historicalViewableDesc.Historical[1]) { streamViewNum = 0; polledViewNum = 1; } // if all-historical join, check dependency var isAllHistoricalNoSubordinate = false; if (historicalViewableDesc.Historical[0] && historicalViewableDesc.Historical[1]) { var graph = new DependencyGraph(2, false); graph.AddDependency(0, historicalViewableDesc.DependenciesPerHistorical[0]); graph.AddDependency(1, historicalViewableDesc.DependenciesPerHistorical[1]); if (graph.FirstCircularDependency != null) { throw new ExprValidationException("Circular dependency detected between historical streams"); } // if both streams are independent if (graph.RootNodes.Count == 2) { isAllHistoricalNoSubordinate = true; // No parameters used by either historical } else { if (graph.GetDependenciesForStream(0).Count == 0) { streamViewNum = 0; polledViewNum = 1; } else { streamViewNum = 1; polledViewNum = 0; } } } // Build an outer join expression node var isOuterJoin = false; ExprNode outerJoinEqualsNode = null; var isInnerJoinOnly = false; var outerJoinPerStream = new bool[2]; if (outerJoinDescs != null && outerJoinDescs.Length > 0) { var outerJoinDesc = outerJoinDescs[0]; isInnerJoinOnly = outerJoinDesc.OuterJoinType.Equals(OuterJoinType.INNER); if (isAllHistoricalNoSubordinate) { if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.FULL)) { isOuterJoin = true; outerJoinPerStream[0] = true; outerJoinPerStream[1] = true; } else if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.LEFT)) { isOuterJoin = true; outerJoinPerStream[0] = true; } else if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.RIGHT)) { isOuterJoin = true; outerJoinPerStream[1] = true; } } else { if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.FULL)) { isOuterJoin = true; outerJoinPerStream[0] = true; outerJoinPerStream[1] = true; } else if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.LEFT) && streamViewNum == 0) { isOuterJoin = true; outerJoinPerStream[0] = true; } else if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.RIGHT) && streamViewNum == 1) { isOuterJoin = true; outerJoinPerStream[1] = true; } } outerJoinEqualsNode = outerJoinDesc.MakeExprNode(statementRawInfo, services); } // Determine filter for indexing purposes ExprNode filterForIndexing = null; if (outerJoinEqualsNode != null && whereClause != null && isInnerJoinOnly) { // both filter and outer join, add filterForIndexing = new ExprAndNodeImpl(); filterForIndexing.AddChildNode(whereClause); filterForIndexing.AddChildNode(outerJoinEqualsNode); } else if (outerJoinEqualsNode == null && whereClause != null) { filterForIndexing = whereClause; } else if (outerJoinEqualsNode != null) { filterForIndexing = outerJoinEqualsNode; } var indexStrategies = DetermineIndexing( filterForIndexing, streamTypes[polledViewNum], streamTypes[streamViewNum], polledViewNum, streamViewNum, streamNames, statementRawInfo, services); QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook( statementRawInfo.Annotations, services.ImportServiceCompileTime); if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) { QUERY_PLAN_LOG.Info("historical lookup strategy: " + indexStrategies.LookupForge.ToQueryPlan()); QUERY_PLAN_LOG.Info("historical index strategy: " + indexStrategies.IndexingForge.ToQueryPlan()); hook?.Historical( new QueryPlanIndexDescHistorical( indexStrategies.LookupForge.GetType().GetSimpleName(), indexStrategies.IndexingForge.GetType().GetSimpleName())); } JoinSetComposerPrototypeHistorical2StreamForge forge = new JoinSetComposerPrototypeHistorical2StreamForge( streamTypes, whereClause, isOuterJoin, polledViewNum, streamViewNum, outerJoinEqualsNode, indexStrategies.LookupForge, indexStrategies.IndexingForge, isAllHistoricalNoSubordinate, outerJoinPerStream); return new JoinSetComposerPrototypeHistorical2StreamDesc( forge, indexStrategies.AdditionalForgeables); }