public static void QueryPlanLogOnSubq(bool queryPlanLogging, ILog queryPlanLog, SubordinateQueryPlanDesc plan, int subqueryNum, Attribute[] annotations) { var hook = QueryPlanIndexHookUtil.GetHook(annotations); if (queryPlanLogging && (queryPlanLog.IsInfoEnabled || hook != null)) { var prefix = "Subquery " + subqueryNum + " "; var strategy = (plan == null || plan.LookupStrategyFactory == null) ? "table scan" : plan.LookupStrategyFactory.ToQueryPlan(); queryPlanLog.Info(prefix + "strategy " + strategy); if (plan != null) { if (plan.IndexDescs != null) { for (var i = 0; i < plan.IndexDescs.Length; i++) { var indexName = plan.IndexDescs[i].IndexName; var indexText = indexName != null ? "index " + indexName + " " : "(implicit) "; queryPlanLog.Info(prefix + "shared index"); queryPlanLog.Info(prefix + indexText); } } } if (hook != null) { var pairs = plan == null ? new IndexNameAndDescPair[0] : GetPairs(plan.IndexDescs); string factory = plan == null ? null : plan.LookupStrategyFactory.GetType().Name; hook.Subquery(new QueryPlanIndexDescSubquery(pairs, subqueryNum, factory)); } } }
private static void QueryPlanReport( string indexNameOrNull, EventTable eventTableOrNull, Attribute[] annotations, AgentInstanceContext agentInstanceContext, string objectName) { var hook = QueryPlanIndexHookUtil.GetHook(annotations, agentInstanceContext.ImportServiceRuntime); var queryPlanLogging = agentInstanceContext.RuntimeSettingsService.ConfigurationCommon.Logging .IsEnableQueryPlan; if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) { var prefix = "Fire-and-forget or init-time-query from " + objectName + " "; var indexText = indexNameOrNull != null ? "index " + indexNameOrNull + " " : "full table scan "; indexText += "(snapshot only, for join see separate query plan) "; if (eventTableOrNull == null) { QUERY_PLAN_LOG.Info(prefix + indexText); } else { QUERY_PLAN_LOG.Info(prefix + indexText + eventTableOrNull.ToQueryPlan()); } hook?.FireAndForget( new QueryPlanIndexDescFAF( new[] { new IndexNameAndDescPair( indexNameOrNull, eventTableOrNull != null ? eventTableOrNull.ProviderClass.Name : null) })); } }
public static void QueryPlanLogOnExpr(bool queryPlanLogging, ILog queryPlanLog, SubordinateWMatchExprQueryPlanResult strategy, Attribute[] annotations) { var hook = QueryPlanIndexHookUtil.GetHook(annotations); if (queryPlanLogging && (queryPlanLog.IsInfoEnabled || hook != null)) { var prefix = "On-Expr "; queryPlanLog.Info(prefix + "strategy " + strategy.Factory.ToQueryPlan()); if (strategy.IndexDescs == null) { queryPlanLog.Info(prefix + "full table scan"); } else { for (var i = 0; i < strategy.IndexDescs.Length; i++) { var indexName = strategy.IndexDescs[i].IndexName; var indexText = indexName != null ? "index " + indexName + " " : "(implicit) (" + i + ")"; queryPlanLog.Info(prefix + indexText); } } if (hook != null) { var pairs = GetPairs(strategy.IndexDescs); var inner = strategy.Factory.OptionalInnerStrategy; hook.InfraOnExpr(new QueryPlanIndexDescOnExpr(pairs, strategy.Factory.GetType().Name, inner == null ? null : inner.GetType().Name)); } } }
/// <summary> /// The queryPlanReport /// </summary> /// <param name="indexNameOrNull">The <see cref="string"/></param> /// <param name="eventTableOrNull">The <see cref="EventTable"/></param> /// <param name="attributes">The <see cref="Attribute"/> array</param> /// <param name="agentInstanceContext">The <see cref="AgentInstanceContext"/></param> /// <param name="queryPlanLogging">The <see cref="bool"/></param> /// <param name="queryPlanLogDestination">The <see cref="ILog"/></param> /// <param name="objectName">The <see cref="string"/></param> private static void queryPlanReport( string indexNameOrNull, EventTable eventTableOrNull, Attribute[] attributes, AgentInstanceContext agentInstanceContext, bool queryPlanLogging, ILog queryPlanLogDestination, string objectName) { var hook = QueryPlanIndexHookUtil.GetHook(attributes, agentInstanceContext.StatementContext.EngineImportService); if (queryPlanLogging && (queryPlanLogDestination.IsInfoEnabled || hook != null)) { var prefix = "Fire-and-forget from " + objectName + " "; var indexText = indexNameOrNull != null ? "index " + indexNameOrNull + " " : "full table scan "; indexText += "(snapshot only, for join see separate query plan) "; if (eventTableOrNull == null) { queryPlanLogDestination.Info(prefix + indexText); } else { queryPlanLogDestination.Info(prefix + indexText + eventTableOrNull.ToQueryPlan()); } if (hook != null) { hook.FireAndForget(new QueryPlanIndexDescFAF( new IndexNameAndDescPair[] { new IndexNameAndDescPair(indexNameOrNull, eventTableOrNull != null ? eventTableOrNull.ProviderClass.Name : null) })); } } }
public static void QueryPlanLogOnExpr( bool queryPlanLogging, ILog queryPlanLog, SubordinateWMatchExprQueryPlanForge strategy, Attribute[] annotations, ImportService importService) { QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook(annotations, importService); if (queryPlanLogging && (queryPlanLog.IsInfoEnabled || hook != null)) { var prefix = "On-Expr "; queryPlanLog.Info(prefix + "strategy " + strategy.Strategy.ToQueryPlan()); if (strategy.Indexes == null) { queryPlanLog.Info(prefix + "full table scan"); } else { for (var i = 0; i < strategy.Indexes.Length; i++) { string indexName = strategy.Indexes[i].IndexName; var indexText = indexName != null ? "index " + indexName + " " : "(implicit) (" + i + ")"; queryPlanLog.Info(prefix + indexText); } } if (hook != null) { var pairs = GetPairs(strategy.Indexes); SubordTableLookupStrategyFactoryForge inner = strategy.Strategy.OptionalInnerStrategy; hook.InfraOnExpr( new QueryPlanIndexDescOnExpr( pairs, strategy.Strategy.GetType().GetSimpleName(), inner == null ? null : inner.GetType().GetSimpleName())); } } }
public static ICollection <EventBean> Snapshot( FilterSpecCompiled optionalFilter, Attribute[] annotations, VirtualDWView virtualDataWindow, EventTableIndexRepository indexRepository, bool queryPlanLogging, ILog queryPlanLogDestination, string objectName, AgentInstanceContext agentInstanceContext) { if (optionalFilter == null || optionalFilter.Parameters.Length == 0) { if (virtualDataWindow != null) { var pair = virtualDataWindow.GetFireAndForgetDesc(Collections.GetEmptySet <string>(), Collections.GetEmptySet <string>()); return(virtualDataWindow.GetFireAndForgetData(pair.Second, new object[0], new RangeIndexLookupValue[0], annotations)); } return(null); } // Determine what straight-equals keys and which ranges are available. // Widening/Coercion is part of filter spec compile. ISet <string> keysAvailable = new HashSet <string>(); ISet <string> rangesAvailable = new HashSet <string>(); if (optionalFilter.Parameters.Length == 1) { foreach (FilterSpecParam param in optionalFilter.Parameters[0]) { if (!(param is FilterSpecParamConstant || param is FilterSpecParamRange || param is FilterSpecParamIn)) { continue; } if (param.FilterOperator == FilterOperator.EQUAL || param.FilterOperator == FilterOperator.IS || param.FilterOperator == FilterOperator.IN_LIST_OF_VALUES) { keysAvailable.Add(param.Lookupable.Expression); } else if (param.FilterOperator.IsRangeOperator() || param.FilterOperator.IsInvertedRangeOperator() || param.FilterOperator.IsComparisonOperator()) { rangesAvailable.Add(param.Lookupable.Expression); } else if (param.FilterOperator.IsRangeOperator()) { rangesAvailable.Add(param.Lookupable.Expression); } } } // Find an index that matches the needs Pair <IndexMultiKey, EventTableAndNamePair> tablePair; if (virtualDataWindow != null) { var tablePairNoName = virtualDataWindow.GetFireAndForgetDesc(keysAvailable, rangesAvailable); tablePair = new Pair <IndexMultiKey, EventTableAndNamePair>(tablePairNoName.First, new EventTableAndNamePair(tablePairNoName.Second, null)); } else { var indexHint = IndexHint.GetIndexHint(annotations); IList <IndexHintInstruction> optionalIndexHintInstructions = null; if (indexHint != null) { optionalIndexHintInstructions = indexHint.InstructionsFireAndForget; } tablePair = indexRepository.FindTable(keysAvailable, rangesAvailable, optionalIndexHintInstructions); } var hook = QueryPlanIndexHookUtil.GetHook(annotations); if (queryPlanLogging && (queryPlanLogDestination.IsInfoEnabled || hook != null)) { var prefix = "Fire-and-forget from " + objectName + " "; var indexName = tablePair != null && tablePair.Second != null ? tablePair.Second.IndexName : null; var indexText = indexName != null ? "index " + indexName + " " : "full table scan "; indexText += "(snapshot only, for join see separate query plan)"; if (tablePair == null) { queryPlanLogDestination.Info(prefix + indexText); } else { queryPlanLogDestination.Info(prefix + indexText + tablePair.Second.EventTable.ToQueryPlan()); } if (hook != null) { hook.FireAndForget(new QueryPlanIndexDescFAF( new IndexNameAndDescPair[] { new IndexNameAndDescPair(indexName, tablePair != null ? tablePair.Second.EventTable.ProviderClass.Name : null) })); } } if (tablePair == null) { return(null); // indicates table scan } // Compile key sets which contain key index lookup values var keyIndexProps = IndexedPropDesc.GetIndexProperties(tablePair.First.HashIndexedProps); var hasKeyWithInClause = false; var keyValues = new object[keyIndexProps.Length]; for (var keyIndex = 0; keyIndex < keyIndexProps.Length; keyIndex++) { foreach (var param in optionalFilter.Parameters[0]) { if (param.Lookupable.Expression.Equals(keyIndexProps[keyIndex])) { if (param.FilterOperator == FilterOperator.IN_LIST_OF_VALUES) { var keyValuesList = ((MultiKeyUntyped)param.GetFilterValue(null, agentInstanceContext)).Keys; if (keyValuesList.Length == 0) { continue; } else if (keyValuesList.Length == 1) { keyValues[keyIndex] = keyValuesList[0]; } else { keyValues[keyIndex] = keyValuesList; hasKeyWithInClause = true; } } else { keyValues[keyIndex] = param.GetFilterValue(null, agentInstanceContext); } break; } } } // Analyze ranges - these may include key lookup value (EQUALS semantics) var rangeIndexProps = IndexedPropDesc.GetIndexProperties(tablePair.First.RangeIndexedProps); RangeIndexLookupValue[] rangeValues; if (rangeIndexProps.Length > 0) { rangeValues = CompileRangeLookupValues(rangeIndexProps, optionalFilter.Parameters[0], agentInstanceContext); } else { rangeValues = new RangeIndexLookupValue[0]; } var eventTable = tablePair.Second.EventTable; var indexMultiKey = tablePair.First; // table lookup without in-clause if (!hasKeyWithInClause) { return(FafTableLookup(virtualDataWindow, indexMultiKey, eventTable, keyValues, rangeValues, annotations)); } // table lookup with in-clause: determine combinations var combinations = new object[keyIndexProps.Length][]; for (var i = 0; i < keyValues.Length; i++) { if (keyValues[i] is object[]) { combinations[i] = (object[])keyValues[i]; } else { combinations[i] = new object[] { keyValues[i] }; } } // enumerate combinations var enumeration = new CombinationEnumeration(combinations); var events = new HashSet <EventBean>(); for (; enumeration.MoveNext();) { object[] keys = enumeration.Current; var result = FafTableLookup(virtualDataWindow, indexMultiKey, eventTable, keys, rangeValues, annotations); events.AddAll(result); } return(events); }
/// <summary> /// Builds join tuple composer. /// </summary> /// <param name="statementName">Name of the statement.</param> /// <param name="statementId">The statement identifier.</param> /// <param name="outerJoinDescList">list of descriptors for outer join criteria</param> /// <param name="optionalFilterNode">filter tree for analysis to build indexes for fast access</param> /// <param name="streamTypes">types of streams</param> /// <param name="streamNames">names of streams</param> /// <param name="streamJoinAnalysisResult">The stream join analysis result.</param> /// <param name="queryPlanLogging">if set to <c>true</c> [query plan logging].</param> /// <param name="statementContext">The statement context.</param> /// <param name="historicalViewableDesc">The historical viewable desc.</param> /// <param name="exprEvaluatorContext">The expr evaluator context.</param> /// <param name="selectsRemoveStream">if set to <c>true</c> [selects remove stream].</param> /// <param name="hasAggregations">if set to <c>true</c> [has aggregations].</param> /// <param name="tableService">The table service.</param> /// <param name="isOnDemandQuery">if set to <c>true</c> [is on demand query].</param> /// <param name="allowIndexInit">if set to <c>true</c> [allow index initialize].</param> /// <returns> /// composer implementation /// </returns> /// <throws>com.espertech.esper.epl.expression.core.ExprValidationException is thrown to indicate thatvalidation of view use in joins failed. /// {D255958A-8513-4226-94B9-080D98F904A1}</throws> public static JoinSetComposerPrototype MakeComposerPrototype(string statementName, int statementId, OuterJoinDesc[] outerJoinDescList, ExprNode optionalFilterNode, EventType[] streamTypes, string[] streamNames, StreamJoinAnalysisResult streamJoinAnalysisResult, bool queryPlanLogging, StatementContext statementContext, HistoricalViewableDesc historicalViewableDesc, ExprEvaluatorContext exprEvaluatorContext, bool selectsRemoveStream, bool hasAggregations, TableService tableService, bool isOnDemandQuery, bool allowIndexInit) { // Determine if there is a historical stream, and what dependencies exist var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false); for (var i = 0; i < streamTypes.Length; i++) { if (historicalViewableDesc.Historical[i]) { var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i]; historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn); } } if (log.IsDebugEnabled) { log.Debug("Dependency graph: " + historicalDependencyGraph); } // Handle a join with a database or other historical data source for 2 streams if ((historicalViewableDesc.HasHistorical) && (streamTypes.Length == 2)) { return(MakeComposerHistorical2Stream(outerJoinDescList, optionalFilterNode, streamTypes, historicalViewableDesc, queryPlanLogging, exprEvaluatorContext, statementContext, streamNames, allowIndexInit)); } var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList); // Query graph for graph relationships between streams/historicals // For outer joins the query graph will just contain outer join relationships var hint = ExcludePlanHint.GetHint(streamNames, statementContext); var queryGraph = new QueryGraph(streamTypes.Length, hint, false); if (outerJoinDescList.Length > 0) { OuterJoinAnalyzer.Analyze(outerJoinDescList, queryGraph); if (log.IsDebugEnabled) { log.Debug(".makeComposer After outer join queryGraph=\n" + queryGraph); } } // Let the query graph reflect the where-clause if (optionalFilterNode != null) { // Analyze relationships between streams using the optional filter expression. // Relationships are properties in AND and EQUALS nodes of joins. FilterExprAnalyzer.Analyze(optionalFilterNode, queryGraph, isOuterJoins); if (log.IsDebugEnabled) { log.Debug(".makeComposer After filter expression queryGraph=\n" + queryGraph); } // Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c) QueryGraph.FillEquivalentNav(streamTypes, queryGraph); if (log.IsDebugEnabled) { log.Debug(".makeComposer After fill equiv. nav. queryGraph=\n" + queryGraph); } } // Historical index lists var historicalStreamIndexLists = new HistoricalStreamIndexList[streamTypes.Length]; var queryPlan = QueryPlanBuilder.GetPlan(streamTypes, outerJoinDescList, queryGraph, streamNames, historicalViewableDesc, historicalDependencyGraph, historicalStreamIndexLists, streamJoinAnalysisResult, queryPlanLogging, statementContext.Annotations, exprEvaluatorContext); // remove unused indexes - consider all streams or all unidirectional var usedIndexes = new HashSet <TableLookupIndexReqKey>(); var indexSpecs = queryPlan.IndexSpecs; for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) { var planNode = queryPlan.ExecNodeSpecs[streamNum]; if (planNode != null) { planNode.AddIndexes(usedIndexes); } } foreach (var indexSpec in indexSpecs) { if (indexSpec == null) { continue; } var items = indexSpec.Items; var indexNames = items.Keys.ToArray(); foreach (var indexName in indexNames) { if (!usedIndexes.Contains(indexName)) { items.Remove(indexName); } } } var hook = QueryPlanIndexHookUtil.GetHook(statementContext.Annotations); if (queryPlanLogging && (QueryPlanLog.IsInfoEnabled || hook != null)) { QueryPlanLog.Info("Query plan: " + queryPlan.ToQueryPlan()); if (hook != null) { hook.Join(queryPlan); } } // register index-use references for tables if (!isOnDemandQuery) { foreach (var usedIndex in usedIndexes) { if (usedIndex.TableName != null) { tableService.GetTableMetadata(usedIndex.TableName).AddIndexReference(usedIndex.Name, statementName); } } } var joinRemoveStream = selectsRemoveStream || hasAggregations; return(new JoinSetComposerPrototypeImpl( statementName, statementId, outerJoinDescList, optionalFilterNode, streamTypes, streamNames, streamJoinAnalysisResult, statementContext.Annotations, historicalViewableDesc, exprEvaluatorContext, indexSpecs, queryPlan, historicalStreamIndexLists, joinRemoveStream, isOuterJoins, tableService, statementContext.EventTableIndexService)); }
private static JoinSetComposerPrototype MakeComposerHistorical2Stream(OuterJoinDesc[] outerJoinDescList, ExprNode optionalFilterNode, EventType[] streamTypes, HistoricalViewableDesc historicalViewableDesc, bool queryPlanLogging, ExprEvaluatorContext exprEvaluatorContext, StatementContext statementContext, string[] streamNames, bool allowIndexInit) { var polledViewNum = 0; var streamViewNum = 1; if (historicalViewableDesc.Historical[1]) { streamViewNum = 0; polledViewNum = 1; } // if all-historical join, check dependency var isAllHistoricalNoSubordinate = false; if ((historicalViewableDesc.Historical[0]) && historicalViewableDesc.Historical[1]) { var graph = new DependencyGraph(2, false); graph.AddDependency(0, historicalViewableDesc.DependenciesPerHistorical[0]); graph.AddDependency(1, historicalViewableDesc.DependenciesPerHistorical[1]); if (graph.FirstCircularDependency != null) { throw new ExprValidationException("Circular dependency detected between historical streams"); } // if both streams are independent if (graph.RootNodes.Count == 2) { isAllHistoricalNoSubordinate = true; // No parameters used by either historical } else { if ((graph.GetDependenciesForStream(0).Count == 0)) { streamViewNum = 0; polledViewNum = 1; } else { streamViewNum = 1; polledViewNum = 0; } } } // Build an outer join expression node var isOuterJoin = false; var isInnerJoinOnly = false; ExprNode outerJoinEqualsNode = null; if (outerJoinDescList.Length > 0) { var outerJoinDesc = outerJoinDescList[0]; isInnerJoinOnly = outerJoinDesc.OuterJoinType == OuterJoinType.INNER; if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.FULL)) { isOuterJoin = true; } else if ((outerJoinDesc.OuterJoinType.Equals(OuterJoinType.LEFT)) && (streamViewNum == 0)) { isOuterJoin = true; } else if ((outerJoinDesc.OuterJoinType.Equals(OuterJoinType.RIGHT)) && (streamViewNum == 1)) { isOuterJoin = true; } outerJoinEqualsNode = outerJoinDesc.MakeExprNode(exprEvaluatorContext); } // Determine filter for indexing purposes ExprNode filterForIndexing = null; if ((outerJoinEqualsNode != null) && (optionalFilterNode != null) && isInnerJoinOnly) // both filter and outer join, add { filterForIndexing = new ExprAndNodeImpl(); filterForIndexing.AddChildNode(optionalFilterNode); filterForIndexing.AddChildNode(outerJoinEqualsNode); } else if ((outerJoinEqualsNode == null) && (optionalFilterNode != null)) { filterForIndexing = optionalFilterNode; } else if (outerJoinEqualsNode != null) { filterForIndexing = outerJoinEqualsNode; } var indexStrategies = DetermineIndexing(filterForIndexing, streamTypes[polledViewNum], streamTypes[streamViewNum], polledViewNum, streamViewNum, statementContext, streamNames); var hook = QueryPlanIndexHookUtil.GetHook(statementContext.Annotations); if (queryPlanLogging && (QueryPlanLog.IsInfoEnabled || hook != null)) { QueryPlanLog.Info("historical lookup strategy: " + indexStrategies.First.ToQueryPlan()); QueryPlanLog.Info("historical index strategy: " + indexStrategies.Second.ToQueryPlan()); if (hook != null) { hook.Historical(new QueryPlanIndexDescHistorical(indexStrategies.First.GetType().Name, indexStrategies.Second.GetType().Name)); } } return(new JoinSetComposerPrototypeHistorical2StreamImpl( optionalFilterNode, streamTypes, exprEvaluatorContext, polledViewNum, streamViewNum, isOuterJoin, outerJoinEqualsNode, indexStrategies, isAllHistoricalNoSubordinate, outerJoinDescList, allowIndexInit)); }
public static JoinSetComposerPrototypeDesc MakeComposerPrototype( StatementSpecCompiled spec, StreamJoinAnalysisResultCompileTime joinAnalysisResult, StreamTypeService typeService, HistoricalViewableDesc historicalViewableDesc, bool isOnDemandQuery, bool hasAggregations, StatementRawInfo statementRawInfo, StatementCompileTimeServices compileTimeServices) { var streamTypes = typeService.EventTypes; var streamNames = typeService.StreamNames; var whereClause = spec.Raw.WhereClause; var queryPlanLogging = compileTimeServices.Configuration.Common.Logging.IsEnableQueryPlan; var additionalForgeables = new List<StmtClassForgeableFactory>(); // Determine if there is a historical stream, and what dependencies exist var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false); for (var i = 0; i < streamTypes.Length; i++) { if (historicalViewableDesc.Historical[i]) { var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i]; historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn); } } if (Log.IsDebugEnabled) { Log.Debug("Dependency graph: " + historicalDependencyGraph); } // Handle a join with a database or other historical data source for 2 streams var outerJoinDescs = OuterJoinDesc.ToArray(spec.Raw.OuterJoinDescList); if (historicalViewableDesc.IsHistorical && streamTypes.Length == 2) { var desc = MakeComposerHistorical2Stream( outerJoinDescs, whereClause, streamTypes, streamNames, historicalViewableDesc, queryPlanLogging, statementRawInfo, compileTimeServices); return new JoinSetComposerPrototypeDesc(desc.Forge, desc.AdditionalForgeables); } var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescs); // Query graph for graph relationships between streams/historicals // For outer joins the query graph will just contain outer join relationships var hint = ExcludePlanHint.GetHint( typeService.StreamNames, statementRawInfo, compileTimeServices); var queryGraph = new QueryGraphForge(streamTypes.Length, hint, false); if (outerJoinDescs.Length > 0) { OuterJoinAnalyzer.Analyze(outerJoinDescs, queryGraph); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After outer join filterQueryGraph=\n" + queryGraph); } } // Let the query graph reflect the where-clause if (whereClause != null) { // Analyze relationships between streams using the optional filter expression. // Relationships are properties in AND and EQUALS nodes of joins. FilterExprAnalyzer.Analyze(whereClause, queryGraph, isOuterJoins); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After filter expression filterQueryGraph=\n" + queryGraph); } // Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c) QueryGraphForge.FillEquivalentNav(streamTypes, queryGraph); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After fill equiv. nav. filterQueryGraph=\n" + queryGraph); } } // Historical index lists var historicalStreamIndexLists = new HistoricalStreamIndexListForge[streamTypes.Length]; var queryPlanDesc = QueryPlanBuilder.GetPlan( streamTypes, outerJoinDescs, queryGraph, typeService.StreamNames, historicalViewableDesc, historicalDependencyGraph, historicalStreamIndexLists, joinAnalysisResult, queryPlanLogging, statementRawInfo, compileTimeServices); QueryPlanForge queryPlan = queryPlanDesc.Forge; additionalForgeables.AddAll(queryPlanDesc.AdditionalForgeables); // remove unused indexes - consider all streams or all unidirectional var usedIndexes = new HashSet<TableLookupIndexReqKey>(); var indexSpecs = queryPlan.IndexSpecs; for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) { var planNode = queryPlan.ExecNodeSpecs[streamNum]; planNode?.AddIndexes(usedIndexes); } foreach (var indexSpec in indexSpecs) { if (indexSpec == null) { continue; } var items = indexSpec.Items; var indexNames = items.Keys.ToArray(); foreach (var indexName in indexNames) { if (!usedIndexes.Contains(indexName)) { items.Remove(indexName); } } } // plan multikeys IList<StmtClassForgeableFactory> multikeyForgeables = PlanMultikeys( indexSpecs, statementRawInfo, compileTimeServices); additionalForgeables.AddAll(multikeyForgeables); QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook( spec.Annotations, compileTimeServices.ImportServiceCompileTime); if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) { QUERY_PLAN_LOG.Info("Query plan: " + queryPlan.ToQueryPlan()); hook?.Join(queryPlan); } var selectsRemoveStream = spec.Raw.SelectStreamSelectorEnum.IsSelectsRStream() || spec.Raw.OutputLimitSpec != null; var joinRemoveStream = selectsRemoveStream || hasAggregations; ExprNode postJoinEvaluator; if (JoinSetComposerUtil.IsNonUnidirectionalNonSelf( isOuterJoins, joinAnalysisResult.IsUnidirectional, joinAnalysisResult.IsPureSelfJoin)) { postJoinEvaluator = GetFilterExpressionInclOnClause( spec.Raw.WhereClause, outerJoinDescs, statementRawInfo, compileTimeServices); } else { postJoinEvaluator = spec.Raw.WhereClause; } JoinSetComposerPrototypeGeneralForge forge = new JoinSetComposerPrototypeGeneralForge( typeService.EventTypes, postJoinEvaluator, outerJoinDescs.Length > 0, queryPlan, joinAnalysisResult, typeService.StreamNames, joinRemoveStream, historicalViewableDesc.IsHistorical); return new JoinSetComposerPrototypeDesc(forge, additionalForgeables); }
private static JoinSetComposerPrototypeHistorical2StreamDesc MakeComposerHistorical2Stream( OuterJoinDesc[] outerJoinDescs, ExprNode whereClause, EventType[] streamTypes, string[] streamNames, HistoricalViewableDesc historicalViewableDesc, bool queryPlanLogging, StatementRawInfo statementRawInfo, StatementCompileTimeServices services) { var polledViewNum = 0; var streamViewNum = 1; if (historicalViewableDesc.Historical[1]) { streamViewNum = 0; polledViewNum = 1; } // if all-historical join, check dependency var isAllHistoricalNoSubordinate = false; if (historicalViewableDesc.Historical[0] && historicalViewableDesc.Historical[1]) { var graph = new DependencyGraph(2, false); graph.AddDependency(0, historicalViewableDesc.DependenciesPerHistorical[0]); graph.AddDependency(1, historicalViewableDesc.DependenciesPerHistorical[1]); if (graph.FirstCircularDependency != null) { throw new ExprValidationException("Circular dependency detected between historical streams"); } // if both streams are independent if (graph.RootNodes.Count == 2) { isAllHistoricalNoSubordinate = true; // No parameters used by either historical } else { if (graph.GetDependenciesForStream(0).Count == 0) { streamViewNum = 0; polledViewNum = 1; } else { streamViewNum = 1; polledViewNum = 0; } } } // Build an outer join expression node var isOuterJoin = false; ExprNode outerJoinEqualsNode = null; var isInnerJoinOnly = false; var outerJoinPerStream = new bool[2]; if (outerJoinDescs != null && outerJoinDescs.Length > 0) { var outerJoinDesc = outerJoinDescs[0]; isInnerJoinOnly = outerJoinDesc.OuterJoinType.Equals(OuterJoinType.INNER); if (isAllHistoricalNoSubordinate) { if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.FULL)) { isOuterJoin = true; outerJoinPerStream[0] = true; outerJoinPerStream[1] = true; } else if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.LEFT)) { isOuterJoin = true; outerJoinPerStream[0] = true; } else if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.RIGHT)) { isOuterJoin = true; outerJoinPerStream[1] = true; } } else { if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.FULL)) { isOuterJoin = true; outerJoinPerStream[0] = true; outerJoinPerStream[1] = true; } else if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.LEFT) && streamViewNum == 0) { isOuterJoin = true; outerJoinPerStream[0] = true; } else if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.RIGHT) && streamViewNum == 1) { isOuterJoin = true; outerJoinPerStream[1] = true; } } outerJoinEqualsNode = outerJoinDesc.MakeExprNode(statementRawInfo, services); } // Determine filter for indexing purposes ExprNode filterForIndexing = null; if (outerJoinEqualsNode != null && whereClause != null && isInnerJoinOnly) { // both filter and outer join, add filterForIndexing = new ExprAndNodeImpl(); filterForIndexing.AddChildNode(whereClause); filterForIndexing.AddChildNode(outerJoinEqualsNode); } else if (outerJoinEqualsNode == null && whereClause != null) { filterForIndexing = whereClause; } else if (outerJoinEqualsNode != null) { filterForIndexing = outerJoinEqualsNode; } var indexStrategies = DetermineIndexing( filterForIndexing, streamTypes[polledViewNum], streamTypes[streamViewNum], polledViewNum, streamViewNum, streamNames, statementRawInfo, services); QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook( statementRawInfo.Annotations, services.ImportServiceCompileTime); if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) { QUERY_PLAN_LOG.Info("historical lookup strategy: " + indexStrategies.LookupForge.ToQueryPlan()); QUERY_PLAN_LOG.Info("historical index strategy: " + indexStrategies.IndexingForge.ToQueryPlan()); hook?.Historical( new QueryPlanIndexDescHistorical( indexStrategies.LookupForge.GetType().GetSimpleName(), indexStrategies.IndexingForge.GetType().GetSimpleName())); } JoinSetComposerPrototypeHistorical2StreamForge forge = new JoinSetComposerPrototypeHistorical2StreamForge( streamTypes, whereClause, isOuterJoin, polledViewNum, streamViewNum, outerJoinEqualsNode, indexStrategies.LookupForge, indexStrategies.IndexingForge, isAllHistoricalNoSubordinate, outerJoinPerStream); return new JoinSetComposerPrototypeHistorical2StreamDesc( forge, indexStrategies.AdditionalForgeables); }