private ExprNode GetFilterExpressionInclOnClause(ExprNode optionalFilterNode, OuterJoinDesc[] outerJoinDescList) { if (optionalFilterNode == null) // no need to add as query planning is fully based on on-clause { return(null); } if (outerJoinDescList.Length == 0) // not an outer-join syntax { return(optionalFilterNode); } if (!OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList)) // all-inner joins { return(optionalFilterNode); } ExprAndNode andNode = new ExprAndNodeImpl(); andNode.AddChildNode(optionalFilterNode); foreach (var outerJoinDesc in outerJoinDescList) { andNode.AddChildNode(outerJoinDesc.MakeExprNode(null)); } try { andNode.Validate(null); } catch (ExprValidationException ex) { throw new EPRuntimeException("Unexpected exception validating expression: " + ex.Message, ex); } return(andNode); }
private ExprNode GetFilterExpressionInclOnClause(ExprNode optionalFilterNode, OuterJoinDesc[] outerJoinDescList) { if (optionalFilterNode == null) { // no need to add as query planning is fully based on on-clause return(null); } if (outerJoinDescList.Length == 0) { // not an outer-join syntax return(optionalFilterNode); } if (!OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList)) { // all-inner joins return(optionalFilterNode); } var hasOnClauses = OuterJoinDesc.HasOnClauses(outerJoinDescList); if (!hasOnClauses) { return(optionalFilterNode); } var expressions = new List <ExprNode>(); expressions.Add(optionalFilterNode); foreach (var outerJoinDesc in outerJoinDescList) { if (outerJoinDesc.OptLeftNode != null) { expressions.Add(outerJoinDesc.MakeExprNode(null)); } } ExprAndNode andNode = ExprNodeUtility.ConnectExpressionsByLogicalAnd(expressions); try { andNode.Validate(null); } catch (ExprValidationException ex) { throw new EPRuntimeException("Unexpected exception validating expression: " + ex.Message, ex); } return(andNode); }
private static ExprNode GetFilterExpressionInclOnClause( ExprNode whereClause, OuterJoinDesc[] outerJoinDescList, StatementRawInfo rawInfo, StatementCompileTimeServices services) { if (whereClause == null) { // no need to add as query planning is fully based on on-clause return null; } if (outerJoinDescList.Length == 0) { // not an outer-join syntax return whereClause; } if (!OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList)) { // all-inner joins return whereClause; } var hasOnClauses = OuterJoinDesc.HasOnClauses(outerJoinDescList); if (!hasOnClauses) { return whereClause; } IList<ExprNode> expressions = new List<ExprNode>(); expressions.Add(whereClause); foreach (var outerJoinDesc in outerJoinDescList) { if (outerJoinDesc.OptLeftNode != null) { expressions.Add(outerJoinDesc.MakeExprNode(rawInfo, services)); } } var andNode = ExprNodeUtilityMake.ConnectExpressionsByLogicalAnd(expressions); try { andNode.Validate(null); } catch (ExprValidationException ex) { throw new EPRuntimeException("Unexpected exception validating expression: " + ex.Message, ex); } return andNode; }
/// <summary> /// Build query plan using the filter. /// </summary> /// <param name="typesPerStream">event types for each stream</param> /// <param name="outerJoinDescList">list of outer join criteria, or null if there are no outer joins</param> /// <param name="queryGraph">relationships between streams based on filter expressions and outer-join on-criteria</param> /// <param name="streamNames">names of streams</param> /// <param name="historicalViewableDesc">The historical viewable desc.</param> /// <param name="dependencyGraph">dependencies between historical streams</param> /// <param name="historicalStreamIndexLists">index management, populated for the query plan</param> /// <param name="streamJoinAnalysisResult">The stream join analysis result.</param> /// <param name="isQueryPlanLogging">if set to <c>true</c> [is query plan logging].</param> /// <param name="annotations">The annotations.</param> /// <param name="exprEvaluatorContext">The expr evaluator context.</param> /// <returns> /// query plan /// </returns> /// <exception cref="System.ArgumentException"> /// Number of join stream types is less then 2 /// or /// Too many outer join descriptors found /// </exception> /// <throws>ExprValidationException if the query plan fails</throws> public static QueryPlan GetPlan(EventType[] typesPerStream, OuterJoinDesc[] outerJoinDescList, QueryGraph queryGraph, string[] streamNames, HistoricalViewableDesc historicalViewableDesc, DependencyGraph dependencyGraph, HistoricalStreamIndexList[] historicalStreamIndexLists, StreamJoinAnalysisResult streamJoinAnalysisResult, bool isQueryPlanLogging, Attribute[] annotations, ExprEvaluatorContext exprEvaluatorContext) { string methodName = ".getPlan "; int numStreams = typesPerStream.Length; if (numStreams < 2) { throw new ArgumentException("Number of join stream types is less then 2"); } if (outerJoinDescList.Length >= numStreams) { throw new ArgumentException("Too many outer join descriptors found"); } if (numStreams == 2) { OuterJoinType?outerJoinType = null; if (outerJoinDescList.Length > 0) { outerJoinType = outerJoinDescList[0].OuterJoinType; } QueryPlan queryPlanX = TwoStreamQueryPlanBuilder.Build(typesPerStream, queryGraph, outerJoinType, streamJoinAnalysisResult.UniqueKeys, streamJoinAnalysisResult.TablesPerStream); RemoveUnidirectionalAndTable(queryPlanX, streamJoinAnalysisResult); if (log.IsDebugEnabled) { log.Debug(methodName + "2-Stream queryPlan=" + queryPlanX); } return(queryPlanX); } bool hasPreferMergeJoin = HintEnum.PREFER_MERGE_JOIN.GetHint(annotations) != null; bool hasForceNestedIter = HintEnum.FORCE_NESTED_ITER.GetHint(annotations) != null; bool isAllInnerJoins = outerJoinDescList.Length == 0 || OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList); if (isAllInnerJoins && !hasPreferMergeJoin) { QueryPlan queryPlanX = NStreamQueryPlanBuilder.Build(queryGraph, typesPerStream, historicalViewableDesc, dependencyGraph, historicalStreamIndexLists, hasForceNestedIter, streamJoinAnalysisResult.UniqueKeys, streamJoinAnalysisResult.TablesPerStream); if (queryPlanX != null) { RemoveUnidirectionalAndTable(queryPlanX, streamJoinAnalysisResult); if (log.IsDebugEnabled) { log.Debug(methodName + "Count-Stream inner-join queryPlan=" + queryPlanX); } return(queryPlanX); } if (isQueryPlanLogging && queryPlanLog.IsInfoEnabled) { log.Info("Switching to Outer-NStream algorithm for query plan"); } } QueryPlan queryPlan = NStreamOuterQueryPlanBuilder.Build(queryGraph, outerJoinDescList, streamNames, typesPerStream, historicalViewableDesc, dependencyGraph, historicalStreamIndexLists, exprEvaluatorContext, streamJoinAnalysisResult.UniqueKeys, streamJoinAnalysisResult.TablesPerStream); RemoveUnidirectionalAndTable(queryPlan, streamJoinAnalysisResult); return(queryPlan); }
/// <summary> /// Builds join tuple composer. /// </summary> /// <param name="statementName">Name of the statement.</param> /// <param name="statementId">The statement identifier.</param> /// <param name="outerJoinDescList">list of descriptors for outer join criteria</param> /// <param name="optionalFilterNode">filter tree for analysis to build indexes for fast access</param> /// <param name="streamTypes">types of streams</param> /// <param name="streamNames">names of streams</param> /// <param name="streamJoinAnalysisResult">The stream join analysis result.</param> /// <param name="queryPlanLogging">if set to <c>true</c> [query plan logging].</param> /// <param name="statementContext">The statement context.</param> /// <param name="historicalViewableDesc">The historical viewable desc.</param> /// <param name="exprEvaluatorContext">The expr evaluator context.</param> /// <param name="selectsRemoveStream">if set to <c>true</c> [selects remove stream].</param> /// <param name="hasAggregations">if set to <c>true</c> [has aggregations].</param> /// <param name="tableService">The table service.</param> /// <param name="isOnDemandQuery">if set to <c>true</c> [is on demand query].</param> /// <param name="allowIndexInit">if set to <c>true</c> [allow index initialize].</param> /// <returns> /// composer implementation /// </returns> /// <throws>com.espertech.esper.epl.expression.core.ExprValidationException is thrown to indicate thatvalidation of view use in joins failed. /// {D255958A-8513-4226-94B9-080D98F904A1}</throws> public static JoinSetComposerPrototype MakeComposerPrototype(string statementName, int statementId, OuterJoinDesc[] outerJoinDescList, ExprNode optionalFilterNode, EventType[] streamTypes, string[] streamNames, StreamJoinAnalysisResult streamJoinAnalysisResult, bool queryPlanLogging, StatementContext statementContext, HistoricalViewableDesc historicalViewableDesc, ExprEvaluatorContext exprEvaluatorContext, bool selectsRemoveStream, bool hasAggregations, TableService tableService, bool isOnDemandQuery, bool allowIndexInit) { // Determine if there is a historical stream, and what dependencies exist var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false); for (var i = 0; i < streamTypes.Length; i++) { if (historicalViewableDesc.Historical[i]) { var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i]; historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn); } } if (log.IsDebugEnabled) { log.Debug("Dependency graph: " + historicalDependencyGraph); } // Handle a join with a database or other historical data source for 2 streams if ((historicalViewableDesc.HasHistorical) && (streamTypes.Length == 2)) { return(MakeComposerHistorical2Stream(outerJoinDescList, optionalFilterNode, streamTypes, historicalViewableDesc, queryPlanLogging, exprEvaluatorContext, statementContext, streamNames, allowIndexInit)); } var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList); // Query graph for graph relationships between streams/historicals // For outer joins the query graph will just contain outer join relationships var hint = ExcludePlanHint.GetHint(streamNames, statementContext); var queryGraph = new QueryGraph(streamTypes.Length, hint, false); if (outerJoinDescList.Length > 0) { OuterJoinAnalyzer.Analyze(outerJoinDescList, queryGraph); if (log.IsDebugEnabled) { log.Debug(".makeComposer After outer join queryGraph=\n" + queryGraph); } } // Let the query graph reflect the where-clause if (optionalFilterNode != null) { // Analyze relationships between streams using the optional filter expression. // Relationships are properties in AND and EQUALS nodes of joins. FilterExprAnalyzer.Analyze(optionalFilterNode, queryGraph, isOuterJoins); if (log.IsDebugEnabled) { log.Debug(".makeComposer After filter expression queryGraph=\n" + queryGraph); } // Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c) QueryGraph.FillEquivalentNav(streamTypes, queryGraph); if (log.IsDebugEnabled) { log.Debug(".makeComposer After fill equiv. nav. queryGraph=\n" + queryGraph); } } // Historical index lists var historicalStreamIndexLists = new HistoricalStreamIndexList[streamTypes.Length]; var queryPlan = QueryPlanBuilder.GetPlan(streamTypes, outerJoinDescList, queryGraph, streamNames, historicalViewableDesc, historicalDependencyGraph, historicalStreamIndexLists, streamJoinAnalysisResult, queryPlanLogging, statementContext.Annotations, exprEvaluatorContext); // remove unused indexes - consider all streams or all unidirectional var usedIndexes = new HashSet <TableLookupIndexReqKey>(); var indexSpecs = queryPlan.IndexSpecs; for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) { var planNode = queryPlan.ExecNodeSpecs[streamNum]; if (planNode != null) { planNode.AddIndexes(usedIndexes); } } foreach (var indexSpec in indexSpecs) { if (indexSpec == null) { continue; } var items = indexSpec.Items; var indexNames = items.Keys.ToArray(); foreach (var indexName in indexNames) { if (!usedIndexes.Contains(indexName)) { items.Remove(indexName); } } } var hook = QueryPlanIndexHookUtil.GetHook(statementContext.Annotations); if (queryPlanLogging && (QueryPlanLog.IsInfoEnabled || hook != null)) { QueryPlanLog.Info("Query plan: " + queryPlan.ToQueryPlan()); if (hook != null) { hook.Join(queryPlan); } } // register index-use references for tables if (!isOnDemandQuery) { foreach (var usedIndex in usedIndexes) { if (usedIndex.TableName != null) { tableService.GetTableMetadata(usedIndex.TableName).AddIndexReference(usedIndex.Name, statementName); } } } var joinRemoveStream = selectsRemoveStream || hasAggregations; return(new JoinSetComposerPrototypeImpl( statementName, statementId, outerJoinDescList, optionalFilterNode, streamTypes, streamNames, streamJoinAnalysisResult, statementContext.Annotations, historicalViewableDesc, exprEvaluatorContext, indexSpecs, queryPlan, historicalStreamIndexLists, joinRemoveStream, isOuterJoins, tableService, statementContext.EventTableIndexService)); }
public static QueryPlanForgeDesc GetPlan( EventType[] typesPerStream, OuterJoinDesc[] outerJoinDescList, QueryGraphForge queryGraph, string[] streamNames, HistoricalViewableDesc historicalViewableDesc, DependencyGraph dependencyGraph, HistoricalStreamIndexListForge[] historicalStreamIndexLists, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, bool isQueryPlanLogging, StatementRawInfo statementRawInfo, StatementCompileTimeServices services) { string methodName = ".getPlan "; int numStreams = typesPerStream.Length; if (numStreams < 2) { throw new ArgumentException("Number of join stream types is less then 2"); } if (outerJoinDescList.Length >= numStreams) { throw new ArgumentException("Too many outer join descriptors found"); } if (numStreams == 2) { OuterJoinType? outerJoinType = null; if (outerJoinDescList.Length > 0) { outerJoinType = outerJoinDescList[0].OuterJoinType; } QueryPlanForgeDesc queryPlan = TwoStreamQueryPlanBuilder.Build( typesPerStream, queryGraph, outerJoinType, streamJoinAnalysisResult, statementRawInfo); RemoveUnidirectionalAndTable(queryPlan.Forge, streamJoinAnalysisResult); if (Log.IsDebugEnabled) { Log.Debug(methodName + "2-Stream queryPlan=" + queryPlan); } return queryPlan; } bool hasPreferMergeJoin = HintEnum.PREFER_MERGE_JOIN.GetHint(statementRawInfo.Annotations) != null; bool hasForceNestedIter = HintEnum.FORCE_NESTED_ITER.GetHint(statementRawInfo.Annotations) != null; bool isAllInnerJoins = outerJoinDescList.Length == 0 || OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList); if (isAllInnerJoins && !hasPreferMergeJoin) { QueryPlanForgeDesc queryPlan = NStreamQueryPlanBuilder.Build( queryGraph, typesPerStream, historicalViewableDesc, dependencyGraph, historicalStreamIndexLists, hasForceNestedIter, streamJoinAnalysisResult.UniqueKeys, streamJoinAnalysisResult.TablesPerStream, streamJoinAnalysisResult, statementRawInfo, services.SerdeResolver); if (queryPlan != null) { RemoveUnidirectionalAndTable(queryPlan.Forge, streamJoinAnalysisResult); if (Log.IsDebugEnabled) { Log.Debug(methodName + "N-Stream inner-join queryPlan=" + queryPlan); } return queryPlan; } if (isQueryPlanLogging && QUERY_PLAN_LOG.IsInfoEnabled) { Log.Info("Switching to Outer-NStream algorithm for query plan"); } } QueryPlanForgeDesc queryPlanX = NStreamOuterQueryPlanBuilder.Build( queryGraph, outerJoinDescList, streamNames, typesPerStream, historicalViewableDesc, dependencyGraph, historicalStreamIndexLists, streamJoinAnalysisResult.UniqueKeys, streamJoinAnalysisResult.TablesPerStream, streamJoinAnalysisResult, statementRawInfo, services); RemoveUnidirectionalAndTable(queryPlanX.Forge, streamJoinAnalysisResult); return queryPlanX; }
public static JoinSetComposerPrototypeDesc MakeComposerPrototype( StatementSpecCompiled spec, StreamJoinAnalysisResultCompileTime joinAnalysisResult, StreamTypeService typeService, HistoricalViewableDesc historicalViewableDesc, bool isOnDemandQuery, bool hasAggregations, StatementRawInfo statementRawInfo, StatementCompileTimeServices compileTimeServices) { var streamTypes = typeService.EventTypes; var streamNames = typeService.StreamNames; var whereClause = spec.Raw.WhereClause; var queryPlanLogging = compileTimeServices.Configuration.Common.Logging.IsEnableQueryPlan; var additionalForgeables = new List<StmtClassForgeableFactory>(); // Determine if there is a historical stream, and what dependencies exist var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false); for (var i = 0; i < streamTypes.Length; i++) { if (historicalViewableDesc.Historical[i]) { var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i]; historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn); } } if (Log.IsDebugEnabled) { Log.Debug("Dependency graph: " + historicalDependencyGraph); } // Handle a join with a database or other historical data source for 2 streams var outerJoinDescs = OuterJoinDesc.ToArray(spec.Raw.OuterJoinDescList); if (historicalViewableDesc.IsHistorical && streamTypes.Length == 2) { var desc = MakeComposerHistorical2Stream( outerJoinDescs, whereClause, streamTypes, streamNames, historicalViewableDesc, queryPlanLogging, statementRawInfo, compileTimeServices); return new JoinSetComposerPrototypeDesc(desc.Forge, desc.AdditionalForgeables); } var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescs); // Query graph for graph relationships between streams/historicals // For outer joins the query graph will just contain outer join relationships var hint = ExcludePlanHint.GetHint( typeService.StreamNames, statementRawInfo, compileTimeServices); var queryGraph = new QueryGraphForge(streamTypes.Length, hint, false); if (outerJoinDescs.Length > 0) { OuterJoinAnalyzer.Analyze(outerJoinDescs, queryGraph); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After outer join filterQueryGraph=\n" + queryGraph); } } // Let the query graph reflect the where-clause if (whereClause != null) { // Analyze relationships between streams using the optional filter expression. // Relationships are properties in AND and EQUALS nodes of joins. FilterExprAnalyzer.Analyze(whereClause, queryGraph, isOuterJoins); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After filter expression filterQueryGraph=\n" + queryGraph); } // Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c) QueryGraphForge.FillEquivalentNav(streamTypes, queryGraph); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After fill equiv. nav. filterQueryGraph=\n" + queryGraph); } } // Historical index lists var historicalStreamIndexLists = new HistoricalStreamIndexListForge[streamTypes.Length]; var queryPlanDesc = QueryPlanBuilder.GetPlan( streamTypes, outerJoinDescs, queryGraph, typeService.StreamNames, historicalViewableDesc, historicalDependencyGraph, historicalStreamIndexLists, joinAnalysisResult, queryPlanLogging, statementRawInfo, compileTimeServices); QueryPlanForge queryPlan = queryPlanDesc.Forge; additionalForgeables.AddAll(queryPlanDesc.AdditionalForgeables); // remove unused indexes - consider all streams or all unidirectional var usedIndexes = new HashSet<TableLookupIndexReqKey>(); var indexSpecs = queryPlan.IndexSpecs; for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) { var planNode = queryPlan.ExecNodeSpecs[streamNum]; planNode?.AddIndexes(usedIndexes); } foreach (var indexSpec in indexSpecs) { if (indexSpec == null) { continue; } var items = indexSpec.Items; var indexNames = items.Keys.ToArray(); foreach (var indexName in indexNames) { if (!usedIndexes.Contains(indexName)) { items.Remove(indexName); } } } // plan multikeys IList<StmtClassForgeableFactory> multikeyForgeables = PlanMultikeys( indexSpecs, statementRawInfo, compileTimeServices); additionalForgeables.AddAll(multikeyForgeables); QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook( spec.Annotations, compileTimeServices.ImportServiceCompileTime); if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) { QUERY_PLAN_LOG.Info("Query plan: " + queryPlan.ToQueryPlan()); hook?.Join(queryPlan); } var selectsRemoveStream = spec.Raw.SelectStreamSelectorEnum.IsSelectsRStream() || spec.Raw.OutputLimitSpec != null; var joinRemoveStream = selectsRemoveStream || hasAggregations; ExprNode postJoinEvaluator; if (JoinSetComposerUtil.IsNonUnidirectionalNonSelf( isOuterJoins, joinAnalysisResult.IsUnidirectional, joinAnalysisResult.IsPureSelfJoin)) { postJoinEvaluator = GetFilterExpressionInclOnClause( spec.Raw.WhereClause, outerJoinDescs, statementRawInfo, compileTimeServices); } else { postJoinEvaluator = spec.Raw.WhereClause; } JoinSetComposerPrototypeGeneralForge forge = new JoinSetComposerPrototypeGeneralForge( typeService.EventTypes, postJoinEvaluator, outerJoinDescs.Length > 0, queryPlan, joinAnalysisResult, typeService.StreamNames, joinRemoveStream, historicalViewableDesc.IsHistorical); return new JoinSetComposerPrototypeDesc(forge, additionalForgeables); }