private ExprNode GetFilterExpressionInclOnClause(ExprNode optionalFilterNode, OuterJoinDesc[] outerJoinDescList)
{
if (optionalFilterNode == null) // no need to add as query planning is fully based on on-clause
{
return(null);
}
if (outerJoinDescList.Length == 0) // not an outer-join syntax
{
return(optionalFilterNode);
}
if (!OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList)) // all-inner joins
{
return(optionalFilterNode);
}
ExprAndNode andNode = new ExprAndNodeImpl();
andNode.AddChildNode(optionalFilterNode);
foreach (var outerJoinDesc in outerJoinDescList)
{
andNode.AddChildNode(outerJoinDesc.MakeExprNode(null));
}
try {
andNode.Validate(null);
}
catch (ExprValidationException ex) {
throw new EPRuntimeException("Unexpected exception validating expression: " + ex.Message, ex);
}
return(andNode);
}
private ExprNode GetFilterExpressionInclOnClause(ExprNode optionalFilterNode, OuterJoinDesc[] outerJoinDescList)
{
if (optionalFilterNode == null)
{ // no need to add as query planning is fully based on on-clause
return(null);
}
if (outerJoinDescList.Length == 0)
{ // not an outer-join syntax
return(optionalFilterNode);
}
if (!OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList))
{ // all-inner joins
return(optionalFilterNode);
}
var hasOnClauses = OuterJoinDesc.HasOnClauses(outerJoinDescList);
if (!hasOnClauses)
{
return(optionalFilterNode);
}
var expressions = new List <ExprNode>();
expressions.Add(optionalFilterNode);
foreach (var outerJoinDesc in outerJoinDescList)
{
if (outerJoinDesc.OptLeftNode != null)
{
expressions.Add(outerJoinDesc.MakeExprNode(null));
}
}
ExprAndNode andNode = ExprNodeUtility.ConnectExpressionsByLogicalAnd(expressions);
try
{
andNode.Validate(null);
}
catch (ExprValidationException ex)
{
throw new EPRuntimeException("Unexpected exception validating expression: " + ex.Message, ex);
}
return(andNode);
}
private static ExprNode GetFilterExpressionInclOnClause(
ExprNode whereClause,
OuterJoinDesc[] outerJoinDescList,
StatementRawInfo rawInfo,
StatementCompileTimeServices services)
{
if (whereClause == null) { // no need to add as query planning is fully based on on-clause
return null;
}
if (outerJoinDescList.Length == 0) { // not an outer-join syntax
return whereClause;
}
if (!OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList)) { // all-inner joins
return whereClause;
}
var hasOnClauses = OuterJoinDesc.HasOnClauses(outerJoinDescList);
if (!hasOnClauses) {
return whereClause;
}
IList<ExprNode> expressions = new List<ExprNode>();
expressions.Add(whereClause);
foreach (var outerJoinDesc in outerJoinDescList) {
if (outerJoinDesc.OptLeftNode != null) {
expressions.Add(outerJoinDesc.MakeExprNode(rawInfo, services));
}
}
var andNode = ExprNodeUtilityMake.ConnectExpressionsByLogicalAnd(expressions);
try {
andNode.Validate(null);
}
catch (ExprValidationException ex) {
throw new EPRuntimeException("Unexpected exception validating expression: " + ex.Message, ex);
}
return andNode;
}
/// <summary>
/// Build query plan using the filter.
/// </summary>
/// <param name="typesPerStream">event types for each stream</param>
/// <param name="outerJoinDescList">list of outer join criteria, or null if there are no outer joins</param>
/// <param name="queryGraph">relationships between streams based on filter expressions and outer-join on-criteria</param>
/// <param name="streamNames">names of streams</param>
/// <param name="historicalViewableDesc">The historical viewable desc.</param>
/// <param name="dependencyGraph">dependencies between historical streams</param>
/// <param name="historicalStreamIndexLists">index management, populated for the query plan</param>
/// <param name="streamJoinAnalysisResult">The stream join analysis result.</param>
/// <param name="isQueryPlanLogging">if set to <c>true</c> [is query plan logging].</param>
/// <param name="annotations">The annotations.</param>
/// <param name="exprEvaluatorContext">The expr evaluator context.</param>
/// <returns>
/// query plan
/// </returns>
/// <exception cref="System.ArgumentException">
/// Number of join stream types is less then 2
/// or
/// Too many outer join descriptors found
/// </exception>
/// <throws>ExprValidationException if the query plan fails</throws>
public static QueryPlan GetPlan(EventType[] typesPerStream,
OuterJoinDesc[] outerJoinDescList,
QueryGraph queryGraph,
string[] streamNames,
HistoricalViewableDesc historicalViewableDesc,
DependencyGraph dependencyGraph,
HistoricalStreamIndexList[] historicalStreamIndexLists,
StreamJoinAnalysisResult streamJoinAnalysisResult,
bool isQueryPlanLogging,
Attribute[] annotations,
ExprEvaluatorContext exprEvaluatorContext)
{
string methodName = ".getPlan ";
int numStreams = typesPerStream.Length;
if (numStreams < 2)
{
throw new ArgumentException("Number of join stream types is less then 2");
}
if (outerJoinDescList.Length >= numStreams)
{
throw new ArgumentException("Too many outer join descriptors found");
}
if (numStreams == 2)
{
OuterJoinType?outerJoinType = null;
if (outerJoinDescList.Length > 0)
{
outerJoinType = outerJoinDescList[0].OuterJoinType;
}
QueryPlan queryPlanX = TwoStreamQueryPlanBuilder.Build(typesPerStream, queryGraph, outerJoinType, streamJoinAnalysisResult.UniqueKeys, streamJoinAnalysisResult.TablesPerStream);
RemoveUnidirectionalAndTable(queryPlanX, streamJoinAnalysisResult);
if (log.IsDebugEnabled)
{
log.Debug(methodName + "2-Stream queryPlan=" + queryPlanX);
}
return(queryPlanX);
}
bool hasPreferMergeJoin = HintEnum.PREFER_MERGE_JOIN.GetHint(annotations) != null;
bool hasForceNestedIter = HintEnum.FORCE_NESTED_ITER.GetHint(annotations) != null;
bool isAllInnerJoins = outerJoinDescList.Length == 0 || OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList);
if (isAllInnerJoins && !hasPreferMergeJoin)
{
QueryPlan queryPlanX = NStreamQueryPlanBuilder.Build(queryGraph, typesPerStream,
historicalViewableDesc, dependencyGraph, historicalStreamIndexLists,
hasForceNestedIter, streamJoinAnalysisResult.UniqueKeys,
streamJoinAnalysisResult.TablesPerStream);
if (queryPlanX != null)
{
RemoveUnidirectionalAndTable(queryPlanX, streamJoinAnalysisResult);
if (log.IsDebugEnabled)
{
log.Debug(methodName + "Count-Stream inner-join queryPlan=" + queryPlanX);
}
return(queryPlanX);
}
if (isQueryPlanLogging && queryPlanLog.IsInfoEnabled)
{
log.Info("Switching to Outer-NStream algorithm for query plan");
}
}
QueryPlan queryPlan = NStreamOuterQueryPlanBuilder.Build(queryGraph, outerJoinDescList, streamNames, typesPerStream,
historicalViewableDesc, dependencyGraph, historicalStreamIndexLists, exprEvaluatorContext, streamJoinAnalysisResult.UniqueKeys,
streamJoinAnalysisResult.TablesPerStream);
RemoveUnidirectionalAndTable(queryPlan, streamJoinAnalysisResult);
return(queryPlan);
}
/// <summary>
/// Builds join tuple composer.
/// </summary>
/// <param name="statementName">Name of the statement.</param>
/// <param name="statementId">The statement identifier.</param>
/// <param name="outerJoinDescList">list of descriptors for outer join criteria</param>
/// <param name="optionalFilterNode">filter tree for analysis to build indexes for fast access</param>
/// <param name="streamTypes">types of streams</param>
/// <param name="streamNames">names of streams</param>
/// <param name="streamJoinAnalysisResult">The stream join analysis result.</param>
/// <param name="queryPlanLogging">if set to <c>true</c> [query plan logging].</param>
/// <param name="statementContext">The statement context.</param>
/// <param name="historicalViewableDesc">The historical viewable desc.</param>
/// <param name="exprEvaluatorContext">The expr evaluator context.</param>
/// <param name="selectsRemoveStream">if set to <c>true</c> [selects remove stream].</param>
/// <param name="hasAggregations">if set to <c>true</c> [has aggregations].</param>
/// <param name="tableService">The table service.</param>
/// <param name="isOnDemandQuery">if set to <c>true</c> [is on demand query].</param>
/// <param name="allowIndexInit">if set to <c>true</c> [allow index initialize].</param>
/// <returns>
/// composer implementation
/// </returns>
/// <throws>com.espertech.esper.epl.expression.core.ExprValidationException is thrown to indicate thatvalidation of view use in joins failed.
/// {D255958A-8513-4226-94B9-080D98F904A1}</throws>
public static JoinSetComposerPrototype MakeComposerPrototype(string statementName, int statementId, OuterJoinDesc[] outerJoinDescList, ExprNode optionalFilterNode, EventType[] streamTypes, string[] streamNames, StreamJoinAnalysisResult streamJoinAnalysisResult, bool queryPlanLogging, StatementContext statementContext, HistoricalViewableDesc historicalViewableDesc, ExprEvaluatorContext exprEvaluatorContext, bool selectsRemoveStream, bool hasAggregations, TableService tableService, bool isOnDemandQuery, bool allowIndexInit)
{
// Determine if there is a historical stream, and what dependencies exist
var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false);
for (var i = 0; i < streamTypes.Length; i++)
{
if (historicalViewableDesc.Historical[i])
{
var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i];
historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn);
}
}
if (log.IsDebugEnabled)
{
log.Debug("Dependency graph: " + historicalDependencyGraph);
}
// Handle a join with a database or other historical data source for 2 streams
if ((historicalViewableDesc.HasHistorical) && (streamTypes.Length == 2))
{
return(MakeComposerHistorical2Stream(outerJoinDescList, optionalFilterNode, streamTypes, historicalViewableDesc, queryPlanLogging, exprEvaluatorContext, statementContext, streamNames, allowIndexInit));
}
var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList);
// Query graph for graph relationships between streams/historicals
// For outer joins the query graph will just contain outer join relationships
var hint = ExcludePlanHint.GetHint(streamNames, statementContext);
var queryGraph = new QueryGraph(streamTypes.Length, hint, false);
if (outerJoinDescList.Length > 0)
{
OuterJoinAnalyzer.Analyze(outerJoinDescList, queryGraph);
if (log.IsDebugEnabled)
{
log.Debug(".makeComposer After outer join queryGraph=\n" + queryGraph);
}
}
// Let the query graph reflect the where-clause
if (optionalFilterNode != null)
{
// Analyze relationships between streams using the optional filter expression.
// Relationships are properties in AND and EQUALS nodes of joins.
FilterExprAnalyzer.Analyze(optionalFilterNode, queryGraph, isOuterJoins);
if (log.IsDebugEnabled)
{
log.Debug(".makeComposer After filter expression queryGraph=\n" + queryGraph);
}
// Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c)
QueryGraph.FillEquivalentNav(streamTypes, queryGraph);
if (log.IsDebugEnabled)
{
log.Debug(".makeComposer After fill equiv. nav. queryGraph=\n" + queryGraph);
}
}
// Historical index lists
var historicalStreamIndexLists = new HistoricalStreamIndexList[streamTypes.Length];
var queryPlan = QueryPlanBuilder.GetPlan(streamTypes, outerJoinDescList, queryGraph, streamNames,
historicalViewableDesc, historicalDependencyGraph, historicalStreamIndexLists,
streamJoinAnalysisResult, queryPlanLogging, statementContext.Annotations, exprEvaluatorContext);
// remove unused indexes - consider all streams or all unidirectional
var usedIndexes = new HashSet <TableLookupIndexReqKey>();
var indexSpecs = queryPlan.IndexSpecs;
for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++)
{
var planNode = queryPlan.ExecNodeSpecs[streamNum];
if (planNode != null)
{
planNode.AddIndexes(usedIndexes);
}
}
foreach (var indexSpec in indexSpecs)
{
if (indexSpec == null)
{
continue;
}
var items = indexSpec.Items;
var indexNames = items.Keys.ToArray();
foreach (var indexName in indexNames)
{
if (!usedIndexes.Contains(indexName))
{
items.Remove(indexName);
}
}
}
var hook = QueryPlanIndexHookUtil.GetHook(statementContext.Annotations);
if (queryPlanLogging && (QueryPlanLog.IsInfoEnabled || hook != null))
{
QueryPlanLog.Info("Query plan: " + queryPlan.ToQueryPlan());
if (hook != null)
{
hook.Join(queryPlan);
}
}
// register index-use references for tables
if (!isOnDemandQuery)
{
foreach (var usedIndex in usedIndexes)
{
if (usedIndex.TableName != null)
{
tableService.GetTableMetadata(usedIndex.TableName).AddIndexReference(usedIndex.Name, statementName);
}
}
}
var joinRemoveStream = selectsRemoveStream || hasAggregations;
return(new JoinSetComposerPrototypeImpl(
statementName,
statementId,
outerJoinDescList,
optionalFilterNode,
streamTypes,
streamNames,
streamJoinAnalysisResult,
statementContext.Annotations,
historicalViewableDesc,
exprEvaluatorContext,
indexSpecs,
queryPlan,
historicalStreamIndexLists,
joinRemoveStream,
isOuterJoins,
tableService,
statementContext.EventTableIndexService));
}
public static QueryPlanForgeDesc GetPlan(
EventType[] typesPerStream,
OuterJoinDesc[] outerJoinDescList,
QueryGraphForge queryGraph,
string[] streamNames,
HistoricalViewableDesc historicalViewableDesc,
DependencyGraph dependencyGraph,
HistoricalStreamIndexListForge[] historicalStreamIndexLists,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
bool isQueryPlanLogging,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
string methodName = ".getPlan ";
int numStreams = typesPerStream.Length;
if (numStreams < 2) {
throw new ArgumentException("Number of join stream types is less then 2");
}
if (outerJoinDescList.Length >= numStreams) {
throw new ArgumentException("Too many outer join descriptors found");
}
if (numStreams == 2) {
OuterJoinType? outerJoinType = null;
if (outerJoinDescList.Length > 0) {
outerJoinType = outerJoinDescList[0].OuterJoinType;
}
QueryPlanForgeDesc queryPlan = TwoStreamQueryPlanBuilder.Build(
typesPerStream,
queryGraph,
outerJoinType,
streamJoinAnalysisResult,
statementRawInfo);
RemoveUnidirectionalAndTable(queryPlan.Forge, streamJoinAnalysisResult);
if (Log.IsDebugEnabled) {
Log.Debug(methodName + "2-Stream queryPlan=" + queryPlan);
}
return queryPlan;
}
bool hasPreferMergeJoin = HintEnum.PREFER_MERGE_JOIN.GetHint(statementRawInfo.Annotations) != null;
bool hasForceNestedIter = HintEnum.FORCE_NESTED_ITER.GetHint(statementRawInfo.Annotations) != null;
bool isAllInnerJoins = outerJoinDescList.Length == 0 ||
OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList);
if (isAllInnerJoins && !hasPreferMergeJoin) {
QueryPlanForgeDesc queryPlan = NStreamQueryPlanBuilder.Build(
queryGraph,
typesPerStream,
historicalViewableDesc,
dependencyGraph,
historicalStreamIndexLists,
hasForceNestedIter,
streamJoinAnalysisResult.UniqueKeys,
streamJoinAnalysisResult.TablesPerStream,
streamJoinAnalysisResult,
statementRawInfo,
services.SerdeResolver);
if (queryPlan != null) {
RemoveUnidirectionalAndTable(queryPlan.Forge, streamJoinAnalysisResult);
if (Log.IsDebugEnabled) {
Log.Debug(methodName + "N-Stream inner-join queryPlan=" + queryPlan);
}
return queryPlan;
}
if (isQueryPlanLogging && QUERY_PLAN_LOG.IsInfoEnabled) {
Log.Info("Switching to Outer-NStream algorithm for query plan");
}
}
QueryPlanForgeDesc queryPlanX = NStreamOuterQueryPlanBuilder.Build(
queryGraph,
outerJoinDescList,
streamNames,
typesPerStream,
historicalViewableDesc,
dependencyGraph,
historicalStreamIndexLists,
streamJoinAnalysisResult.UniqueKeys,
streamJoinAnalysisResult.TablesPerStream,
streamJoinAnalysisResult,
statementRawInfo,
services);
RemoveUnidirectionalAndTable(queryPlanX.Forge, streamJoinAnalysisResult);
return queryPlanX;
}
public static JoinSetComposerPrototypeDesc MakeComposerPrototype(
StatementSpecCompiled spec,
StreamJoinAnalysisResultCompileTime joinAnalysisResult,
StreamTypeService typeService,
HistoricalViewableDesc historicalViewableDesc,
bool isOnDemandQuery,
bool hasAggregations,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices compileTimeServices)
{
var streamTypes = typeService.EventTypes;
var streamNames = typeService.StreamNames;
var whereClause = spec.Raw.WhereClause;
var queryPlanLogging = compileTimeServices.Configuration.Common.Logging.IsEnableQueryPlan;
var additionalForgeables = new List<StmtClassForgeableFactory>();
// Determine if there is a historical stream, and what dependencies exist
var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false);
for (var i = 0; i < streamTypes.Length; i++) {
if (historicalViewableDesc.Historical[i]) {
var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i];
historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn);
}
}
if (Log.IsDebugEnabled) {
Log.Debug("Dependency graph: " + historicalDependencyGraph);
}
// Handle a join with a database or other historical data source for 2 streams
var outerJoinDescs = OuterJoinDesc.ToArray(spec.Raw.OuterJoinDescList);
if (historicalViewableDesc.IsHistorical && streamTypes.Length == 2) {
var desc = MakeComposerHistorical2Stream(
outerJoinDescs,
whereClause,
streamTypes,
streamNames,
historicalViewableDesc,
queryPlanLogging,
statementRawInfo,
compileTimeServices);
return new JoinSetComposerPrototypeDesc(desc.Forge, desc.AdditionalForgeables);
}
var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescs);
// Query graph for graph relationships between streams/historicals
// For outer joins the query graph will just contain outer join relationships
var hint = ExcludePlanHint.GetHint(
typeService.StreamNames,
statementRawInfo,
compileTimeServices);
var queryGraph = new QueryGraphForge(streamTypes.Length, hint, false);
if (outerJoinDescs.Length > 0) {
OuterJoinAnalyzer.Analyze(outerJoinDescs, queryGraph);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After outer join filterQueryGraph=\n" + queryGraph);
}
}
// Let the query graph reflect the where-clause
if (whereClause != null) {
// Analyze relationships between streams using the optional filter expression.
// Relationships are properties in AND and EQUALS nodes of joins.
FilterExprAnalyzer.Analyze(whereClause, queryGraph, isOuterJoins);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After filter expression filterQueryGraph=\n" + queryGraph);
}
// Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c)
QueryGraphForge.FillEquivalentNav(streamTypes, queryGraph);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After fill equiv. nav. filterQueryGraph=\n" + queryGraph);
}
}
// Historical index lists
var historicalStreamIndexLists =
new HistoricalStreamIndexListForge[streamTypes.Length];
var queryPlanDesc = QueryPlanBuilder.GetPlan(
streamTypes,
outerJoinDescs,
queryGraph,
typeService.StreamNames,
historicalViewableDesc,
historicalDependencyGraph,
historicalStreamIndexLists,
joinAnalysisResult,
queryPlanLogging,
statementRawInfo,
compileTimeServices);
QueryPlanForge queryPlan = queryPlanDesc.Forge;
additionalForgeables.AddAll(queryPlanDesc.AdditionalForgeables);
// remove unused indexes - consider all streams or all unidirectional
var usedIndexes = new HashSet<TableLookupIndexReqKey>();
var indexSpecs = queryPlan.IndexSpecs;
for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) {
var planNode = queryPlan.ExecNodeSpecs[streamNum];
planNode?.AddIndexes(usedIndexes);
}
foreach (var indexSpec in indexSpecs) {
if (indexSpec == null) {
continue;
}
var items = indexSpec.Items;
var indexNames = items.Keys.ToArray();
foreach (var indexName in indexNames) {
if (!usedIndexes.Contains(indexName)) {
items.Remove(indexName);
}
}
}
// plan multikeys
IList<StmtClassForgeableFactory> multikeyForgeables = PlanMultikeys(
indexSpecs, statementRawInfo, compileTimeServices);
additionalForgeables.AddAll(multikeyForgeables);
QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook(
spec.Annotations,
compileTimeServices.ImportServiceCompileTime);
if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) {
QUERY_PLAN_LOG.Info("Query plan: " + queryPlan.ToQueryPlan());
hook?.Join(queryPlan);
}
var selectsRemoveStream =
spec.Raw.SelectStreamSelectorEnum.IsSelectsRStream() || spec.Raw.OutputLimitSpec != null;
var joinRemoveStream = selectsRemoveStream || hasAggregations;
ExprNode postJoinEvaluator;
if (JoinSetComposerUtil.IsNonUnidirectionalNonSelf(
isOuterJoins,
joinAnalysisResult.IsUnidirectional,
joinAnalysisResult.IsPureSelfJoin)) {
postJoinEvaluator = GetFilterExpressionInclOnClause(
spec.Raw.WhereClause,
outerJoinDescs,
statementRawInfo,
compileTimeServices);
}
else {
postJoinEvaluator = spec.Raw.WhereClause;
}
JoinSetComposerPrototypeGeneralForge forge = new JoinSetComposerPrototypeGeneralForge(
typeService.EventTypes,
postJoinEvaluator,
outerJoinDescs.Length > 0,
queryPlan,
joinAnalysisResult,
typeService.StreamNames,
joinRemoveStream,
historicalViewableDesc.IsHistorical);
return new JoinSetComposerPrototypeDesc(forge, additionalForgeables);
}
