public static void QueryPlanLogOnSubq(bool queryPlanLogging, ILog queryPlanLog, SubordinateQueryPlanDesc plan, int subqueryNum, Attribute[] annotations)
{
var hook = QueryPlanIndexHookUtil.GetHook(annotations);
if (queryPlanLogging && (queryPlanLog.IsInfoEnabled || hook != null))
{
var prefix = "Subquery " + subqueryNum + " ";
var strategy = (plan == null || plan.LookupStrategyFactory == null) ? "table scan" : plan.LookupStrategyFactory.ToQueryPlan();
queryPlanLog.Info(prefix + "strategy " + strategy);
if (plan != null)
{
if (plan.IndexDescs != null)
{
for (var i = 0; i < plan.IndexDescs.Length; i++)
{
var indexName = plan.IndexDescs[i].IndexName;
var indexText = indexName != null ? "index " + indexName + " " : "(implicit) ";
queryPlanLog.Info(prefix + "shared index");
queryPlanLog.Info(prefix + indexText);
}
}
}
if (hook != null)
{
var pairs = plan == null ? new IndexNameAndDescPair[0] : GetPairs(plan.IndexDescs);
string factory = plan == null ? null : plan.LookupStrategyFactory.GetType().Name;
hook.Subquery(new QueryPlanIndexDescSubquery(pairs, subqueryNum, factory));
}
}
}
private static void QueryPlanReport(
string indexNameOrNull,
EventTable eventTableOrNull,
Attribute[] annotations,
AgentInstanceContext agentInstanceContext,
string objectName)
{
var hook = QueryPlanIndexHookUtil.GetHook(annotations, agentInstanceContext.ImportServiceRuntime);
var queryPlanLogging = agentInstanceContext.RuntimeSettingsService.ConfigurationCommon.Logging
.IsEnableQueryPlan;
if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) {
var prefix = "Fire-and-forget or init-time-query from " + objectName + " ";
var indexText = indexNameOrNull != null ? "index " + indexNameOrNull + " " : "full table scan ";
indexText += "(snapshot only, for join see separate query plan) ";
if (eventTableOrNull == null) {
QUERY_PLAN_LOG.Info(prefix + indexText);
}
else {
QUERY_PLAN_LOG.Info(prefix + indexText + eventTableOrNull.ToQueryPlan());
}
hook?.FireAndForget(
new QueryPlanIndexDescFAF(
new[] {
new IndexNameAndDescPair(
indexNameOrNull,
eventTableOrNull != null ? eventTableOrNull.ProviderClass.Name : null)
}));
}
}
public static void QueryPlanLogOnExpr(bool queryPlanLogging, ILog queryPlanLog, SubordinateWMatchExprQueryPlanResult strategy, Attribute[] annotations)
{
var hook = QueryPlanIndexHookUtil.GetHook(annotations);
if (queryPlanLogging && (queryPlanLog.IsInfoEnabled || hook != null))
{
var prefix = "On-Expr ";
queryPlanLog.Info(prefix + "strategy " + strategy.Factory.ToQueryPlan());
if (strategy.IndexDescs == null)
{
queryPlanLog.Info(prefix + "full table scan");
}
else
{
for (var i = 0; i < strategy.IndexDescs.Length; i++)
{
var indexName = strategy.IndexDescs[i].IndexName;
var indexText = indexName != null ? "index " + indexName + " " : "(implicit) (" + i + ")";
queryPlanLog.Info(prefix + indexText);
}
}
if (hook != null)
{
var pairs = GetPairs(strategy.IndexDescs);
var inner = strategy.Factory.OptionalInnerStrategy;
hook.InfraOnExpr(new QueryPlanIndexDescOnExpr(pairs,
strategy.Factory.GetType().Name,
inner == null ? null : inner.GetType().Name));
}
}
}
/// <summary>
/// The queryPlanReport
/// </summary>
/// <param name="indexNameOrNull">The <see cref="string"/></param>
/// <param name="eventTableOrNull">The <see cref="EventTable"/></param>
/// <param name="attributes">The <see cref="Attribute"/> array</param>
/// <param name="agentInstanceContext">The <see cref="AgentInstanceContext"/></param>
/// <param name="queryPlanLogging">The <see cref="bool"/></param>
/// <param name="queryPlanLogDestination">The <see cref="ILog"/></param>
/// <param name="objectName">The <see cref="string"/></param>
private static void queryPlanReport(
string indexNameOrNull,
EventTable eventTableOrNull,
Attribute[] attributes,
AgentInstanceContext agentInstanceContext,
bool queryPlanLogging,
ILog queryPlanLogDestination,
string objectName)
{
var hook = QueryPlanIndexHookUtil.GetHook(attributes, agentInstanceContext.StatementContext.EngineImportService);
if (queryPlanLogging && (queryPlanLogDestination.IsInfoEnabled || hook != null))
{
var prefix = "Fire-and-forget from " + objectName + " ";
var indexText = indexNameOrNull != null ? "index " + indexNameOrNull + " " : "full table scan ";
indexText += "(snapshot only, for join see separate query plan) ";
if (eventTableOrNull == null)
{
queryPlanLogDestination.Info(prefix + indexText);
}
else
{
queryPlanLogDestination.Info(prefix + indexText + eventTableOrNull.ToQueryPlan());
}
if (hook != null)
{
hook.FireAndForget(new QueryPlanIndexDescFAF(
new IndexNameAndDescPair[] {
new IndexNameAndDescPair(indexNameOrNull, eventTableOrNull != null ? eventTableOrNull.ProviderClass.Name : null)
}));
}
}
}
public static void QueryPlanLogOnExpr(
bool queryPlanLogging,
ILog queryPlanLog,
SubordinateWMatchExprQueryPlanForge strategy,
Attribute[] annotations,
ImportService importService)
{
QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook(annotations, importService);
if (queryPlanLogging && (queryPlanLog.IsInfoEnabled || hook != null)) {
var prefix = "On-Expr ";
queryPlanLog.Info(prefix + "strategy " + strategy.Strategy.ToQueryPlan());
if (strategy.Indexes == null) {
queryPlanLog.Info(prefix + "full table scan");
}
else {
for (var i = 0; i < strategy.Indexes.Length; i++) {
string indexName = strategy.Indexes[i].IndexName;
var indexText = indexName != null ? "index " + indexName + " " : "(implicit) (" + i + ")";
queryPlanLog.Info(prefix + indexText);
}
}
if (hook != null) {
var pairs = GetPairs(strategy.Indexes);
SubordTableLookupStrategyFactoryForge inner = strategy.Strategy.OptionalInnerStrategy;
hook.InfraOnExpr(
new QueryPlanIndexDescOnExpr(
pairs,
strategy.Strategy.GetType().GetSimpleName(),
inner == null ? null : inner.GetType().GetSimpleName()));
}
}
}
public static ICollection <EventBean> Snapshot(
FilterSpecCompiled optionalFilter,
Attribute[] annotations,
VirtualDWView virtualDataWindow,
EventTableIndexRepository indexRepository,
bool queryPlanLogging,
ILog queryPlanLogDestination,
string objectName,
AgentInstanceContext agentInstanceContext)
{
if (optionalFilter == null || optionalFilter.Parameters.Length == 0)
{
if (virtualDataWindow != null)
{
var pair = virtualDataWindow.GetFireAndForgetDesc(Collections.GetEmptySet <string>(), Collections.GetEmptySet <string>());
return(virtualDataWindow.GetFireAndForgetData(pair.Second, new object[0], new RangeIndexLookupValue[0], annotations));
}
return(null);
}
// Determine what straight-equals keys and which ranges are available.
// Widening/Coercion is part of filter spec compile.
ISet <string> keysAvailable = new HashSet <string>();
ISet <string> rangesAvailable = new HashSet <string>();
if (optionalFilter.Parameters.Length == 1)
{
foreach (FilterSpecParam param in optionalFilter.Parameters[0])
{
if (!(param is FilterSpecParamConstant ||
param is FilterSpecParamRange ||
param is FilterSpecParamIn))
{
continue;
}
if (param.FilterOperator == FilterOperator.EQUAL ||
param.FilterOperator == FilterOperator.IS ||
param.FilterOperator == FilterOperator.IN_LIST_OF_VALUES)
{
keysAvailable.Add(param.Lookupable.Expression);
}
else if (param.FilterOperator.IsRangeOperator() ||
param.FilterOperator.IsInvertedRangeOperator() ||
param.FilterOperator.IsComparisonOperator())
{
rangesAvailable.Add(param.Lookupable.Expression);
}
else if (param.FilterOperator.IsRangeOperator())
{
rangesAvailable.Add(param.Lookupable.Expression);
}
}
}
// Find an index that matches the needs
Pair <IndexMultiKey, EventTableAndNamePair> tablePair;
if (virtualDataWindow != null)
{
var tablePairNoName = virtualDataWindow.GetFireAndForgetDesc(keysAvailable, rangesAvailable);
tablePair = new Pair <IndexMultiKey, EventTableAndNamePair>(tablePairNoName.First, new EventTableAndNamePair(tablePairNoName.Second, null));
}
else
{
var indexHint = IndexHint.GetIndexHint(annotations);
IList <IndexHintInstruction> optionalIndexHintInstructions = null;
if (indexHint != null)
{
optionalIndexHintInstructions = indexHint.InstructionsFireAndForget;
}
tablePair = indexRepository.FindTable(keysAvailable, rangesAvailable, optionalIndexHintInstructions);
}
var hook = QueryPlanIndexHookUtil.GetHook(annotations);
if (queryPlanLogging && (queryPlanLogDestination.IsInfoEnabled || hook != null))
{
var prefix = "Fire-and-forget from " + objectName + " ";
var indexName = tablePair != null && tablePair.Second != null ? tablePair.Second.IndexName : null;
var indexText = indexName != null ? "index " + indexName + " " : "full table scan ";
indexText += "(snapshot only, for join see separate query plan)";
if (tablePair == null)
{
queryPlanLogDestination.Info(prefix + indexText);
}
else
{
queryPlanLogDestination.Info(prefix + indexText + tablePair.Second.EventTable.ToQueryPlan());
}
if (hook != null)
{
hook.FireAndForget(new QueryPlanIndexDescFAF(
new IndexNameAndDescPair[] {
new IndexNameAndDescPair(indexName, tablePair != null ?
tablePair.Second.EventTable.ProviderClass.Name : null)
}));
}
}
if (tablePair == null)
{
return(null); // indicates table scan
}
// Compile key sets which contain key index lookup values
var keyIndexProps = IndexedPropDesc.GetIndexProperties(tablePair.First.HashIndexedProps);
var hasKeyWithInClause = false;
var keyValues = new object[keyIndexProps.Length];
for (var keyIndex = 0; keyIndex < keyIndexProps.Length; keyIndex++)
{
foreach (var param in optionalFilter.Parameters[0])
{
if (param.Lookupable.Expression.Equals(keyIndexProps[keyIndex]))
{
if (param.FilterOperator == FilterOperator.IN_LIST_OF_VALUES)
{
var keyValuesList = ((MultiKeyUntyped)param.GetFilterValue(null, agentInstanceContext)).Keys;
if (keyValuesList.Length == 0)
{
continue;
}
else if (keyValuesList.Length == 1)
{
keyValues[keyIndex] = keyValuesList[0];
}
else
{
keyValues[keyIndex] = keyValuesList;
hasKeyWithInClause = true;
}
}
else
{
keyValues[keyIndex] = param.GetFilterValue(null, agentInstanceContext);
}
break;
}
}
}
// Analyze ranges - these may include key lookup value (EQUALS semantics)
var rangeIndexProps = IndexedPropDesc.GetIndexProperties(tablePair.First.RangeIndexedProps);
RangeIndexLookupValue[] rangeValues;
if (rangeIndexProps.Length > 0)
{
rangeValues = CompileRangeLookupValues(rangeIndexProps, optionalFilter.Parameters[0], agentInstanceContext);
}
else
{
rangeValues = new RangeIndexLookupValue[0];
}
var eventTable = tablePair.Second.EventTable;
var indexMultiKey = tablePair.First;
// table lookup without in-clause
if (!hasKeyWithInClause)
{
return(FafTableLookup(virtualDataWindow, indexMultiKey, eventTable, keyValues, rangeValues, annotations));
}
// table lookup with in-clause: determine combinations
var combinations = new object[keyIndexProps.Length][];
for (var i = 0; i < keyValues.Length; i++)
{
if (keyValues[i] is object[])
{
combinations[i] = (object[])keyValues[i];
}
else
{
combinations[i] = new object[] { keyValues[i] };
}
}
// enumerate combinations
var enumeration = new CombinationEnumeration(combinations);
var events = new HashSet <EventBean>();
for (; enumeration.MoveNext();)
{
object[] keys = enumeration.Current;
var result = FafTableLookup(virtualDataWindow, indexMultiKey, eventTable, keys, rangeValues, annotations);
events.AddAll(result);
}
return(events);
}
/// <summary>
/// Builds join tuple composer.
/// </summary>
/// <param name="statementName">Name of the statement.</param>
/// <param name="statementId">The statement identifier.</param>
/// <param name="outerJoinDescList">list of descriptors for outer join criteria</param>
/// <param name="optionalFilterNode">filter tree for analysis to build indexes for fast access</param>
/// <param name="streamTypes">types of streams</param>
/// <param name="streamNames">names of streams</param>
/// <param name="streamJoinAnalysisResult">The stream join analysis result.</param>
/// <param name="queryPlanLogging">if set to <c>true</c> [query plan logging].</param>
/// <param name="statementContext">The statement context.</param>
/// <param name="historicalViewableDesc">The historical viewable desc.</param>
/// <param name="exprEvaluatorContext">The expr evaluator context.</param>
/// <param name="selectsRemoveStream">if set to <c>true</c> [selects remove stream].</param>
/// <param name="hasAggregations">if set to <c>true</c> [has aggregations].</param>
/// <param name="tableService">The table service.</param>
/// <param name="isOnDemandQuery">if set to <c>true</c> [is on demand query].</param>
/// <param name="allowIndexInit">if set to <c>true</c> [allow index initialize].</param>
/// <returns>
/// composer implementation
/// </returns>
/// <throws>com.espertech.esper.epl.expression.core.ExprValidationException is thrown to indicate thatvalidation of view use in joins failed.
/// {D255958A-8513-4226-94B9-080D98F904A1}</throws>
public static JoinSetComposerPrototype MakeComposerPrototype(string statementName, int statementId, OuterJoinDesc[] outerJoinDescList, ExprNode optionalFilterNode, EventType[] streamTypes, string[] streamNames, StreamJoinAnalysisResult streamJoinAnalysisResult, bool queryPlanLogging, StatementContext statementContext, HistoricalViewableDesc historicalViewableDesc, ExprEvaluatorContext exprEvaluatorContext, bool selectsRemoveStream, bool hasAggregations, TableService tableService, bool isOnDemandQuery, bool allowIndexInit)
{
// Determine if there is a historical stream, and what dependencies exist
var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false);
for (var i = 0; i < streamTypes.Length; i++)
{
if (historicalViewableDesc.Historical[i])
{
var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i];
historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn);
}
}
if (log.IsDebugEnabled)
{
log.Debug("Dependency graph: " + historicalDependencyGraph);
}
// Handle a join with a database or other historical data source for 2 streams
if ((historicalViewableDesc.HasHistorical) && (streamTypes.Length == 2))
{
return(MakeComposerHistorical2Stream(outerJoinDescList, optionalFilterNode, streamTypes, historicalViewableDesc, queryPlanLogging, exprEvaluatorContext, statementContext, streamNames, allowIndexInit));
}
var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList);
// Query graph for graph relationships between streams/historicals
// For outer joins the query graph will just contain outer join relationships
var hint = ExcludePlanHint.GetHint(streamNames, statementContext);
var queryGraph = new QueryGraph(streamTypes.Length, hint, false);
if (outerJoinDescList.Length > 0)
{
OuterJoinAnalyzer.Analyze(outerJoinDescList, queryGraph);
if (log.IsDebugEnabled)
{
log.Debug(".makeComposer After outer join queryGraph=\n" + queryGraph);
}
}
// Let the query graph reflect the where-clause
if (optionalFilterNode != null)
{
// Analyze relationships between streams using the optional filter expression.
// Relationships are properties in AND and EQUALS nodes of joins.
FilterExprAnalyzer.Analyze(optionalFilterNode, queryGraph, isOuterJoins);
if (log.IsDebugEnabled)
{
log.Debug(".makeComposer After filter expression queryGraph=\n" + queryGraph);
}
// Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c)
QueryGraph.FillEquivalentNav(streamTypes, queryGraph);
if (log.IsDebugEnabled)
{
log.Debug(".makeComposer After fill equiv. nav. queryGraph=\n" + queryGraph);
}
}
// Historical index lists
var historicalStreamIndexLists = new HistoricalStreamIndexList[streamTypes.Length];
var queryPlan = QueryPlanBuilder.GetPlan(streamTypes, outerJoinDescList, queryGraph, streamNames,
historicalViewableDesc, historicalDependencyGraph, historicalStreamIndexLists,
streamJoinAnalysisResult, queryPlanLogging, statementContext.Annotations, exprEvaluatorContext);
// remove unused indexes - consider all streams or all unidirectional
var usedIndexes = new HashSet <TableLookupIndexReqKey>();
var indexSpecs = queryPlan.IndexSpecs;
for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++)
{
var planNode = queryPlan.ExecNodeSpecs[streamNum];
if (planNode != null)
{
planNode.AddIndexes(usedIndexes);
}
}
foreach (var indexSpec in indexSpecs)
{
if (indexSpec == null)
{
continue;
}
var items = indexSpec.Items;
var indexNames = items.Keys.ToArray();
foreach (var indexName in indexNames)
{
if (!usedIndexes.Contains(indexName))
{
items.Remove(indexName);
}
}
}
var hook = QueryPlanIndexHookUtil.GetHook(statementContext.Annotations);
if (queryPlanLogging && (QueryPlanLog.IsInfoEnabled || hook != null))
{
QueryPlanLog.Info("Query plan: " + queryPlan.ToQueryPlan());
if (hook != null)
{
hook.Join(queryPlan);
}
}
// register index-use references for tables
if (!isOnDemandQuery)
{
foreach (var usedIndex in usedIndexes)
{
if (usedIndex.TableName != null)
{
tableService.GetTableMetadata(usedIndex.TableName).AddIndexReference(usedIndex.Name, statementName);
}
}
}
var joinRemoveStream = selectsRemoveStream || hasAggregations;
return(new JoinSetComposerPrototypeImpl(
statementName,
statementId,
outerJoinDescList,
optionalFilterNode,
streamTypes,
streamNames,
streamJoinAnalysisResult,
statementContext.Annotations,
historicalViewableDesc,
exprEvaluatorContext,
indexSpecs,
queryPlan,
historicalStreamIndexLists,
joinRemoveStream,
isOuterJoins,
tableService,
statementContext.EventTableIndexService));
}
private static JoinSetComposerPrototype MakeComposerHistorical2Stream(OuterJoinDesc[] outerJoinDescList, ExprNode optionalFilterNode, EventType[] streamTypes, HistoricalViewableDesc historicalViewableDesc, bool queryPlanLogging, ExprEvaluatorContext exprEvaluatorContext, StatementContext statementContext, string[] streamNames, bool allowIndexInit)
{
var polledViewNum = 0;
var streamViewNum = 1;
if (historicalViewableDesc.Historical[1])
{
streamViewNum = 0;
polledViewNum = 1;
}
// if all-historical join, check dependency
var isAllHistoricalNoSubordinate = false;
if ((historicalViewableDesc.Historical[0]) && historicalViewableDesc.Historical[1])
{
var graph = new DependencyGraph(2, false);
graph.AddDependency(0, historicalViewableDesc.DependenciesPerHistorical[0]);
graph.AddDependency(1, historicalViewableDesc.DependenciesPerHistorical[1]);
if (graph.FirstCircularDependency != null)
{
throw new ExprValidationException("Circular dependency detected between historical streams");
}
// if both streams are independent
if (graph.RootNodes.Count == 2)
{
isAllHistoricalNoSubordinate = true; // No parameters used by either historical
}
else
{
if ((graph.GetDependenciesForStream(0).Count == 0))
{
streamViewNum = 0;
polledViewNum = 1;
}
else
{
streamViewNum = 1;
polledViewNum = 0;
}
}
}
// Build an outer join expression node
var isOuterJoin = false;
var isInnerJoinOnly = false;
ExprNode outerJoinEqualsNode = null;
if (outerJoinDescList.Length > 0)
{
var outerJoinDesc = outerJoinDescList[0];
isInnerJoinOnly = outerJoinDesc.OuterJoinType == OuterJoinType.INNER;
if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.FULL))
{
isOuterJoin = true;
}
else if ((outerJoinDesc.OuterJoinType.Equals(OuterJoinType.LEFT)) &&
(streamViewNum == 0))
{
isOuterJoin = true;
}
else if ((outerJoinDesc.OuterJoinType.Equals(OuterJoinType.RIGHT)) &&
(streamViewNum == 1))
{
isOuterJoin = true;
}
outerJoinEqualsNode = outerJoinDesc.MakeExprNode(exprEvaluatorContext);
}
// Determine filter for indexing purposes
ExprNode filterForIndexing = null;
if ((outerJoinEqualsNode != null) && (optionalFilterNode != null) && isInnerJoinOnly) // both filter and outer join, add
{
filterForIndexing = new ExprAndNodeImpl();
filterForIndexing.AddChildNode(optionalFilterNode);
filterForIndexing.AddChildNode(outerJoinEqualsNode);
}
else if ((outerJoinEqualsNode == null) && (optionalFilterNode != null))
{
filterForIndexing = optionalFilterNode;
}
else if (outerJoinEqualsNode != null)
{
filterForIndexing = outerJoinEqualsNode;
}
var indexStrategies =
DetermineIndexing(filterForIndexing, streamTypes[polledViewNum], streamTypes[streamViewNum], polledViewNum, streamViewNum, statementContext, streamNames);
var hook = QueryPlanIndexHookUtil.GetHook(statementContext.Annotations);
if (queryPlanLogging && (QueryPlanLog.IsInfoEnabled || hook != null))
{
QueryPlanLog.Info("historical lookup strategy: " + indexStrategies.First.ToQueryPlan());
QueryPlanLog.Info("historical index strategy: " + indexStrategies.Second.ToQueryPlan());
if (hook != null)
{
hook.Historical(new QueryPlanIndexDescHistorical(indexStrategies.First.GetType().Name, indexStrategies.Second.GetType().Name));
}
}
return(new JoinSetComposerPrototypeHistorical2StreamImpl(
optionalFilterNode,
streamTypes,
exprEvaluatorContext,
polledViewNum,
streamViewNum,
isOuterJoin,
outerJoinEqualsNode,
indexStrategies,
isAllHistoricalNoSubordinate,
outerJoinDescList,
allowIndexInit));
}
public static JoinSetComposerPrototypeDesc MakeComposerPrototype(
StatementSpecCompiled spec,
StreamJoinAnalysisResultCompileTime joinAnalysisResult,
StreamTypeService typeService,
HistoricalViewableDesc historicalViewableDesc,
bool isOnDemandQuery,
bool hasAggregations,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices compileTimeServices)
{
var streamTypes = typeService.EventTypes;
var streamNames = typeService.StreamNames;
var whereClause = spec.Raw.WhereClause;
var queryPlanLogging = compileTimeServices.Configuration.Common.Logging.IsEnableQueryPlan;
var additionalForgeables = new List<StmtClassForgeableFactory>();
// Determine if there is a historical stream, and what dependencies exist
var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false);
for (var i = 0; i < streamTypes.Length; i++) {
if (historicalViewableDesc.Historical[i]) {
var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i];
historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn);
}
}
if (Log.IsDebugEnabled) {
Log.Debug("Dependency graph: " + historicalDependencyGraph);
}
// Handle a join with a database or other historical data source for 2 streams
var outerJoinDescs = OuterJoinDesc.ToArray(spec.Raw.OuterJoinDescList);
if (historicalViewableDesc.IsHistorical && streamTypes.Length == 2) {
var desc = MakeComposerHistorical2Stream(
outerJoinDescs,
whereClause,
streamTypes,
streamNames,
historicalViewableDesc,
queryPlanLogging,
statementRawInfo,
compileTimeServices);
return new JoinSetComposerPrototypeDesc(desc.Forge, desc.AdditionalForgeables);
}
var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescs);
// Query graph for graph relationships between streams/historicals
// For outer joins the query graph will just contain outer join relationships
var hint = ExcludePlanHint.GetHint(
typeService.StreamNames,
statementRawInfo,
compileTimeServices);
var queryGraph = new QueryGraphForge(streamTypes.Length, hint, false);
if (outerJoinDescs.Length > 0) {
OuterJoinAnalyzer.Analyze(outerJoinDescs, queryGraph);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After outer join filterQueryGraph=\n" + queryGraph);
}
}
// Let the query graph reflect the where-clause
if (whereClause != null) {
// Analyze relationships between streams using the optional filter expression.
// Relationships are properties in AND and EQUALS nodes of joins.
FilterExprAnalyzer.Analyze(whereClause, queryGraph, isOuterJoins);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After filter expression filterQueryGraph=\n" + queryGraph);
}
// Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c)
QueryGraphForge.FillEquivalentNav(streamTypes, queryGraph);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After fill equiv. nav. filterQueryGraph=\n" + queryGraph);
}
}
// Historical index lists
var historicalStreamIndexLists =
new HistoricalStreamIndexListForge[streamTypes.Length];
var queryPlanDesc = QueryPlanBuilder.GetPlan(
streamTypes,
outerJoinDescs,
queryGraph,
typeService.StreamNames,
historicalViewableDesc,
historicalDependencyGraph,
historicalStreamIndexLists,
joinAnalysisResult,
queryPlanLogging,
statementRawInfo,
compileTimeServices);
QueryPlanForge queryPlan = queryPlanDesc.Forge;
additionalForgeables.AddAll(queryPlanDesc.AdditionalForgeables);
// remove unused indexes - consider all streams or all unidirectional
var usedIndexes = new HashSet<TableLookupIndexReqKey>();
var indexSpecs = queryPlan.IndexSpecs;
for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) {
var planNode = queryPlan.ExecNodeSpecs[streamNum];
planNode?.AddIndexes(usedIndexes);
}
foreach (var indexSpec in indexSpecs) {
if (indexSpec == null) {
continue;
}
var items = indexSpec.Items;
var indexNames = items.Keys.ToArray();
foreach (var indexName in indexNames) {
if (!usedIndexes.Contains(indexName)) {
items.Remove(indexName);
}
}
}
// plan multikeys
IList<StmtClassForgeableFactory> multikeyForgeables = PlanMultikeys(
indexSpecs, statementRawInfo, compileTimeServices);
additionalForgeables.AddAll(multikeyForgeables);
QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook(
spec.Annotations,
compileTimeServices.ImportServiceCompileTime);
if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) {
QUERY_PLAN_LOG.Info("Query plan: " + queryPlan.ToQueryPlan());
hook?.Join(queryPlan);
}
var selectsRemoveStream =
spec.Raw.SelectStreamSelectorEnum.IsSelectsRStream() || spec.Raw.OutputLimitSpec != null;
var joinRemoveStream = selectsRemoveStream || hasAggregations;
ExprNode postJoinEvaluator;
if (JoinSetComposerUtil.IsNonUnidirectionalNonSelf(
isOuterJoins,
joinAnalysisResult.IsUnidirectional,
joinAnalysisResult.IsPureSelfJoin)) {
postJoinEvaluator = GetFilterExpressionInclOnClause(
spec.Raw.WhereClause,
outerJoinDescs,
statementRawInfo,
compileTimeServices);
}
else {
postJoinEvaluator = spec.Raw.WhereClause;
}
JoinSetComposerPrototypeGeneralForge forge = new JoinSetComposerPrototypeGeneralForge(
typeService.EventTypes,
postJoinEvaluator,
outerJoinDescs.Length > 0,
queryPlan,
joinAnalysisResult,
typeService.StreamNames,
joinRemoveStream,
historicalViewableDesc.IsHistorical);
return new JoinSetComposerPrototypeDesc(forge, additionalForgeables);
}
private static JoinSetComposerPrototypeHistorical2StreamDesc MakeComposerHistorical2Stream(
OuterJoinDesc[] outerJoinDescs,
ExprNode whereClause,
EventType[] streamTypes,
string[] streamNames,
HistoricalViewableDesc historicalViewableDesc,
bool queryPlanLogging,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
var polledViewNum = 0;
var streamViewNum = 1;
if (historicalViewableDesc.Historical[1]) {
streamViewNum = 0;
polledViewNum = 1;
}
// if all-historical join, check dependency
var isAllHistoricalNoSubordinate = false;
if (historicalViewableDesc.Historical[0] && historicalViewableDesc.Historical[1]) {
var graph = new DependencyGraph(2, false);
graph.AddDependency(0, historicalViewableDesc.DependenciesPerHistorical[0]);
graph.AddDependency(1, historicalViewableDesc.DependenciesPerHistorical[1]);
if (graph.FirstCircularDependency != null) {
throw new ExprValidationException("Circular dependency detected between historical streams");
}
// if both streams are independent
if (graph.RootNodes.Count == 2) {
isAllHistoricalNoSubordinate = true; // No parameters used by either historical
}
else {
if (graph.GetDependenciesForStream(0).Count == 0) {
streamViewNum = 0;
polledViewNum = 1;
}
else {
streamViewNum = 1;
polledViewNum = 0;
}
}
}
// Build an outer join expression node
var isOuterJoin = false;
ExprNode outerJoinEqualsNode = null;
var isInnerJoinOnly = false;
var outerJoinPerStream = new bool[2];
if (outerJoinDescs != null && outerJoinDescs.Length > 0) {
var outerJoinDesc = outerJoinDescs[0];
isInnerJoinOnly = outerJoinDesc.OuterJoinType.Equals(OuterJoinType.INNER);
if (isAllHistoricalNoSubordinate) {
if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.FULL)) {
isOuterJoin = true;
outerJoinPerStream[0] = true;
outerJoinPerStream[1] = true;
}
else if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.LEFT)) {
isOuterJoin = true;
outerJoinPerStream[0] = true;
}
else if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.RIGHT)) {
isOuterJoin = true;
outerJoinPerStream[1] = true;
}
}
else {
if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.FULL)) {
isOuterJoin = true;
outerJoinPerStream[0] = true;
outerJoinPerStream[1] = true;
}
else if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.LEFT) &&
streamViewNum == 0) {
isOuterJoin = true;
outerJoinPerStream[0] = true;
}
else if (outerJoinDesc.OuterJoinType.Equals(OuterJoinType.RIGHT) &&
streamViewNum == 1) {
isOuterJoin = true;
outerJoinPerStream[1] = true;
}
}
outerJoinEqualsNode = outerJoinDesc.MakeExprNode(statementRawInfo, services);
}
// Determine filter for indexing purposes
ExprNode filterForIndexing = null;
if (outerJoinEqualsNode != null && whereClause != null && isInnerJoinOnly) {
// both filter and outer join, add
filterForIndexing = new ExprAndNodeImpl();
filterForIndexing.AddChildNode(whereClause);
filterForIndexing.AddChildNode(outerJoinEqualsNode);
}
else if (outerJoinEqualsNode == null && whereClause != null) {
filterForIndexing = whereClause;
}
else if (outerJoinEqualsNode != null) {
filterForIndexing = outerJoinEqualsNode;
}
var indexStrategies =
DetermineIndexing(
filterForIndexing,
streamTypes[polledViewNum],
streamTypes[streamViewNum],
polledViewNum,
streamViewNum,
streamNames,
statementRawInfo,
services);
QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook(
statementRawInfo.Annotations,
services.ImportServiceCompileTime);
if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) {
QUERY_PLAN_LOG.Info("historical lookup strategy: " + indexStrategies.LookupForge.ToQueryPlan());
QUERY_PLAN_LOG.Info("historical index strategy: " + indexStrategies.IndexingForge.ToQueryPlan());
hook?.Historical(
new QueryPlanIndexDescHistorical(
indexStrategies.LookupForge.GetType().GetSimpleName(),
indexStrategies.IndexingForge.GetType().GetSimpleName()));
}
JoinSetComposerPrototypeHistorical2StreamForge forge = new JoinSetComposerPrototypeHistorical2StreamForge(
streamTypes,
whereClause,
isOuterJoin,
polledViewNum,
streamViewNum,
outerJoinEqualsNode,
indexStrategies.LookupForge,
indexStrategies.IndexingForge,
isAllHistoricalNoSubordinate,
outerJoinPerStream);
return new JoinSetComposerPrototypeHistorical2StreamDesc(
forge, indexStrategies.AdditionalForgeables);
}
