public static void ValidateFilterWQueryGraphSafe(
QueryGraphForge queryGraph,
ExprNode filterExpression,
StreamTypeServiceImpl typeService,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
try {
var validationContext = new ExprValidationContextBuilder(typeService, statementRawInfo, services)
.WithAllowBindingConsumption(true)
.WithIsFilterExpression(true)
.Build();
var validated = ExprNodeUtilityValidate.GetValidatedSubtree(
ExprNodeOrigin.FILTER,
filterExpression,
validationContext);
FilterExprAnalyzer.Analyze(validated, queryGraph, false);
}
catch (Exception ex) {
Log.Warn(
"Unexpected exception analyzing filterable expression '" +
ExprNodeUtilityPrint.ToExpressionStringMinPrecedenceSafe(filterExpression) +
"': " +
ex.Message,
ex);
}
}
private static Pair <HistoricalIndexLookupStrategy, PollResultIndexingStrategy> DetermineIndexing(
ExprNode filterForIndexing,
EventType polledViewType,
EventType streamViewType,
int polledViewStreamNum,
int streamViewStreamNum,
StatementContext statementContext,
string[] streamNames)
{
// No filter means unindexed event tables
if (filterForIndexing == null)
{
return(new Pair <HistoricalIndexLookupStrategy, PollResultIndexingStrategy>(
new HistoricalIndexLookupStrategyNoIndex(), new PollResultIndexingStrategyNoIndex()));
}
// analyze query graph; Whereas stream0=named window, stream1=delete-expr filter
var hint = ExcludePlanHint.GetHint(streamNames, statementContext);
var queryGraph = new QueryGraph(2, hint, false);
FilterExprAnalyzer.Analyze(filterForIndexing, queryGraph, false);
return(DetermineIndexing(
queryGraph, polledViewType, streamViewType, polledViewStreamNum, streamViewStreamNum));
}
private static JoinSetComposerPrototypeHistoricalDesc DetermineIndexing(
ExprNode filterForIndexing,
EventType polledViewType,
EventType streamViewType,
int polledViewStreamNum,
int streamViewStreamNum,
string[] streamNames,
StatementRawInfo rawInfo,
StatementCompileTimeServices services)
{
// No filter means unindexed event tables
if (filterForIndexing == null) {
return new JoinSetComposerPrototypeHistoricalDesc(
HistoricalIndexLookupStrategyNoIndexForge.INSTANCE,
PollResultIndexingStrategyNoIndexForge.INSTANCE,
EmptyList<StmtClassForgeableFactory>.Instance);
}
// analyze query graph; Whereas stream0=named window, stream1=delete-expr filter
var hint = ExcludePlanHint.GetHint(streamNames, rawInfo, services);
var queryGraph = new QueryGraphForge(2, hint, false);
FilterExprAnalyzer.Analyze(filterForIndexing, queryGraph, false);
return DetermineIndexing(
queryGraph,
polledViewType,
streamViewType,
polledViewStreamNum,
streamViewStreamNum,
rawInfo,
services.SerdeResolver);
}
public static SubordPropPlan GetJoinProps(
ExprNode filterExpr,
int outsideStreamCount,
EventType[] allStreamTypesZeroIndexed,
ExcludePlanHint excludePlanHint)
{
// No filter expression means full table scan
if (filterExpr == null) {
return new SubordPropPlan();
}
// analyze query graph
var queryGraph = new QueryGraphForge(outsideStreamCount + 1, excludePlanHint, true);
FilterExprAnalyzer.Analyze(filterExpr, queryGraph, false);
// Build a list of streams and indexes
var joinProps = new LinkedHashMap<string, SubordPropHashKeyForge>();
var rangeProps = new LinkedHashMap<string, SubordPropRangeKeyForge>();
IDictionary<QueryGraphValueEntryCustomKeyForge, QueryGraphValueEntryCustomOperationForge> customIndexOps =
EmptyDictionary<QueryGraphValueEntryCustomKeyForge, QueryGraphValueEntryCustomOperationForge>.Instance;
for (var stream = 0; stream < outsideStreamCount; stream++) {
var lookupStream = stream + 1;
var queryGraphValue = queryGraph.GetGraphValue(lookupStream, 0);
var hashKeysAndIndexes = queryGraphValue.HashKeyProps;
// determine application functions
foreach (var item in queryGraphValue.Items) {
if (item.Entry is QueryGraphValueEntryCustomForge) {
if (customIndexOps.IsEmpty()) {
customIndexOps =
new Dictionary<QueryGraphValueEntryCustomKeyForge,
QueryGraphValueEntryCustomOperationForge>();
}
var custom = (QueryGraphValueEntryCustomForge) item.Entry;
custom.MergeInto(customIndexOps);
}
}
// handle key-lookups
var keyPropertiesJoin = hashKeysAndIndexes.Keys;
var indexPropertiesJoin = hashKeysAndIndexes.Indexed;
if (!keyPropertiesJoin.IsEmpty()) {
if (keyPropertiesJoin.Count != indexPropertiesJoin.Length) {
throw new IllegalStateException(
"Invalid query key and index property collection for stream " + stream);
}
for (var i = 0; i < keyPropertiesJoin.Count; i++) {
QueryGraphValueEntryHashKeyedForge keyDesc = keyPropertiesJoin[i];
var compareNode = keyDesc.KeyExpr;
var keyPropType = compareNode.Forge.EvaluationType.GetBoxedType();
var indexedPropType = allStreamTypesZeroIndexed[0]
.GetPropertyType(indexPropertiesJoin[i])
.GetBoxedType();
var coercionType = indexedPropType;
if (keyPropType != indexedPropType) {
coercionType = keyPropType.GetCompareToCoercionType(indexedPropType);
}
SubordPropHashKeyForge desc;
if (keyPropertiesJoin[i] is QueryGraphValueEntryHashKeyedForgeExpr) {
var keyExpr = (QueryGraphValueEntryHashKeyedForgeExpr) keyPropertiesJoin[i];
var keyStreamNum = keyExpr.IsRequiresKey ? (int?) stream : null;
desc = new SubordPropHashKeyForge(keyDesc, keyStreamNum, coercionType);
}
else {
var prop = (QueryGraphValueEntryHashKeyedForgeProp) keyDesc;
desc = new SubordPropHashKeyForge(prop, stream, coercionType);
}
joinProps.Put(indexPropertiesJoin[i], desc);
}
}
// handle range lookups
var rangeKeysAndIndexes = queryGraphValue.RangeProps;
var rangeIndexes = rangeKeysAndIndexes.Indexed;
var rangeDescs = rangeKeysAndIndexes.Keys;
if (rangeDescs.IsEmpty()) {
continue;
}
// get all ranges lookups
var count = -1;
foreach (var rangeDesc in rangeDescs) {
count++;
var rangeIndexProp = rangeIndexes[count];
var subqRangeDesc = rangeProps.Get(rangeIndexProp);
// other streams may specify the start or end endpoint of a range, therefore this operation can be additive
if (subqRangeDesc != null) {
if (subqRangeDesc.RangeInfo.Type.IsRange()) {
continue;
}
// see if we can make this additive by using a range
var relOpOther = (QueryGraphValueEntryRangeRelOpForge) subqRangeDesc.RangeInfo;
var relOpThis = (QueryGraphValueEntryRangeRelOpForge) rangeDesc;
var opsDesc = QueryGraphRangeUtil.GetCanConsolidate(relOpThis.Type, relOpOther.Type);
if (opsDesc != null) {
ExprNode start;
ExprNode end;
if (!opsDesc.IsReverse) {
start = relOpOther.Expression;
end = relOpThis.Expression;
}
else {
start = relOpThis.Expression;
end = relOpOther.Expression;
}
var allowRangeReversal = relOpOther.IsBetweenPart && relOpThis.IsBetweenPart;
var rangeIn = new QueryGraphValueEntryRangeInForge(
opsDesc.Type,
start,
end,
allowRangeReversal);
var indexedPropType = allStreamTypesZeroIndexed[0]
.GetPropertyType(rangeIndexProp)
.GetBoxedType();
var coercionType = indexedPropType;
var proposedType = CoercionUtil.GetCoercionTypeRangeIn(
indexedPropType,
rangeIn.ExprStart,
rangeIn.ExprEnd);
if (proposedType != null && proposedType != indexedPropType) {
coercionType = proposedType;
}
subqRangeDesc = new SubordPropRangeKeyForge(rangeIn, coercionType);
rangeProps.Put(rangeIndexProp, subqRangeDesc);
}
// ignore
continue;
}
// an existing entry has not been found
if (rangeDesc.Type.IsRange()) {
var rangeIn = (QueryGraphValueEntryRangeInForge) rangeDesc;
var indexedPropType =
allStreamTypesZeroIndexed[0].GetPropertyType(rangeIndexProp).GetBoxedType();
var coercionType = indexedPropType;
var proposedType = CoercionUtil.GetCoercionTypeRangeIn(
indexedPropType,
rangeIn.ExprStart,
rangeIn.ExprEnd);
if (proposedType != null && proposedType != indexedPropType) {
coercionType = proposedType;
}
subqRangeDesc = new SubordPropRangeKeyForge(rangeDesc, coercionType);
}
else {
var relOp = (QueryGraphValueEntryRangeRelOpForge) rangeDesc;
var keyPropType = relOp.Expression.Forge.EvaluationType;
var indexedPropType =
allStreamTypesZeroIndexed[0].GetPropertyType(rangeIndexProp).GetBoxedType();
var coercionType = indexedPropType;
if (keyPropType != indexedPropType) {
coercionType = keyPropType.GetCompareToCoercionType(indexedPropType);
}
subqRangeDesc = new SubordPropRangeKeyForge(rangeDesc, coercionType);
}
rangeProps.Put(rangeIndexProp, subqRangeDesc);
}
}
SubordPropInKeywordSingleIndex inKeywordSingleIdxProp = null;
SubordPropInKeywordMultiIndex inKeywordMultiIdxProp = null;
if (joinProps.IsEmpty() && rangeProps.IsEmpty()) {
for (var stream = 0; stream < outsideStreamCount; stream++) {
var lookupStream = stream + 1;
var queryGraphValue = queryGraph.GetGraphValue(lookupStream, 0);
var inkwSingles = queryGraphValue.InKeywordSingles;
if (inkwSingles.Indexed.Length != 0) {
ExprNode[] keys = inkwSingles.Key[0].KeyExprs;
var key = inkwSingles.Indexed[0];
if (inKeywordSingleIdxProp != null) {
continue;
}
var coercionType = keys[0].Forge.EvaluationType; // for in-comparison the same type is required
inKeywordSingleIdxProp = new SubordPropInKeywordSingleIndex(key, coercionType, keys);
}
var inkwMultis = queryGraphValue.InKeywordMulti;
if (!inkwMultis.IsEmpty()) {
QueryGraphValuePairInKWMultiIdx multi = inkwMultis[0];
inKeywordMultiIdxProp = new SubordPropInKeywordMultiIndex(
ExprNodeUtilityQuery.GetIdentResolvedPropertyNames(multi.Indexed),
multi.Indexed[0].Forge.EvaluationType,
multi.Key.KeyExpr);
}
if (inKeywordSingleIdxProp != null && inKeywordMultiIdxProp != null) {
inKeywordMultiIdxProp = null;
}
}
}
return new SubordPropPlan(
joinProps,
rangeProps,
inKeywordSingleIdxProp,
inKeywordMultiIdxProp,
customIndexOps);
}
public void TestGetPlan()
{
OuterJoinDesc[] descList =
{
SupportOuterJoinDescFactory.MakeDesc(
container, "IntPrimitive", "s0", "IntBoxed", "s1", OuterJoinType.LEFT)
};
var queryGraph = new QueryGraphForge(2, null, false);
var plan = QueryPlanBuilder.GetPlan(
typesPerStream,
new OuterJoinDesc[0],
queryGraph,
null,
new HistoricalViewableDesc(5),
dependencyGraph,
null,
new StreamJoinAnalysisResultCompileTime(2),
true,
null,
null);
AssertPlan(plan.Forge);
plan = QueryPlanBuilder.GetPlan(
typesPerStream,
descList,
queryGraph,
null,
new HistoricalViewableDesc(5),
dependencyGraph,
null,
new StreamJoinAnalysisResultCompileTime(2),
true,
null,
null);
AssertPlan(plan.Forge);
FilterExprAnalyzer.Analyze(SupportExprNodeFactory.GetInstance(container).MakeEqualsNode(), queryGraph, false);
plan = QueryPlanBuilder.GetPlan(
typesPerStream,
descList,
queryGraph,
null,
new HistoricalViewableDesc(5),
dependencyGraph,
null,
new StreamJoinAnalysisResultCompileTime(2),
true,
null,
null);
AssertPlan(plan.Forge);
plan = QueryPlanBuilder.GetPlan(
typesPerStream,
new OuterJoinDesc[0],
queryGraph,
null,
new HistoricalViewableDesc(5),
dependencyGraph,
null,
new StreamJoinAnalysisResultCompileTime(2),
true,
null,
null);
AssertPlan(plan.Forge);
}
/// <summary>
/// Builds join tuple composer.
/// </summary>
/// <param name="statementName">Name of the statement.</param>
/// <param name="statementId">The statement identifier.</param>
/// <param name="outerJoinDescList">list of descriptors for outer join criteria</param>
/// <param name="optionalFilterNode">filter tree for analysis to build indexes for fast access</param>
/// <param name="streamTypes">types of streams</param>
/// <param name="streamNames">names of streams</param>
/// <param name="streamJoinAnalysisResult">The stream join analysis result.</param>
/// <param name="queryPlanLogging">if set to <c>true</c> [query plan logging].</param>
/// <param name="statementContext">The statement context.</param>
/// <param name="historicalViewableDesc">The historical viewable desc.</param>
/// <param name="exprEvaluatorContext">The expr evaluator context.</param>
/// <param name="selectsRemoveStream">if set to <c>true</c> [selects remove stream].</param>
/// <param name="hasAggregations">if set to <c>true</c> [has aggregations].</param>
/// <param name="tableService">The table service.</param>
/// <param name="isOnDemandQuery">if set to <c>true</c> [is on demand query].</param>
/// <param name="allowIndexInit">if set to <c>true</c> [allow index initialize].</param>
/// <returns>
/// composer implementation
/// </returns>
/// <throws>com.espertech.esper.epl.expression.core.ExprValidationException is thrown to indicate thatvalidation of view use in joins failed.
/// {D255958A-8513-4226-94B9-080D98F904A1}</throws>
public static JoinSetComposerPrototype MakeComposerPrototype(string statementName, int statementId, OuterJoinDesc[] outerJoinDescList, ExprNode optionalFilterNode, EventType[] streamTypes, string[] streamNames, StreamJoinAnalysisResult streamJoinAnalysisResult, bool queryPlanLogging, StatementContext statementContext, HistoricalViewableDesc historicalViewableDesc, ExprEvaluatorContext exprEvaluatorContext, bool selectsRemoveStream, bool hasAggregations, TableService tableService, bool isOnDemandQuery, bool allowIndexInit)
{
// Determine if there is a historical stream, and what dependencies exist
var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false);
for (var i = 0; i < streamTypes.Length; i++)
{
if (historicalViewableDesc.Historical[i])
{
var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i];
historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn);
}
}
if (log.IsDebugEnabled)
{
log.Debug("Dependency graph: " + historicalDependencyGraph);
}
// Handle a join with a database or other historical data source for 2 streams
if ((historicalViewableDesc.HasHistorical) && (streamTypes.Length == 2))
{
return(MakeComposerHistorical2Stream(outerJoinDescList, optionalFilterNode, streamTypes, historicalViewableDesc, queryPlanLogging, exprEvaluatorContext, statementContext, streamNames, allowIndexInit));
}
var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList);
// Query graph for graph relationships between streams/historicals
// For outer joins the query graph will just contain outer join relationships
var hint = ExcludePlanHint.GetHint(streamNames, statementContext);
var queryGraph = new QueryGraph(streamTypes.Length, hint, false);
if (outerJoinDescList.Length > 0)
{
OuterJoinAnalyzer.Analyze(outerJoinDescList, queryGraph);
if (log.IsDebugEnabled)
{
log.Debug(".makeComposer After outer join queryGraph=\n" + queryGraph);
}
}
// Let the query graph reflect the where-clause
if (optionalFilterNode != null)
{
// Analyze relationships between streams using the optional filter expression.
// Relationships are properties in AND and EQUALS nodes of joins.
FilterExprAnalyzer.Analyze(optionalFilterNode, queryGraph, isOuterJoins);
if (log.IsDebugEnabled)
{
log.Debug(".makeComposer After filter expression queryGraph=\n" + queryGraph);
}
// Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c)
QueryGraph.FillEquivalentNav(streamTypes, queryGraph);
if (log.IsDebugEnabled)
{
log.Debug(".makeComposer After fill equiv. nav. queryGraph=\n" + queryGraph);
}
}
// Historical index lists
var historicalStreamIndexLists = new HistoricalStreamIndexList[streamTypes.Length];
var queryPlan = QueryPlanBuilder.GetPlan(streamTypes, outerJoinDescList, queryGraph, streamNames,
historicalViewableDesc, historicalDependencyGraph, historicalStreamIndexLists,
streamJoinAnalysisResult, queryPlanLogging, statementContext.Annotations, exprEvaluatorContext);
// remove unused indexes - consider all streams or all unidirectional
var usedIndexes = new HashSet <TableLookupIndexReqKey>();
var indexSpecs = queryPlan.IndexSpecs;
for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++)
{
var planNode = queryPlan.ExecNodeSpecs[streamNum];
if (planNode != null)
{
planNode.AddIndexes(usedIndexes);
}
}
foreach (var indexSpec in indexSpecs)
{
if (indexSpec == null)
{
continue;
}
var items = indexSpec.Items;
var indexNames = items.Keys.ToArray();
foreach (var indexName in indexNames)
{
if (!usedIndexes.Contains(indexName))
{
items.Remove(indexName);
}
}
}
var hook = QueryPlanIndexHookUtil.GetHook(statementContext.Annotations);
if (queryPlanLogging && (QueryPlanLog.IsInfoEnabled || hook != null))
{
QueryPlanLog.Info("Query plan: " + queryPlan.ToQueryPlan());
if (hook != null)
{
hook.Join(queryPlan);
}
}
// register index-use references for tables
if (!isOnDemandQuery)
{
foreach (var usedIndex in usedIndexes)
{
if (usedIndex.TableName != null)
{
tableService.GetTableMetadata(usedIndex.TableName).AddIndexReference(usedIndex.Name, statementName);
}
}
}
var joinRemoveStream = selectsRemoveStream || hasAggregations;
return(new JoinSetComposerPrototypeImpl(
statementName,
statementId,
outerJoinDescList,
optionalFilterNode,
streamTypes,
streamNames,
streamJoinAnalysisResult,
statementContext.Annotations,
historicalViewableDesc,
exprEvaluatorContext,
indexSpecs,
queryPlan,
historicalStreamIndexLists,
joinRemoveStream,
isOuterJoins,
tableService,
statementContext.EventTableIndexService));
}
/// <summary>
/// Ctor.
/// </summary>
/// <param name="statementSpec">
/// is a container for the definition of all statement constructs that may have been used in the
/// statement, i.e. if defines the select clauses, insert into, outer joins etc.
/// </param>
/// <param name="services">is the service instances for dependency injection</param>
/// <param name="statementContext">is statement-level information and statement services</param>
/// <throws>ExprValidationException if the preparation failed</throws>
public EPPreparedExecuteMethodQuery(
StatementSpecCompiled statementSpec,
EPServicesContext services,
StatementContext statementContext)
{
var queryPlanLogging = services.ConfigSnapshot.EngineDefaults.Logging.IsEnableQueryPlan;
if (queryPlanLogging)
{
QueryPlanLog.Info("Query plans for Fire-and-forget query '" + statementContext.Expression + "'");
}
_hasTableAccess = (statementSpec.TableNodes != null && statementSpec.TableNodes.Length > 0);
foreach (var streamSpec in statementSpec.StreamSpecs)
{
_hasTableAccess |= streamSpec is TableQueryStreamSpec;
}
_statementSpec = statementSpec;
_services = services;
EPPreparedExecuteMethodHelper.ValidateFAFQuery(statementSpec);
var numStreams = statementSpec.StreamSpecs.Length;
var typesPerStream = new EventType[numStreams];
var namesPerStream = new string[numStreams];
_processors = new FireAndForgetProcessor[numStreams];
_agentInstanceContext = new AgentInstanceContext(statementContext, null, -1, null, null, statementContext.DefaultAgentInstanceScriptContext);
// resolve types and processors
for (var i = 0; i < numStreams; i++)
{
var streamSpec = statementSpec.StreamSpecs[i];
_processors[i] = FireAndForgetProcessorFactory.ValidateResolveProcessor(streamSpec, services);
string streamName = _processors[i].NamedWindowOrTableName;
if (streamSpec.OptionalStreamName != null)
{
streamName = streamSpec.OptionalStreamName;
}
namesPerStream[i] = streamName;
typesPerStream[i] = _processors[i].EventTypeResultSetProcessor;
}
// compile filter to optimize access to named window
var types = new StreamTypeServiceImpl(typesPerStream, namesPerStream, new bool[numStreams], services.EngineURI, false);
var excludePlanHint = ExcludePlanHint.GetHint(types.StreamNames, statementContext);
_queryGraph = new QueryGraph(numStreams, excludePlanHint, false);
if (statementSpec.FilterRootNode != null)
{
for (var i = 0; i < numStreams; i++)
{
try
{
ExprEvaluatorContextStatement evaluatorContextStmt = new ExprEvaluatorContextStatement(statementContext, false);
ExprValidationContext validationContext = new ExprValidationContext(
statementContext.Container,
types,
statementContext.EngineImportService,
statementContext.StatementExtensionServicesContext, null,
statementContext.TimeProvider,
statementContext.VariableService,
statementContext.TableService, evaluatorContextStmt,
statementContext.EventAdapterService,
statementContext.StatementName,
statementContext.StatementId,
statementContext.Annotations,
statementContext.ContextDescriptor,
statementContext.ScriptingService,
false, false, true, false, null, true);
ExprNode validated = ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.FILTER, statementSpec.FilterRootNode, validationContext);
FilterExprAnalyzer.Analyze(validated, _queryGraph, false);
}
catch (Exception ex)
{
Log.Warn("Unexpected exception analyzing filter paths: " + ex.Message, ex);
}
}
}
// obtain result set processor
var isIStreamOnly = new bool[namesPerStream.Length];
CompatExtensions.Fill(isIStreamOnly, true);
StreamTypeService typeService = new StreamTypeServiceImpl(typesPerStream, namesPerStream, isIStreamOnly, services.EngineURI, true);
EPStatementStartMethodHelperValidate.ValidateNodes(statementSpec, statementContext, typeService, null);
var resultSetProcessorPrototype = ResultSetProcessorFactoryFactory.GetProcessorPrototype(statementSpec, statementContext, typeService, null, new bool[0], true, ContextPropertyRegistryImpl.EMPTY_REGISTRY, null, services.ConfigSnapshot, services.ResultSetProcessorHelperFactory, true, false);
_resultSetProcessor = EPStatementStartMethodHelperAssignExpr.GetAssignResultSetProcessor(_agentInstanceContext, resultSetProcessorPrototype, false, null, true);
if (statementSpec.SelectClauseSpec.IsDistinct)
{
if (_resultSetProcessor.ResultEventType is EventTypeSPI)
{
_eventBeanReader = ((EventTypeSPI)_resultSetProcessor.ResultEventType).Reader;
}
if (_eventBeanReader == null)
{
_eventBeanReader = new EventBeanReaderDefaultImpl(_resultSetProcessor.ResultEventType);
}
}
// check context partition use
if (statementSpec.OptionalContextName != null)
{
if (numStreams > 1)
{
throw new ExprValidationException("Joins in runtime queries for context partitions are not supported");
}
}
// plan joins or simple queries
if (numStreams > 1)
{
var streamJoinAnalysisResult = new StreamJoinAnalysisResult(numStreams);
CompatExtensions.Fill(streamJoinAnalysisResult.NamedWindow, true);
for (var i = 0; i < numStreams; i++)
{
var processorInstance = _processors[i].GetProcessorInstance(_agentInstanceContext);
if (_processors[i].IsVirtualDataWindow)
{
streamJoinAnalysisResult.ViewExternal[i] = agentInstanceContext => processorInstance.VirtualDataWindow;
}
var uniqueIndexes = _processors[i].GetUniqueIndexes(processorInstance);
streamJoinAnalysisResult.UniqueKeys[i] = uniqueIndexes;
}
var hasAggregations = !resultSetProcessorPrototype.AggregationServiceFactoryDesc.Expressions.IsEmpty();
_joinSetComposerPrototype = JoinSetComposerPrototypeFactory.MakeComposerPrototype(null, -1,
statementSpec.OuterJoinDescList, statementSpec.FilterRootNode, typesPerStream, namesPerStream,
streamJoinAnalysisResult, queryPlanLogging, statementContext, new HistoricalViewableDesc(numStreams),
_agentInstanceContext, false, hasAggregations, services.TableService, true,
services.EventTableIndexService.AllowInitIndex(false));
}
}
public static JoinSetComposerPrototypeDesc MakeComposerPrototype(
StatementSpecCompiled spec,
StreamJoinAnalysisResultCompileTime joinAnalysisResult,
StreamTypeService typeService,
HistoricalViewableDesc historicalViewableDesc,
bool isOnDemandQuery,
bool hasAggregations,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices compileTimeServices)
{
var streamTypes = typeService.EventTypes;
var streamNames = typeService.StreamNames;
var whereClause = spec.Raw.WhereClause;
var queryPlanLogging = compileTimeServices.Configuration.Common.Logging.IsEnableQueryPlan;
var additionalForgeables = new List<StmtClassForgeableFactory>();
// Determine if there is a historical stream, and what dependencies exist
var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false);
for (var i = 0; i < streamTypes.Length; i++) {
if (historicalViewableDesc.Historical[i]) {
var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i];
historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn);
}
}
if (Log.IsDebugEnabled) {
Log.Debug("Dependency graph: " + historicalDependencyGraph);
}
// Handle a join with a database or other historical data source for 2 streams
var outerJoinDescs = OuterJoinDesc.ToArray(spec.Raw.OuterJoinDescList);
if (historicalViewableDesc.IsHistorical && streamTypes.Length == 2) {
var desc = MakeComposerHistorical2Stream(
outerJoinDescs,
whereClause,
streamTypes,
streamNames,
historicalViewableDesc,
queryPlanLogging,
statementRawInfo,
compileTimeServices);
return new JoinSetComposerPrototypeDesc(desc.Forge, desc.AdditionalForgeables);
}
var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescs);
// Query graph for graph relationships between streams/historicals
// For outer joins the query graph will just contain outer join relationships
var hint = ExcludePlanHint.GetHint(
typeService.StreamNames,
statementRawInfo,
compileTimeServices);
var queryGraph = new QueryGraphForge(streamTypes.Length, hint, false);
if (outerJoinDescs.Length > 0) {
OuterJoinAnalyzer.Analyze(outerJoinDescs, queryGraph);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After outer join filterQueryGraph=\n" + queryGraph);
}
}
// Let the query graph reflect the where-clause
if (whereClause != null) {
// Analyze relationships between streams using the optional filter expression.
// Relationships are properties in AND and EQUALS nodes of joins.
FilterExprAnalyzer.Analyze(whereClause, queryGraph, isOuterJoins);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After filter expression filterQueryGraph=\n" + queryGraph);
}
// Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c)
QueryGraphForge.FillEquivalentNav(streamTypes, queryGraph);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After fill equiv. nav. filterQueryGraph=\n" + queryGraph);
}
}
// Historical index lists
var historicalStreamIndexLists =
new HistoricalStreamIndexListForge[streamTypes.Length];
var queryPlanDesc = QueryPlanBuilder.GetPlan(
streamTypes,
outerJoinDescs,
queryGraph,
typeService.StreamNames,
historicalViewableDesc,
historicalDependencyGraph,
historicalStreamIndexLists,
joinAnalysisResult,
queryPlanLogging,
statementRawInfo,
compileTimeServices);
QueryPlanForge queryPlan = queryPlanDesc.Forge;
additionalForgeables.AddAll(queryPlanDesc.AdditionalForgeables);
// remove unused indexes - consider all streams or all unidirectional
var usedIndexes = new HashSet<TableLookupIndexReqKey>();
var indexSpecs = queryPlan.IndexSpecs;
for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) {
var planNode = queryPlan.ExecNodeSpecs[streamNum];
planNode?.AddIndexes(usedIndexes);
}
foreach (var indexSpec in indexSpecs) {
if (indexSpec == null) {
continue;
}
var items = indexSpec.Items;
var indexNames = items.Keys.ToArray();
foreach (var indexName in indexNames) {
if (!usedIndexes.Contains(indexName)) {
items.Remove(indexName);
}
}
}
// plan multikeys
IList<StmtClassForgeableFactory> multikeyForgeables = PlanMultikeys(
indexSpecs, statementRawInfo, compileTimeServices);
additionalForgeables.AddAll(multikeyForgeables);
QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook(
spec.Annotations,
compileTimeServices.ImportServiceCompileTime);
if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) {
QUERY_PLAN_LOG.Info("Query plan: " + queryPlan.ToQueryPlan());
hook?.Join(queryPlan);
}
var selectsRemoveStream =
spec.Raw.SelectStreamSelectorEnum.IsSelectsRStream() || spec.Raw.OutputLimitSpec != null;
var joinRemoveStream = selectsRemoveStream || hasAggregations;
ExprNode postJoinEvaluator;
if (JoinSetComposerUtil.IsNonUnidirectionalNonSelf(
isOuterJoins,
joinAnalysisResult.IsUnidirectional,
joinAnalysisResult.IsPureSelfJoin)) {
postJoinEvaluator = GetFilterExpressionInclOnClause(
spec.Raw.WhereClause,
outerJoinDescs,
statementRawInfo,
compileTimeServices);
}
else {
postJoinEvaluator = spec.Raw.WhereClause;
}
JoinSetComposerPrototypeGeneralForge forge = new JoinSetComposerPrototypeGeneralForge(
typeService.EventTypes,
postJoinEvaluator,
outerJoinDescs.Length > 0,
queryPlan,
joinAnalysisResult,
typeService.StreamNames,
joinRemoveStream,
historicalViewableDesc.IsHistorical);
return new JoinSetComposerPrototypeDesc(forge, additionalForgeables);
}
public FAFQueryMethodSelectDesc(
StatementSpecCompiled statementSpec,
Compilable compilable,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
Annotations = statementSpec.Annotations;
ContextName = statementSpec.Raw.OptionalContextName;
var queryPlanLogging = services.Configuration.Common.Logging.IsEnableQueryPlan;
if (queryPlanLogging) {
QUERY_PLAN_LOG.Info("Query plans for Fire-and-forget query '" + compilable.ToEPL() + "'");
}
HasTableAccess = statementSpec.TableAccessNodes != null && statementSpec.TableAccessNodes.Count > 0;
foreach (var streamSpec in statementSpec.StreamSpecs) {
HasTableAccess |= streamSpec is TableQueryStreamSpec;
}
HasTableAccess |= StatementLifecycleSvcUtil.IsSubqueryWithTable(
statementSpec.SubselectNodes, services.TableCompileTimeResolver);
IsDistinct = statementSpec.SelectClauseCompiled.IsDistinct;
FAFQueryMethodHelper.ValidateFAFQuery(statementSpec);
var numStreams = statementSpec.StreamSpecs.Length;
var typesPerStream = new EventType[numStreams];
var namesPerStream = new string[numStreams];
var eventTypeNames = new string[numStreams];
Processors = new FireAndForgetProcessorForge[numStreams];
ConsumerFilters = new ExprNode[numStreams];
// check context partition use
if (statementSpec.Raw.OptionalContextName != null) {
if (numStreams > 1) {
throw new ExprValidationException(
"Joins in runtime queries for context partitions are not supported");
}
}
// resolve types and processors
for (var i = 0; i < numStreams; i++) {
var streamSpec = statementSpec.StreamSpecs[i];
Processors[i] = FireAndForgetProcessorForgeFactory.ValidateResolveProcessor(streamSpec);
if (numStreams > 1 && Processors[i].ContextName != null) {
throw new ExprValidationException(
"Joins against named windows that are under context are not supported");
}
var streamName = Processors[i].NamedWindowOrTableName;
if (streamSpec.OptionalStreamName != null) {
streamName = streamSpec.OptionalStreamName;
}
namesPerStream[i] = streamName;
typesPerStream[i] = Processors[i].EventTypeRspInputEvents;
eventTypeNames[i] = typesPerStream[i].Name;
IList<ExprNode> consumerFilterExprs;
if (streamSpec is NamedWindowConsumerStreamSpec) {
var namedSpec = (NamedWindowConsumerStreamSpec) streamSpec;
consumerFilterExprs = namedSpec.FilterExpressions;
}
else {
var tableSpec = (TableQueryStreamSpec) streamSpec;
consumerFilterExprs = tableSpec.FilterExpressions;
}
ConsumerFilters[i] = ExprNodeUtilityMake.ConnectExpressionsByLogicalAndWhenNeeded(consumerFilterExprs);
}
// compile filter to optimize access to named window
var optionalStreamsIfAny = OuterJoinAnalyzer.OptionalStreamsIfAny(statementSpec.Raw.OuterJoinDescList);
var types = new StreamTypeServiceImpl(
typesPerStream,
namesPerStream,
new bool[numStreams],
false,
optionalStreamsIfAny);
var excludePlanHint = ExcludePlanHint.GetHint(types.StreamNames, statementRawInfo, services);
QueryGraph = new QueryGraphForge(numStreams, excludePlanHint, false);
if (statementSpec.Raw.WhereClause != null) {
for (var i = 0; i < numStreams; i++) {
try {
var validationContext = new ExprValidationContextBuilder(types, statementRawInfo, services)
.WithAllowBindingConsumption(true)
.WithIsFilterExpression(true)
.Build();
var validated = ExprNodeUtilityValidate.GetValidatedSubtree(
ExprNodeOrigin.FILTER,
statementSpec.Raw.WhereClause,
validationContext);
FilterExprAnalyzer.Analyze(validated, QueryGraph, false);
}
catch (Exception ex) {
Log.Warn("Unexpected exception analyzing filter paths: " + ex.Message, ex);
}
}
}
// handle subselects
// first we create streams for subselects, if there are any
var @base = new StatementBaseInfo(compilable, statementSpec, null, statementRawInfo, null);
var subqueryNamedWindowConsumers = new List<NamedWindowConsumerStreamSpec>();
SubSelectActivationDesc subSelectActivationDesc = SubSelectHelperActivations.CreateSubSelectActivation(
EmptyList<FilterSpecCompiled>.Instance, subqueryNamedWindowConsumers, @base, services);
IDictionary<ExprSubselectNode, SubSelectActivationPlan> subselectActivation = subSelectActivationDesc.Subselects;
AdditionalForgeables.AddAll(subSelectActivationDesc.AdditionalForgeables);
SubSelectHelperForgePlan subSelectForgePlan = SubSelectHelperForgePlanner.PlanSubSelect(
@base, subselectActivation, namesPerStream, typesPerStream, eventTypeNames, services);
SubselectForges = subSelectForgePlan.Subselects;
AdditionalForgeables.AddAll(subSelectForgePlan.AdditionalForgeables);
// obtain result set processor
var isIStreamOnly = new bool[namesPerStream.Length];
isIStreamOnly.Fill(true);
StreamTypeService typeService = new StreamTypeServiceImpl(
typesPerStream,
namesPerStream,
isIStreamOnly,
true,
optionalStreamsIfAny);
WhereClause = EPStatementStartMethodHelperValidate.ValidateNodes(
statementSpec.Raw,
typeService,
null,
statementRawInfo,
services);
var resultSetSpec = new ResultSetSpec(statementSpec);
ResultSetProcessor = ResultSetProcessorFactoryFactory.GetProcessorPrototype(
resultSetSpec,
typeService,
null,
new bool[0],
true,
null,
true,
false,
statementRawInfo,
services);
AdditionalForgeables.AddAll(ResultSetProcessor.AdditionalForgeables);
// plan table access
TableAccessForges = ExprTableEvalHelperPlan.PlanTableAccess(statementSpec.Raw.TableExpressions);
// plan joins or simple queries
if (numStreams > 1) {
var streamJoinAnalysisResult = new StreamJoinAnalysisResultCompileTime(numStreams);
CompatExtensions.Fill(streamJoinAnalysisResult.NamedWindowsPerStream, (NamedWindowMetaData) null);
for (var i = 0; i < numStreams; i++) {
var uniqueIndexes = Processors[i].UniqueIndexes;
streamJoinAnalysisResult.UniqueKeys[i] = uniqueIndexes;
}
var hasAggregations = ResultSetProcessor.ResultSetProcessorType.IsAggregated();
var desc = JoinSetComposerPrototypeForgeFactory.MakeComposerPrototype(
statementSpec,
streamJoinAnalysisResult,
types,
new HistoricalViewableDesc(numStreams),
true,
hasAggregations,
statementRawInfo,
services);
AdditionalForgeables.AddAll(desc.AdditionalForgeables);
Joins = desc.Forge;
}
else {
Joins = null;
}
var multiKeyPlan = MultiKeyPlanner.PlanMultiKeyDistinct(
IsDistinct, ResultSetProcessor.ResultEventType, statementRawInfo, SerdeCompileTimeResolverNonHA.INSTANCE);
AdditionalForgeables.AddAll(multiKeyPlan.MultiKeyForgeables);
DistinctMultiKey = multiKeyPlan.ClassRef;
}
