/// <summary>
/// Builds join tuple composer.
/// </summary>
/// <param name="statementName">Name of the statement.</param>
/// <param name="statementId">The statement identifier.</param>
/// <param name="outerJoinDescList">list of descriptors for outer join criteria</param>
/// <param name="optionalFilterNode">filter tree for analysis to build indexes for fast access</param>
/// <param name="streamTypes">types of streams</param>
/// <param name="streamNames">names of streams</param>
/// <param name="streamJoinAnalysisResult">The stream join analysis result.</param>
/// <param name="queryPlanLogging">if set to <c>true</c> [query plan logging].</param>
/// <param name="statementContext">The statement context.</param>
/// <param name="historicalViewableDesc">The historical viewable desc.</param>
/// <param name="exprEvaluatorContext">The expr evaluator context.</param>
/// <param name="selectsRemoveStream">if set to <c>true</c> [selects remove stream].</param>
/// <param name="hasAggregations">if set to <c>true</c> [has aggregations].</param>
/// <param name="tableService">The table service.</param>
/// <param name="isOnDemandQuery">if set to <c>true</c> [is on demand query].</param>
/// <param name="allowIndexInit">if set to <c>true</c> [allow index initialize].</param>
/// <returns>
/// composer implementation
/// </returns>
/// <throws>com.espertech.esper.epl.expression.core.ExprValidationException is thrown to indicate thatvalidation of view use in joins failed.
/// {D255958A-8513-4226-94B9-080D98F904A1}</throws>
public static JoinSetComposerPrototype MakeComposerPrototype(string statementName, int statementId, OuterJoinDesc[] outerJoinDescList, ExprNode optionalFilterNode, EventType[] streamTypes, string[] streamNames, StreamJoinAnalysisResult streamJoinAnalysisResult, bool queryPlanLogging, StatementContext statementContext, HistoricalViewableDesc historicalViewableDesc, ExprEvaluatorContext exprEvaluatorContext, bool selectsRemoveStream, bool hasAggregations, TableService tableService, bool isOnDemandQuery, bool allowIndexInit)
{
// Determine if there is a historical stream, and what dependencies exist
var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false);
for (var i = 0; i < streamTypes.Length; i++)
{
if (historicalViewableDesc.Historical[i])
{
var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i];
historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn);
}
}
if (log.IsDebugEnabled)
{
log.Debug("Dependency graph: " + historicalDependencyGraph);
}
// Handle a join with a database or other historical data source for 2 streams
if ((historicalViewableDesc.HasHistorical) && (streamTypes.Length == 2))
{
return(MakeComposerHistorical2Stream(outerJoinDescList, optionalFilterNode, streamTypes, historicalViewableDesc, queryPlanLogging, exprEvaluatorContext, statementContext, streamNames, allowIndexInit));
}
var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList);
// Query graph for graph relationships between streams/historicals
// For outer joins the query graph will just contain outer join relationships
var hint = ExcludePlanHint.GetHint(streamNames, statementContext);
var queryGraph = new QueryGraph(streamTypes.Length, hint, false);
if (outerJoinDescList.Length > 0)
{
OuterJoinAnalyzer.Analyze(outerJoinDescList, queryGraph);
if (log.IsDebugEnabled)
{
log.Debug(".makeComposer After outer join queryGraph=\n" + queryGraph);
}
}
// Let the query graph reflect the where-clause
if (optionalFilterNode != null)
{
// Analyze relationships between streams using the optional filter expression.
// Relationships are properties in AND and EQUALS nodes of joins.
FilterExprAnalyzer.Analyze(optionalFilterNode, queryGraph, isOuterJoins);
if (log.IsDebugEnabled)
{
log.Debug(".makeComposer After filter expression queryGraph=\n" + queryGraph);
}
// Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c)
QueryGraph.FillEquivalentNav(streamTypes, queryGraph);
if (log.IsDebugEnabled)
{
log.Debug(".makeComposer After fill equiv. nav. queryGraph=\n" + queryGraph);
}
}
// Historical index lists
var historicalStreamIndexLists = new HistoricalStreamIndexList[streamTypes.Length];
var queryPlan = QueryPlanBuilder.GetPlan(streamTypes, outerJoinDescList, queryGraph, streamNames,
historicalViewableDesc, historicalDependencyGraph, historicalStreamIndexLists,
streamJoinAnalysisResult, queryPlanLogging, statementContext.Annotations, exprEvaluatorContext);
// remove unused indexes - consider all streams or all unidirectional
var usedIndexes = new HashSet <TableLookupIndexReqKey>();
var indexSpecs = queryPlan.IndexSpecs;
for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++)
{
var planNode = queryPlan.ExecNodeSpecs[streamNum];
if (planNode != null)
{
planNode.AddIndexes(usedIndexes);
}
}
foreach (var indexSpec in indexSpecs)
{
if (indexSpec == null)
{
continue;
}
var items = indexSpec.Items;
var indexNames = items.Keys.ToArray();
foreach (var indexName in indexNames)
{
if (!usedIndexes.Contains(indexName))
{
items.Remove(indexName);
}
}
}
var hook = QueryPlanIndexHookUtil.GetHook(statementContext.Annotations);
if (queryPlanLogging && (QueryPlanLog.IsInfoEnabled || hook != null))
{
QueryPlanLog.Info("Query plan: " + queryPlan.ToQueryPlan());
if (hook != null)
{
hook.Join(queryPlan);
}
}
// register index-use references for tables
if (!isOnDemandQuery)
{
foreach (var usedIndex in usedIndexes)
{
if (usedIndex.TableName != null)
{
tableService.GetTableMetadata(usedIndex.TableName).AddIndexReference(usedIndex.Name, statementName);
}
}
}
var joinRemoveStream = selectsRemoveStream || hasAggregations;
return(new JoinSetComposerPrototypeImpl(
statementName,
statementId,
outerJoinDescList,
optionalFilterNode,
streamTypes,
streamNames,
streamJoinAnalysisResult,
statementContext.Annotations,
historicalViewableDesc,
exprEvaluatorContext,
indexSpecs,
queryPlan,
historicalStreamIndexLists,
joinRemoveStream,
isOuterJoins,
tableService,
statementContext.EventTableIndexService));
}
public static JoinSetComposerPrototypeDesc MakeComposerPrototype(
StatementSpecCompiled spec,
StreamJoinAnalysisResultCompileTime joinAnalysisResult,
StreamTypeService typeService,
HistoricalViewableDesc historicalViewableDesc,
bool isOnDemandQuery,
bool hasAggregations,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices compileTimeServices)
{
var streamTypes = typeService.EventTypes;
var streamNames = typeService.StreamNames;
var whereClause = spec.Raw.WhereClause;
var queryPlanLogging = compileTimeServices.Configuration.Common.Logging.IsEnableQueryPlan;
var additionalForgeables = new List<StmtClassForgeableFactory>();
// Determine if there is a historical stream, and what dependencies exist
var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false);
for (var i = 0; i < streamTypes.Length; i++) {
if (historicalViewableDesc.Historical[i]) {
var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i];
historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn);
}
}
if (Log.IsDebugEnabled) {
Log.Debug("Dependency graph: " + historicalDependencyGraph);
}
// Handle a join with a database or other historical data source for 2 streams
var outerJoinDescs = OuterJoinDesc.ToArray(spec.Raw.OuterJoinDescList);
if (historicalViewableDesc.IsHistorical && streamTypes.Length == 2) {
var desc = MakeComposerHistorical2Stream(
outerJoinDescs,
whereClause,
streamTypes,
streamNames,
historicalViewableDesc,
queryPlanLogging,
statementRawInfo,
compileTimeServices);
return new JoinSetComposerPrototypeDesc(desc.Forge, desc.AdditionalForgeables);
}
var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescs);
// Query graph for graph relationships between streams/historicals
// For outer joins the query graph will just contain outer join relationships
var hint = ExcludePlanHint.GetHint(
typeService.StreamNames,
statementRawInfo,
compileTimeServices);
var queryGraph = new QueryGraphForge(streamTypes.Length, hint, false);
if (outerJoinDescs.Length > 0) {
OuterJoinAnalyzer.Analyze(outerJoinDescs, queryGraph);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After outer join filterQueryGraph=\n" + queryGraph);
}
}
// Let the query graph reflect the where-clause
if (whereClause != null) {
// Analyze relationships between streams using the optional filter expression.
// Relationships are properties in AND and EQUALS nodes of joins.
FilterExprAnalyzer.Analyze(whereClause, queryGraph, isOuterJoins);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After filter expression filterQueryGraph=\n" + queryGraph);
}
// Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c)
QueryGraphForge.FillEquivalentNav(streamTypes, queryGraph);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After fill equiv. nav. filterQueryGraph=\n" + queryGraph);
}
}
// Historical index lists
var historicalStreamIndexLists =
new HistoricalStreamIndexListForge[streamTypes.Length];
var queryPlanDesc = QueryPlanBuilder.GetPlan(
streamTypes,
outerJoinDescs,
queryGraph,
typeService.StreamNames,
historicalViewableDesc,
historicalDependencyGraph,
historicalStreamIndexLists,
joinAnalysisResult,
queryPlanLogging,
statementRawInfo,
compileTimeServices);
QueryPlanForge queryPlan = queryPlanDesc.Forge;
additionalForgeables.AddAll(queryPlanDesc.AdditionalForgeables);
// remove unused indexes - consider all streams or all unidirectional
var usedIndexes = new HashSet<TableLookupIndexReqKey>();
var indexSpecs = queryPlan.IndexSpecs;
for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) {
var planNode = queryPlan.ExecNodeSpecs[streamNum];
planNode?.AddIndexes(usedIndexes);
}
foreach (var indexSpec in indexSpecs) {
if (indexSpec == null) {
continue;
}
var items = indexSpec.Items;
var indexNames = items.Keys.ToArray();
foreach (var indexName in indexNames) {
if (!usedIndexes.Contains(indexName)) {
items.Remove(indexName);
}
}
}
// plan multikeys
IList<StmtClassForgeableFactory> multikeyForgeables = PlanMultikeys(
indexSpecs, statementRawInfo, compileTimeServices);
additionalForgeables.AddAll(multikeyForgeables);
QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook(
spec.Annotations,
compileTimeServices.ImportServiceCompileTime);
if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) {
QUERY_PLAN_LOG.Info("Query plan: " + queryPlan.ToQueryPlan());
hook?.Join(queryPlan);
}
var selectsRemoveStream =
spec.Raw.SelectStreamSelectorEnum.IsSelectsRStream() || spec.Raw.OutputLimitSpec != null;
var joinRemoveStream = selectsRemoveStream || hasAggregations;
ExprNode postJoinEvaluator;
if (JoinSetComposerUtil.IsNonUnidirectionalNonSelf(
isOuterJoins,
joinAnalysisResult.IsUnidirectional,
joinAnalysisResult.IsPureSelfJoin)) {
postJoinEvaluator = GetFilterExpressionInclOnClause(
spec.Raw.WhereClause,
outerJoinDescs,
statementRawInfo,
compileTimeServices);
}
else {
postJoinEvaluator = spec.Raw.WhereClause;
}
JoinSetComposerPrototypeGeneralForge forge = new JoinSetComposerPrototypeGeneralForge(
typeService.EventTypes,
postJoinEvaluator,
outerJoinDescs.Length > 0,
queryPlan,
joinAnalysisResult,
typeService.StreamNames,
joinRemoveStream,
historicalViewableDesc.IsHistorical);
return new JoinSetComposerPrototypeDesc(forge, additionalForgeables);
}
public static StmtForgeMethodSelectResult Make(
IContainer container,
bool dataflowOperator,
string @namespace,
string classPostfix,
StatementBaseInfo @base,
StatementCompileTimeServices services)
{
var filterSpecCompileds = new List<FilterSpecCompiled>();
var scheduleHandleCallbackProviders = new List<ScheduleHandleCallbackProvider>();
var namedWindowConsumers = new List<NamedWindowConsumerStreamSpec>();
var statementSpec = @base.StatementSpec;
var additionalForgeables = new List<StmtClassForgeableFactory>();
var streamNames = StatementForgeMethodSelectUtil.DetermineStreamNames(statementSpec.StreamSpecs);
var numStreams = streamNames.Length;
if (numStreams == 0) {
throw new ExprValidationException("The from-clause is required but has not been specified");
}
// first we create streams for subselects, if there are any
SubSelectActivationDesc subSelectActivationDesc = SubSelectHelperActivations.CreateSubSelectActivation(
filterSpecCompileds, namedWindowConsumers, @base, services);
IDictionary<ExprSubselectNode, SubSelectActivationPlan> subselectActivation = subSelectActivationDesc.Subselects;
additionalForgeables.AddAll(subSelectActivationDesc.AdditionalForgeables);
// verify for joins that required views are present
StreamJoinAnalysisResultCompileTime joinAnalysisResult = StatementForgeMethodSelectUtil.VerifyJoinViews(statementSpec);
var streamEventTypes = new EventType[statementSpec.StreamSpecs.Length];
var eventTypeNames = new string[numStreams];
var isNamedWindow = new bool[numStreams];
var viewableActivatorForges = new ViewableActivatorForge[numStreams];
var viewForges = new IList<ViewFactoryForge>[numStreams];
var historicalEventViewables = new HistoricalEventViewableForge[numStreams];
for (var stream = 0; stream < numStreams; stream++) {
var streamSpec = statementSpec.StreamSpecs[stream];
var isCanIterateUnbound = streamSpec.ViewSpecs.Length == 0 &&
(services.Configuration.Compiler.ViewResources.IsIterableUnbound ||
AnnotationUtil.HasAnnotation(statementSpec.Annotations, typeof(IterableUnboundAttribute)));
var args = new ViewFactoryForgeArgs(
stream,
false,
-1,
streamSpec.Options,
null,
@base.StatementRawInfo,
services);
if (dataflowOperator) {
var dfResult = HandleDataflowActivation(args, streamSpec);
streamEventTypes[stream] = dfResult.StreamEventType;
eventTypeNames[stream] = dfResult.EventTypeName;
viewableActivatorForges[stream] = dfResult.ViewableActivatorForge;
viewForges[stream] = dfResult.ViewForges;
additionalForgeables.AddAll(dfResult.AdditionalForgeables);
}
else if (streamSpec is FilterStreamSpecCompiled) {
var filterStreamSpec = (FilterStreamSpecCompiled) statementSpec.StreamSpecs[stream];
var filterSpecCompiled = filterStreamSpec.FilterSpecCompiled;
streamEventTypes[stream] = filterSpecCompiled.ResultEventType;
eventTypeNames[stream] = filterStreamSpec.FilterSpecCompiled.FilterForEventTypeName;
viewableActivatorForges[stream] = new ViewableActivatorFilterForge(
filterSpecCompiled,
isCanIterateUnbound,
stream,
false,
-1);
ViewFactoryForgeDesc viewForgeDesc = ViewFactoryForgeUtil.CreateForges(streamSpec.ViewSpecs, args, streamEventTypes[stream]);
viewForges[stream] = viewForgeDesc.Forges;
additionalForgeables.AddAll(viewForgeDesc.MultikeyForges);
filterSpecCompileds.Add(filterSpecCompiled);
}
else if (streamSpec is PatternStreamSpecCompiled) {
var patternStreamSpec = (PatternStreamSpecCompiled) streamSpec;
var forges = patternStreamSpec.Root.CollectFactories();
foreach (var forgeNode in forges) {
forgeNode.CollectSelfFilterAndSchedule(filterSpecCompileds, scheduleHandleCallbackProviders);
}
var patternType = ViewableActivatorPatternForge.MakeRegisterPatternType(
@base,
stream,
patternStreamSpec,
services);
var patternContext = new PatternContext(0, patternStreamSpec.MatchedEventMapMeta, false, -1, false);
viewableActivatorForges[stream] = new ViewableActivatorPatternForge(
patternType,
patternStreamSpec,
patternContext,
isCanIterateUnbound);
streamEventTypes[stream] = patternType;
ViewFactoryForgeDesc viewForgeDesc = ViewFactoryForgeUtil.CreateForges(streamSpec.ViewSpecs, args, patternType);
viewForges[stream] = viewForgeDesc.Forges;
additionalForgeables.AddAll(viewForgeDesc.MultikeyForges);
}
else if (streamSpec is NamedWindowConsumerStreamSpec) {
var namedSpec = (NamedWindowConsumerStreamSpec) streamSpec;
var namedWindow =
services.NamedWindowCompileTimeResolver.Resolve(namedSpec.NamedWindow.EventType.Name);
var namedWindowType = namedWindow.EventType;
if (namedSpec.OptPropertyEvaluator != null) {
namedWindowType = namedSpec.OptPropertyEvaluator.FragmentEventType;
}
var typesFilterValidation = new StreamTypeServiceImpl(
namedWindowType,
namedSpec.OptionalStreamName,
false);
var filterSingle =
ExprNodeUtilityMake.ConnectExpressionsByLogicalAndWhenNeeded(namedSpec.FilterExpressions);
var filterQueryGraph = EPLValidationUtil.ValidateFilterGetQueryGraphSafe(
filterSingle,
typesFilterValidation,
@base.StatementRawInfo,
services);
namedWindowConsumers.Add(namedSpec);
viewableActivatorForges[stream] = new ViewableActivatorNamedWindowForge(
namedSpec,
namedWindow,
filterSingle,
filterQueryGraph,
true,
namedSpec.OptPropertyEvaluator);
streamEventTypes[stream] = namedWindowType;
viewForges[stream] = Collections.GetEmptyList<ViewFactoryForge>();
joinAnalysisResult.SetNamedWindowsPerStream(stream, namedWindow);
eventTypeNames[stream] = namedSpec.NamedWindow.EventType.Name;
isNamedWindow[stream] = true;
// Consumers to named windows cannot declare a data window view onto the named window to avoid duplicate remove streams
ViewFactoryForgeDesc viewForgeDesc = ViewFactoryForgeUtil.CreateForges(streamSpec.ViewSpecs, args, namedWindowType);
viewForges[stream] = viewForgeDesc.Forges;
additionalForgeables.AddAll(viewForgeDesc.MultikeyForges);
EPStatementStartMethodHelperValidate.ValidateNoDataWindowOnNamedWindow(viewForges[stream]);
}
else if (streamSpec is TableQueryStreamSpec) {
ValidateNoViews(streamSpec, "Table data");
var tableStreamSpec = (TableQueryStreamSpec) streamSpec;
if (numStreams > 1 && tableStreamSpec.FilterExpressions.Count > 0) {
throw new ExprValidationException(
"Joins with tables do not allow table filter expressions, please add table filters to the where-clause instead");
}
var table = tableStreamSpec.Table;
EPLValidationUtil.ValidateContextName(
true,
table.TableName,
table.OptionalContextName,
statementSpec.Raw.OptionalContextName,
false);
var filter =
ExprNodeUtilityMake.ConnectExpressionsByLogicalAndWhenNeeded(tableStreamSpec.FilterExpressions);
viewableActivatorForges[stream] = new ViewableActivatorTableForge(table, filter);
viewForges[stream] = Collections.GetEmptyList<ViewFactoryForge>();
eventTypeNames[stream] = tableStreamSpec.Table.TableName;
streamEventTypes[stream] = tableStreamSpec.Table.InternalEventType;
joinAnalysisResult.SetTablesForStream(stream, table);
if (tableStreamSpec.Options.IsUnidirectional) {
throw new ExprValidationException("Tables cannot be marked as unidirectional");
}
if (tableStreamSpec.Options.IsRetainIntersection || tableStreamSpec.Options.IsRetainUnion) {
throw new ExprValidationException("Tables cannot be marked with retain");
}
}
else if (streamSpec is DBStatementStreamSpec) {
ValidateNoViews(streamSpec, "Historical data");
var sqlStreamSpec = (DBStatementStreamSpec) streamSpec;
var typeConversionHook = (SQLColumnTypeConversion) ImportUtil.GetAnnotationHook(
statementSpec.Annotations,
HookType.SQLCOL,
typeof(SQLColumnTypeConversion),
services.ImportServiceCompileTime);
var outputRowConversionHook = (SQLOutputRowConversion) ImportUtil.GetAnnotationHook(
statementSpec.Annotations,
HookType.SQLROW,
typeof(SQLOutputRowConversion),
services.ImportServiceCompileTime);
var viewable = HistoricalEventViewableDatabaseForgeFactory.CreateDBStatementView(
stream,
sqlStreamSpec,
typeConversionHook,
outputRowConversionHook,
@base,
services,
statementSpec.Annotations);
streamEventTypes[stream] = viewable.EventType;
viewForges[stream] = Collections.GetEmptyList<ViewFactoryForge>();
viewableActivatorForges[stream] = new ViewableActivatorHistoricalForge(viewable);
historicalEventViewables[stream] = viewable;
}
else if (streamSpec is MethodStreamSpec) {
ValidateNoViews(streamSpec, "Method data");
var methodStreamSpec = (MethodStreamSpec) streamSpec;
var viewable = HistoricalEventViewableMethodForgeFactory.CreateMethodStatementView(
stream,
methodStreamSpec,
@base,
services);
historicalEventViewables[stream] = viewable;
streamEventTypes[stream] = viewable.EventType;
viewForges[stream] = Collections.GetEmptyList<ViewFactoryForge>();
viewableActivatorForges[stream] = new ViewableActivatorHistoricalForge(viewable);
historicalEventViewables[stream] = viewable;
}
else {
throw new IllegalStateException("Unrecognized stream " + streamSpec);
}
// plan serde for iterate-unbound
if (isCanIterateUnbound) {
var serdeForgeables = SerdeEventTypeUtility.Plan(
streamEventTypes[stream], @base.StatementRawInfo, services.SerdeEventTypeRegistry, services.SerdeResolver);
additionalForgeables.AddAll(serdeForgeables);
}
}
// handle match-recognize pattern
if (statementSpec.Raw.MatchRecognizeSpec != null) {
if (numStreams > 1) {
throw new ExprValidationException("Joins are not allowed when using match-recognize");
}
if (joinAnalysisResult.TablesPerStream[0] != null) {
throw new ExprValidationException("Tables cannot be used with match-recognize");
}
var isUnbound = viewForges[0].IsEmpty() &&
!(statementSpec.StreamSpecs[0] is NamedWindowConsumerStreamSpec);
var eventType = viewForges[0].IsEmpty()
? streamEventTypes[0]
: viewForges[0][(viewForges[0].Count - 1)].EventType;
var plan = RowRecogNFAViewPlanUtil.ValidateAndPlan(services.Container, eventType, isUnbound, @base, services);
var forge = new RowRecogNFAViewFactoryForge(plan.Forge);
additionalForgeables.AddAll(plan.AdditionalForgeables);
scheduleHandleCallbackProviders.Add(forge);
viewForges[0].Add(forge);
var serdeForgeables = SerdeEventTypeUtility.Plan(
eventType, @base.StatementRawInfo, services.SerdeEventTypeRegistry, services.SerdeResolver);
additionalForgeables.AddAll(serdeForgeables);
}
// Obtain event types from view factory chains
for (var i = 0; i < viewForges.Length; i++) {
streamEventTypes[i] = viewForges[i].IsEmpty()
? streamEventTypes[i]
: viewForges[i][(viewForges[i].Count - 1)].EventType;
}
// add unique-information to join analysis
joinAnalysisResult.AddUniquenessInfo(viewForges, statementSpec.Annotations);
// plan sub-selects
SubSelectHelperForgePlan subselectForgePlan = SubSelectHelperForgePlanner.PlanSubSelect(
@base, subselectActivation, streamNames, streamEventTypes, eventTypeNames, services);
var subselectForges = subselectForgePlan.Subselects;
additionalForgeables.AddAll(subselectForgePlan.AdditionalForgeables);
DetermineViewSchedules(subselectForges, scheduleHandleCallbackProviders);
// determine view schedules
var viewResourceDelegateExpr = new ViewResourceDelegateExpr();
ViewFactoryForgeUtil.DetermineViewSchedules(viewForges, scheduleHandleCallbackProviders);
var hasIStreamOnly = StatementForgeMethodSelectUtil.GetHasIStreamOnly(isNamedWindow, viewForges);
var optionalStreamsIfAny = OuterJoinAnalyzer.OptionalStreamsIfAny(statementSpec.Raw.OuterJoinDescList);
StreamTypeService typeService = new StreamTypeServiceImpl(
streamEventTypes,
streamNames,
hasIStreamOnly,
false,
optionalStreamsIfAny);
// Validate views that require validation, specifically streams that don't have
// sub-views such as DB SQL joins
var historicalViewableDesc = new HistoricalViewableDesc(numStreams);
for (var stream = 0; stream < historicalEventViewables.Length; stream++) {
var historicalEventViewable = historicalEventViewables[stream];
if (historicalEventViewable == null) {
continue;
}
scheduleHandleCallbackProviders.Add(historicalEventViewable);
IList<StmtClassForgeableFactory> forgeables = historicalEventViewable.Validate(typeService, @base, services);
additionalForgeables.AddAll(forgeables);
historicalViewableDesc.SetHistorical(stream, historicalEventViewable.RequiredStreams);
if (historicalEventViewable.RequiredStreams.Contains(stream)) {
throw new ExprValidationException(
"Parameters for historical stream " +
stream +
" indicate that the stream is subordinate to itself as stream parameters originate in the same stream");
}
}
// Validate where-clause filter tree, outer join clause and output limit expression
var whereClauseValidated = EPStatementStartMethodHelperValidate.ValidateNodes(
statementSpec.Raw,
typeService,
viewResourceDelegateExpr,
@base.StatementRawInfo,
services);
var whereClauseForge = whereClauseValidated?.Forge;
// Obtain result set processor
var resultSetProcessorDesc = ResultSetProcessorFactoryFactory.GetProcessorPrototype(
new ResultSetSpec(statementSpec),
typeService,
viewResourceDelegateExpr,
joinAnalysisResult.UnidirectionalInd,
true,
@base.ContextPropertyRegistry,
false,
false,
@base.StatementRawInfo,
services);
additionalForgeables.AddAll(resultSetProcessorDesc.AdditionalForgeables);
// Handle 'prior' function nodes in terms of view requirements
var viewResourceDelegateDesc =
ViewResourceVerifyHelper.VerifyPreviousAndPriorRequirements(viewForges, viewResourceDelegateExpr);
var hasPrior = ViewResourceDelegateDesc.HasPrior(viewResourceDelegateDesc);
if (hasPrior) {
for (var stream = 0; stream < numStreams; stream++) {
if (!viewResourceDelegateDesc[stream].PriorRequests.IsEmpty()) {
viewForges[stream].Add(new PriorEventViewForge(viewForges[stream].IsEmpty(), streamEventTypes[stream]));
var serdeForgeables = SerdeEventTypeUtility.Plan(
streamEventTypes[stream], @base.StatementRawInfo, services.SerdeEventTypeRegistry, services.SerdeResolver);
additionalForgeables.AddAll(serdeForgeables);
}
}
}
var outputProcessDesc = OutputProcessViewForgeFactory.Make(
typeService.EventTypes,
resultSetProcessorDesc.ResultEventType,
resultSetProcessorDesc.ResultSetProcessorType,
statementSpec,
@base.StatementRawInfo,
services);
var outputProcessViewFactoryForge = outputProcessDesc.Forge;
additionalForgeables.AddAll(outputProcessDesc.AdditionalForgeables);
outputProcessViewFactoryForge.CollectSchedules(scheduleHandleCallbackProviders);
JoinSetComposerPrototypeForge joinForge = null;
if (numStreams > 1) {
var hasAggregations = !resultSetProcessorDesc.AggregationServiceForgeDesc.Expressions.IsEmpty();
var desc = JoinSetComposerPrototypeForgeFactory.MakeComposerPrototype(
statementSpec,
joinAnalysisResult,
typeService,
historicalViewableDesc,
false,
hasAggregations,
@base.StatementRawInfo,
services);
joinForge = desc.Forge;
additionalForgeables.AddAll(desc.AdditionalForgeables);
HandleIndexDependencies(joinForge.OptionalQueryPlan, services);
}
// plan table access
var tableAccessForges =
ExprTableEvalHelperPlan.PlanTableAccess(@base.StatementSpec.TableAccessNodes);
ValidateTableAccessUse(statementSpec.Raw.IntoTableSpec, statementSpec.Raw.TableExpressions);
if (joinAnalysisResult.IsUnidirectional && statementSpec.Raw.IntoTableSpec != null) {
throw new ExprValidationException("Into-table does not allow unidirectional joins");
}
var orderByWithoutOutputLimit = statementSpec.Raw.OrderByList != null &&
!statementSpec.Raw.OrderByList.IsEmpty() &&
statementSpec.Raw.OutputLimitSpec == null;
var statementAIFactoryProviderClassName = CodeGenerationIDGenerator.GenerateClassNameSimple(
typeof(StatementAIFactoryProvider),
classPostfix);
var resultSetProcessorProviderClassName = CodeGenerationIDGenerator.GenerateClassNameSimple(
typeof(ResultSetProcessorFactoryProvider),
classPostfix);
var outputProcessViewProviderClassName = CodeGenerationIDGenerator.GenerateClassNameSimple(
typeof(OutputProcessViewFactoryProvider),
classPostfix);
var statementProviderClassName =
CodeGenerationIDGenerator.GenerateClassNameSimple(typeof(StatementProvider), classPostfix);
var statementFieldsClassName =
CodeGenerationIDGenerator.GenerateClassNameSimple(typeof(StatementFields), classPostfix);
//var statementFieldsClassName = namespaceScope.FieldsClassNameOptional;
var forgeX = new StatementAgentInstanceFactorySelectForge(
typeService.StreamNames,
viewableActivatorForges,
resultSetProcessorProviderClassName,
viewForges,
viewResourceDelegateDesc,
whereClauseForge,
joinForge,
outputProcessViewProviderClassName,
subselectForges,
tableAccessForges,
orderByWithoutOutputLimit,
joinAnalysisResult.IsUnidirectional);
var namespaceScope = new CodegenNamespaceScope(
@namespace,
statementFieldsClassName,
services.IsInstrumented);
var forgeablesX = additionalForgeables
.Select(additional => additional.Make(namespaceScope, classPostfix))
.ToList();
forgeablesX.Add(
new StmtClassForgeableRSPFactoryProvider(
resultSetProcessorProviderClassName,
resultSetProcessorDesc,
namespaceScope,
@base.StatementRawInfo));
forgeablesX.Add(
new StmtClassForgeableOPVFactoryProvider(
outputProcessViewProviderClassName,
outputProcessViewFactoryForge,
namespaceScope,
numStreams,
@base.StatementRawInfo));
forgeablesX.Add(
new StmtClassForgeableAIFactoryProviderSelect(statementAIFactoryProviderClassName, namespaceScope, forgeX));
forgeablesX.Add(
new StmtClassForgeableStmtFields(statementFieldsClassName, namespaceScope, numStreams));
if (!dataflowOperator) {
var informationals = StatementInformationalsUtil.GetInformationals(
@base,
filterSpecCompileds,
scheduleHandleCallbackProviders,
namedWindowConsumers,
true,
resultSetProcessorDesc.SelectSubscriberDescriptor,
namespaceScope,
services);
forgeablesX.Add(
new StmtClassForgeableStmtProvider(statementAIFactoryProviderClassName, statementProviderClassName, informationals, namespaceScope));
}
var forgeableResult = new StmtForgeMethodResult(
forgeablesX,
filterSpecCompileds,
scheduleHandleCallbackProviders,
namedWindowConsumers,
FilterSpecCompiled.MakeExprNodeList(filterSpecCompileds, EmptyList<FilterSpecParamExprNodeForge>.Instance));
return new StmtForgeMethodSelectResult(forgeableResult, resultSetProcessorDesc.ResultEventType, numStreams);
}
public FAFQueryMethodSelectDesc(
StatementSpecCompiled statementSpec,
Compilable compilable,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
Annotations = statementSpec.Annotations;
ContextName = statementSpec.Raw.OptionalContextName;
var queryPlanLogging = services.Configuration.Common.Logging.IsEnableQueryPlan;
if (queryPlanLogging) {
QUERY_PLAN_LOG.Info("Query plans for Fire-and-forget query '" + compilable.ToEPL() + "'");
}
HasTableAccess = statementSpec.TableAccessNodes != null && statementSpec.TableAccessNodes.Count > 0;
foreach (var streamSpec in statementSpec.StreamSpecs) {
HasTableAccess |= streamSpec is TableQueryStreamSpec;
}
HasTableAccess |= StatementLifecycleSvcUtil.IsSubqueryWithTable(
statementSpec.SubselectNodes, services.TableCompileTimeResolver);
IsDistinct = statementSpec.SelectClauseCompiled.IsDistinct;
FAFQueryMethodHelper.ValidateFAFQuery(statementSpec);
var numStreams = statementSpec.StreamSpecs.Length;
var typesPerStream = new EventType[numStreams];
var namesPerStream = new string[numStreams];
var eventTypeNames = new string[numStreams];
Processors = new FireAndForgetProcessorForge[numStreams];
ConsumerFilters = new ExprNode[numStreams];
// check context partition use
if (statementSpec.Raw.OptionalContextName != null) {
if (numStreams > 1) {
throw new ExprValidationException(
"Joins in runtime queries for context partitions are not supported");
}
}
// resolve types and processors
for (var i = 0; i < numStreams; i++) {
var streamSpec = statementSpec.StreamSpecs[i];
Processors[i] = FireAndForgetProcessorForgeFactory.ValidateResolveProcessor(streamSpec);
if (numStreams > 1 && Processors[i].ContextName != null) {
throw new ExprValidationException(
"Joins against named windows that are under context are not supported");
}
var streamName = Processors[i].NamedWindowOrTableName;
if (streamSpec.OptionalStreamName != null) {
streamName = streamSpec.OptionalStreamName;
}
namesPerStream[i] = streamName;
typesPerStream[i] = Processors[i].EventTypeRspInputEvents;
eventTypeNames[i] = typesPerStream[i].Name;
IList<ExprNode> consumerFilterExprs;
if (streamSpec is NamedWindowConsumerStreamSpec) {
var namedSpec = (NamedWindowConsumerStreamSpec) streamSpec;
consumerFilterExprs = namedSpec.FilterExpressions;
}
else {
var tableSpec = (TableQueryStreamSpec) streamSpec;
consumerFilterExprs = tableSpec.FilterExpressions;
}
ConsumerFilters[i] = ExprNodeUtilityMake.ConnectExpressionsByLogicalAndWhenNeeded(consumerFilterExprs);
}
// compile filter to optimize access to named window
var optionalStreamsIfAny = OuterJoinAnalyzer.OptionalStreamsIfAny(statementSpec.Raw.OuterJoinDescList);
var types = new StreamTypeServiceImpl(
typesPerStream,
namesPerStream,
new bool[numStreams],
false,
optionalStreamsIfAny);
var excludePlanHint = ExcludePlanHint.GetHint(types.StreamNames, statementRawInfo, services);
QueryGraph = new QueryGraphForge(numStreams, excludePlanHint, false);
if (statementSpec.Raw.WhereClause != null) {
for (var i = 0; i < numStreams; i++) {
try {
var validationContext = new ExprValidationContextBuilder(types, statementRawInfo, services)
.WithAllowBindingConsumption(true)
.WithIsFilterExpression(true)
.Build();
var validated = ExprNodeUtilityValidate.GetValidatedSubtree(
ExprNodeOrigin.FILTER,
statementSpec.Raw.WhereClause,
validationContext);
FilterExprAnalyzer.Analyze(validated, QueryGraph, false);
}
catch (Exception ex) {
Log.Warn("Unexpected exception analyzing filter paths: " + ex.Message, ex);
}
}
}
// handle subselects
// first we create streams for subselects, if there are any
var @base = new StatementBaseInfo(compilable, statementSpec, null, statementRawInfo, null);
var subqueryNamedWindowConsumers = new List<NamedWindowConsumerStreamSpec>();
SubSelectActivationDesc subSelectActivationDesc = SubSelectHelperActivations.CreateSubSelectActivation(
EmptyList<FilterSpecCompiled>.Instance, subqueryNamedWindowConsumers, @base, services);
IDictionary<ExprSubselectNode, SubSelectActivationPlan> subselectActivation = subSelectActivationDesc.Subselects;
AdditionalForgeables.AddAll(subSelectActivationDesc.AdditionalForgeables);
SubSelectHelperForgePlan subSelectForgePlan = SubSelectHelperForgePlanner.PlanSubSelect(
@base, subselectActivation, namesPerStream, typesPerStream, eventTypeNames, services);
SubselectForges = subSelectForgePlan.Subselects;
AdditionalForgeables.AddAll(subSelectForgePlan.AdditionalForgeables);
// obtain result set processor
var isIStreamOnly = new bool[namesPerStream.Length];
isIStreamOnly.Fill(true);
StreamTypeService typeService = new StreamTypeServiceImpl(
typesPerStream,
namesPerStream,
isIStreamOnly,
true,
optionalStreamsIfAny);
WhereClause = EPStatementStartMethodHelperValidate.ValidateNodes(
statementSpec.Raw,
typeService,
null,
statementRawInfo,
services);
var resultSetSpec = new ResultSetSpec(statementSpec);
ResultSetProcessor = ResultSetProcessorFactoryFactory.GetProcessorPrototype(
resultSetSpec,
typeService,
null,
new bool[0],
true,
null,
true,
false,
statementRawInfo,
services);
AdditionalForgeables.AddAll(ResultSetProcessor.AdditionalForgeables);
// plan table access
TableAccessForges = ExprTableEvalHelperPlan.PlanTableAccess(statementSpec.Raw.TableExpressions);
// plan joins or simple queries
if (numStreams > 1) {
var streamJoinAnalysisResult = new StreamJoinAnalysisResultCompileTime(numStreams);
CompatExtensions.Fill(streamJoinAnalysisResult.NamedWindowsPerStream, (NamedWindowMetaData) null);
for (var i = 0; i < numStreams; i++) {
var uniqueIndexes = Processors[i].UniqueIndexes;
streamJoinAnalysisResult.UniqueKeys[i] = uniqueIndexes;
}
var hasAggregations = ResultSetProcessor.ResultSetProcessorType.IsAggregated();
var desc = JoinSetComposerPrototypeForgeFactory.MakeComposerPrototype(
statementSpec,
streamJoinAnalysisResult,
types,
new HistoricalViewableDesc(numStreams),
true,
hasAggregations,
statementRawInfo,
services);
AdditionalForgeables.AddAll(desc.AdditionalForgeables);
Joins = desc.Forge;
}
else {
Joins = null;
}
var multiKeyPlan = MultiKeyPlanner.PlanMultiKeyDistinct(
IsDistinct, ResultSetProcessor.ResultEventType, statementRawInfo, SerdeCompileTimeResolverNonHA.INSTANCE);
AdditionalForgeables.AddAll(multiKeyPlan.MultiKeyForgeables);
DistinctMultiKey = multiKeyPlan.ClassRef;
}
