public DataFlowOpInitializeResult Initialize(DataFlowOpInitializateContext context) { if (context.InputPorts.IsEmpty()) { throw new ArgumentException("Select operator requires at least one input stream"); } if (context.OutputPorts.Count != 1) { throw new ArgumentException("Select operator requires one output stream but produces " + context.OutputPorts.Count + " streams"); } DataFlowOpOutputPort portZero = context.OutputPorts[0]; if (portZero.OptionalDeclaredType != null && !portZero.OptionalDeclaredType.IsUnderlying) { _submitEventBean = true; } // determine adapter factories for each type int numStreams = context.InputPorts.Count; _adapterFactories = new EventBeanAdapterFactory[numStreams]; for (int i = 0; i < numStreams; i++) { EventType eventType = context.InputPorts.Get(i).TypeDesc.EventType; _adapterFactories[i] = context.StatementContext.EventAdapterService.GetAdapterFactoryForType(eventType); } // Compile and prepare execution // StatementContext statementContext = context.StatementContext; EPServicesContext servicesContext = context.ServicesContext; AgentInstanceContext agentInstanceContext = context.AgentInstanceContext; // validate if (select.InsertIntoDesc != null) { throw new ExprValidationException("Insert-into clause is not supported"); } if (select.SelectStreamSelectorEnum != SelectClauseStreamSelectorEnum.ISTREAM_ONLY) { throw new ExprValidationException("Selecting remove-stream is not supported"); } ExprNodeSubselectDeclaredDotVisitor visitor = StatementSpecRawAnalyzer.WalkSubselectAndDeclaredDotExpr(select); GroupByClauseExpressions groupByExpressions = GroupByExpressionHelper.GetGroupByRollupExpressions( servicesContext.Container, select.GroupByExpressions, select.SelectClauseSpec, select.HavingExprRootNode, select.OrderByList, visitor); if (!visitor.Subselects.IsEmpty()) { throw new ExprValidationException("Subselects are not supported"); } IDictionary <int, FilterStreamSpecRaw> streams = new Dictionary <int, FilterStreamSpecRaw>(); for (int streamNum = 0; streamNum < select.StreamSpecs.Count; streamNum++) { var rawStreamSpec = select.StreamSpecs[streamNum]; if (!(rawStreamSpec is FilterStreamSpecRaw)) { throw new ExprValidationException("From-clause must contain only streams and cannot contain patterns or other constructs"); } streams.Put(streamNum, (FilterStreamSpecRaw)rawStreamSpec); } // compile offered streams IList <StreamSpecCompiled> streamSpecCompileds = new List <StreamSpecCompiled>(); for (int streamNum = 0; streamNum < select.StreamSpecs.Count; streamNum++) { var filter = streams.Get(streamNum); var inputPort = FindInputPort(filter.RawFilterSpec.EventTypeName, context.InputPorts); if (inputPort == null) { throw new ExprValidationException( string.Format("Failed to find stream '{0}' among input ports, input ports are {1}", filter.RawFilterSpec.EventTypeName, GetInputPortNames(context.InputPorts).Render(", ", "[]"))); } var eventType = inputPort.Value.Value.TypeDesc.EventType; var streamAlias = filter.OptionalStreamName; var filterSpecCompiled = new FilterSpecCompiled(eventType, streamAlias, new IList <FilterSpecParam>[] { Collections.GetEmptyList <FilterSpecParam>() }, null); var filterStreamSpecCompiled = new FilterStreamSpecCompiled(filterSpecCompiled, select.StreamSpecs[0].ViewSpecs, streamAlias, StreamSpecOptions.DEFAULT); streamSpecCompileds.Add(filterStreamSpecCompiled); } // create compiled statement spec SelectClauseSpecCompiled selectClauseCompiled = StatementLifecycleSvcUtil.CompileSelectClause(select.SelectClauseSpec); // determine if snapshot output is needed OutputLimitSpec outputLimitSpec = select.OutputLimitSpec; _isOutputLimited = outputLimitSpec != null; if (iterate) { if (outputLimitSpec != null) { throw new ExprValidationException("Output rate limiting is not supported with 'iterate'"); } outputLimitSpec = new OutputLimitSpec(OutputLimitLimitType.SNAPSHOT, OutputLimitRateType.TERM); } var mergedAnnotations = AnnotationUtil.MergeAnnotations(statementContext.Annotations, context.OperatorAnnotations); var orderByArray = OrderByItem.ToArray(select.OrderByList); var outerJoinArray = OuterJoinDesc.ToArray(select.OuterJoinDescList); var streamSpecArray = streamSpecCompileds.ToArray(); var compiled = new StatementSpecCompiled(null, null, null, null, null, null, null, SelectClauseStreamSelectorEnum.ISTREAM_ONLY, selectClauseCompiled, streamSpecArray, outerJoinArray, select.FilterExprRootNode, select.HavingExprRootNode, outputLimitSpec, orderByArray, ExprSubselectNode.EMPTY_SUBSELECT_ARRAY, ExprNodeUtility.EMPTY_DECLARED_ARR, ExprNodeUtility.EMPTY_SCRIPTS, select.ReferencedVariables, select.RowLimitSpec, CollectionUtil.EMPTY_STRING_ARRAY, mergedAnnotations, null, null, null, null, null, null, null, null, null, groupByExpressions, null, null); // create viewable per port var viewables = new EPLSelectViewable[context.InputPorts.Count]; _viewablesPerPort = viewables; foreach (var entry in context.InputPorts) { EPLSelectViewable viewable = new EPLSelectViewable(entry.Value.TypeDesc.EventType); viewables[entry.Key] = viewable; } var activatorFactory = new ProxyViewableActivatorFactory { ProcCreateActivatorSimple = filterStreamSpec => { EPLSelectViewable found = null; foreach (EPLSelectViewable sviewable in viewables) { if (sviewable.EventType == filterStreamSpec.FilterSpec.FilterForEventType) { found = sviewable; } } if (found == null) { throw new IllegalStateException("Failed to find viewable for filter"); } EPLSelectViewable viewable = found; return(new ProxyViewableActivator( (agentInstanceContext2, isSubselect, isRecoveringResilient) => new ViewableActivationResult( viewable, new ProxyStopCallback(() => { }), null, null, null, false, false, null))); } }; // for per-row deliver, register select expression result callback OutputProcessViewCallback optionalOutputProcessViewCallback = null; if (!iterate && !_isOutputLimited) { _deliveryCallback = new EPLSelectDeliveryCallback(); optionalOutputProcessViewCallback = this; } // prepare EPStatementStartMethodSelectDesc selectDesc = EPStatementStartMethodSelectUtil.Prepare(compiled, servicesContext, statementContext, false, agentInstanceContext, false, activatorFactory, optionalOutputProcessViewCallback, _deliveryCallback); // start _selectResult = (StatementAgentInstanceFactorySelectResult)selectDesc.StatementAgentInstanceFactorySelect.NewContext(agentInstanceContext, false); // for output-rate-limited, register a dispatch view if (_isOutputLimited) { _selectResult.FinalView.AddView(new EPLSelectUpdateDispatchView(this)); } // assign strategies to expression nodes EPStatementStartMethodHelperAssignExpr.AssignExpressionStrategies( selectDesc, _selectResult.OptionalAggegationService, _selectResult.SubselectStrategies, _selectResult.PriorNodeStrategies, _selectResult.PreviousNodeStrategies, null, null, _selectResult.TableAccessEvalStrategies); EventType outputEventType = selectDesc.ResultSetProcessorPrototypeDesc.ResultSetProcessorFactory.ResultEventType; _agentInstanceContext = agentInstanceContext; return(new DataFlowOpInitializeResult(new GraphTypeDesc[] { new GraphTypeDesc(false, true, outputEventType) })); }
public static JoinSetComposerPrototypeDesc MakeComposerPrototype( StatementSpecCompiled spec, StreamJoinAnalysisResultCompileTime joinAnalysisResult, StreamTypeService typeService, HistoricalViewableDesc historicalViewableDesc, bool isOnDemandQuery, bool hasAggregations, StatementRawInfo statementRawInfo, StatementCompileTimeServices compileTimeServices) { var streamTypes = typeService.EventTypes; var streamNames = typeService.StreamNames; var whereClause = spec.Raw.WhereClause; var queryPlanLogging = compileTimeServices.Configuration.Common.Logging.IsEnableQueryPlan; var additionalForgeables = new List<StmtClassForgeableFactory>(); // Determine if there is a historical stream, and what dependencies exist var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false); for (var i = 0; i < streamTypes.Length; i++) { if (historicalViewableDesc.Historical[i]) { var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i]; historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn); } } if (Log.IsDebugEnabled) { Log.Debug("Dependency graph: " + historicalDependencyGraph); } // Handle a join with a database or other historical data source for 2 streams var outerJoinDescs = OuterJoinDesc.ToArray(spec.Raw.OuterJoinDescList); if (historicalViewableDesc.IsHistorical && streamTypes.Length == 2) { var desc = MakeComposerHistorical2Stream( outerJoinDescs, whereClause, streamTypes, streamNames, historicalViewableDesc, queryPlanLogging, statementRawInfo, compileTimeServices); return new JoinSetComposerPrototypeDesc(desc.Forge, desc.AdditionalForgeables); } var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescs); // Query graph for graph relationships between streams/historicals // For outer joins the query graph will just contain outer join relationships var hint = ExcludePlanHint.GetHint( typeService.StreamNames, statementRawInfo, compileTimeServices); var queryGraph = new QueryGraphForge(streamTypes.Length, hint, false); if (outerJoinDescs.Length > 0) { OuterJoinAnalyzer.Analyze(outerJoinDescs, queryGraph); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After outer join filterQueryGraph=\n" + queryGraph); } } // Let the query graph reflect the where-clause if (whereClause != null) { // Analyze relationships between streams using the optional filter expression. // Relationships are properties in AND and EQUALS nodes of joins. FilterExprAnalyzer.Analyze(whereClause, queryGraph, isOuterJoins); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After filter expression filterQueryGraph=\n" + queryGraph); } // Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c) QueryGraphForge.FillEquivalentNav(streamTypes, queryGraph); if (Log.IsDebugEnabled) { Log.Debug(".makeComposer After fill equiv. nav. filterQueryGraph=\n" + queryGraph); } } // Historical index lists var historicalStreamIndexLists = new HistoricalStreamIndexListForge[streamTypes.Length]; var queryPlanDesc = QueryPlanBuilder.GetPlan( streamTypes, outerJoinDescs, queryGraph, typeService.StreamNames, historicalViewableDesc, historicalDependencyGraph, historicalStreamIndexLists, joinAnalysisResult, queryPlanLogging, statementRawInfo, compileTimeServices); QueryPlanForge queryPlan = queryPlanDesc.Forge; additionalForgeables.AddAll(queryPlanDesc.AdditionalForgeables); // remove unused indexes - consider all streams or all unidirectional var usedIndexes = new HashSet<TableLookupIndexReqKey>(); var indexSpecs = queryPlan.IndexSpecs; for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) { var planNode = queryPlan.ExecNodeSpecs[streamNum]; planNode?.AddIndexes(usedIndexes); } foreach (var indexSpec in indexSpecs) { if (indexSpec == null) { continue; } var items = indexSpec.Items; var indexNames = items.Keys.ToArray(); foreach (var indexName in indexNames) { if (!usedIndexes.Contains(indexName)) { items.Remove(indexName); } } } // plan multikeys IList<StmtClassForgeableFactory> multikeyForgeables = PlanMultikeys( indexSpecs, statementRawInfo, compileTimeServices); additionalForgeables.AddAll(multikeyForgeables); QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook( spec.Annotations, compileTimeServices.ImportServiceCompileTime); if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) { QUERY_PLAN_LOG.Info("Query plan: " + queryPlan.ToQueryPlan()); hook?.Join(queryPlan); } var selectsRemoveStream = spec.Raw.SelectStreamSelectorEnum.IsSelectsRStream() || spec.Raw.OutputLimitSpec != null; var joinRemoveStream = selectsRemoveStream || hasAggregations; ExprNode postJoinEvaluator; if (JoinSetComposerUtil.IsNonUnidirectionalNonSelf( isOuterJoins, joinAnalysisResult.IsUnidirectional, joinAnalysisResult.IsPureSelfJoin)) { postJoinEvaluator = GetFilterExpressionInclOnClause( spec.Raw.WhereClause, outerJoinDescs, statementRawInfo, compileTimeServices); } else { postJoinEvaluator = spec.Raw.WhereClause; } JoinSetComposerPrototypeGeneralForge forge = new JoinSetComposerPrototypeGeneralForge( typeService.EventTypes, postJoinEvaluator, outerJoinDescs.Length > 0, queryPlan, joinAnalysisResult, typeService.StreamNames, joinRemoveStream, historicalViewableDesc.IsHistorical); return new JoinSetComposerPrototypeDesc(forge, additionalForgeables); }