コード例 #1
0
ファイル: EPLValidationUtil.cs プロジェクト: lanicon/nesper
 public static void ValidateFilterWQueryGraphSafe(
     QueryGraphForge queryGraph,
     ExprNode filterExpression,
     StreamTypeServiceImpl typeService,
     StatementRawInfo statementRawInfo,
     StatementCompileTimeServices services)
 {
     try {
         var validationContext = new ExprValidationContextBuilder(typeService, statementRawInfo, services)
             .WithAllowBindingConsumption(true)
             .WithIsFilterExpression(true)
             .Build();
         var validated = ExprNodeUtilityValidate.GetValidatedSubtree(
             ExprNodeOrigin.FILTER,
             filterExpression,
             validationContext);
         FilterExprAnalyzer.Analyze(validated, queryGraph, false);
     }
     catch (Exception ex) {
         Log.Warn(
             "Unexpected exception analyzing filterable expression '" +
             ExprNodeUtilityPrint.ToExpressionStringMinPrecedenceSafe(filterExpression) +
             "': " +
             ex.Message,
             ex);
     }
 }
コード例 #2
0
        private static Pair <HistoricalIndexLookupStrategy, PollResultIndexingStrategy> DetermineIndexing(
            ExprNode filterForIndexing,
            EventType polledViewType,
            EventType streamViewType,
            int polledViewStreamNum,
            int streamViewStreamNum,
            StatementContext statementContext,
            string[] streamNames)
        {
            // No filter means unindexed event tables
            if (filterForIndexing == null)
            {
                return(new Pair <HistoricalIndexLookupStrategy, PollResultIndexingStrategy>(
                           new HistoricalIndexLookupStrategyNoIndex(), new PollResultIndexingStrategyNoIndex()));
            }

            // analyze query graph; Whereas stream0=named window, stream1=delete-expr filter
            var hint       = ExcludePlanHint.GetHint(streamNames, statementContext);
            var queryGraph = new QueryGraph(2, hint, false);

            FilterExprAnalyzer.Analyze(filterForIndexing, queryGraph, false);

            return(DetermineIndexing(
                       queryGraph, polledViewType, streamViewType, polledViewStreamNum, streamViewStreamNum));
        }
コード例 #3
0
        private static JoinSetComposerPrototypeHistoricalDesc DetermineIndexing(
            ExprNode filterForIndexing,
            EventType polledViewType,
            EventType streamViewType,
            int polledViewStreamNum,
            int streamViewStreamNum,
            string[] streamNames,
            StatementRawInfo rawInfo,
            StatementCompileTimeServices services)
        {
            // No filter means unindexed event tables
            if (filterForIndexing == null) {
                return new JoinSetComposerPrototypeHistoricalDesc(
                    HistoricalIndexLookupStrategyNoIndexForge.INSTANCE,
                    PollResultIndexingStrategyNoIndexForge.INSTANCE,
                    EmptyList<StmtClassForgeableFactory>.Instance);
            }

            // analyze query graph; Whereas stream0=named window, stream1=delete-expr filter
            var hint = ExcludePlanHint.GetHint(streamNames, rawInfo, services);
            var queryGraph = new QueryGraphForge(2, hint, false);
            FilterExprAnalyzer.Analyze(filterForIndexing, queryGraph, false);

            return DetermineIndexing(
                queryGraph,
                polledViewType,
                streamViewType,
                polledViewStreamNum,
                streamViewStreamNum,
                rawInfo,
                services.SerdeResolver);
        }
コード例 #4
0
        public static SubordPropPlan GetJoinProps(
            ExprNode filterExpr,
            int outsideStreamCount,
            EventType[] allStreamTypesZeroIndexed,
            ExcludePlanHint excludePlanHint)
        {
            // No filter expression means full table scan
            if (filterExpr == null) {
                return new SubordPropPlan();
            }

            // analyze query graph
            var queryGraph = new QueryGraphForge(outsideStreamCount + 1, excludePlanHint, true);
            FilterExprAnalyzer.Analyze(filterExpr, queryGraph, false);

            // Build a list of streams and indexes
            var joinProps = new LinkedHashMap<string, SubordPropHashKeyForge>();
            var rangeProps = new LinkedHashMap<string, SubordPropRangeKeyForge>();
            IDictionary<QueryGraphValueEntryCustomKeyForge, QueryGraphValueEntryCustomOperationForge> customIndexOps =
                EmptyDictionary<QueryGraphValueEntryCustomKeyForge, QueryGraphValueEntryCustomOperationForge>.Instance;

            for (var stream = 0; stream < outsideStreamCount; stream++) {
                var lookupStream = stream + 1;

                var queryGraphValue = queryGraph.GetGraphValue(lookupStream, 0);
                var hashKeysAndIndexes = queryGraphValue.HashKeyProps;

                // determine application functions
                foreach (var item in queryGraphValue.Items) {
                    if (item.Entry is QueryGraphValueEntryCustomForge) {
                        if (customIndexOps.IsEmpty()) {
                            customIndexOps =
                                new Dictionary<QueryGraphValueEntryCustomKeyForge,
                                    QueryGraphValueEntryCustomOperationForge>();
                        }

                        var custom = (QueryGraphValueEntryCustomForge) item.Entry;
                        custom.MergeInto(customIndexOps);
                    }
                }

                // handle key-lookups
                var keyPropertiesJoin = hashKeysAndIndexes.Keys;
                var indexPropertiesJoin = hashKeysAndIndexes.Indexed;
                if (!keyPropertiesJoin.IsEmpty()) {
                    if (keyPropertiesJoin.Count != indexPropertiesJoin.Length) {
                        throw new IllegalStateException(
                            "Invalid query key and index property collection for stream " + stream);
                    }

                    for (var i = 0; i < keyPropertiesJoin.Count; i++) {
                        QueryGraphValueEntryHashKeyedForge keyDesc = keyPropertiesJoin[i];
                        var compareNode = keyDesc.KeyExpr;

                        var keyPropType = compareNode.Forge.EvaluationType.GetBoxedType();
                        var indexedPropType = allStreamTypesZeroIndexed[0]
                            .GetPropertyType(indexPropertiesJoin[i])
                            .GetBoxedType();
                        var coercionType = indexedPropType;
                        if (keyPropType != indexedPropType) {
                            coercionType = keyPropType.GetCompareToCoercionType(indexedPropType);
                        }

                        SubordPropHashKeyForge desc;
                        if (keyPropertiesJoin[i] is QueryGraphValueEntryHashKeyedForgeExpr) {
                            var keyExpr = (QueryGraphValueEntryHashKeyedForgeExpr) keyPropertiesJoin[i];
                            var keyStreamNum = keyExpr.IsRequiresKey ? (int?) stream : null;
                            desc = new SubordPropHashKeyForge(keyDesc, keyStreamNum, coercionType);
                        }
                        else {
                            var prop = (QueryGraphValueEntryHashKeyedForgeProp) keyDesc;
                            desc = new SubordPropHashKeyForge(prop, stream, coercionType);
                        }

                        joinProps.Put(indexPropertiesJoin[i], desc);
                    }
                }

                // handle range lookups
                var rangeKeysAndIndexes = queryGraphValue.RangeProps;
                var rangeIndexes = rangeKeysAndIndexes.Indexed;
                var rangeDescs = rangeKeysAndIndexes.Keys;
                if (rangeDescs.IsEmpty()) {
                    continue;
                }

                // get all ranges lookups
                var count = -1;
                foreach (var rangeDesc in rangeDescs) {
                    count++;
                    var rangeIndexProp = rangeIndexes[count];

                    var subqRangeDesc = rangeProps.Get(rangeIndexProp);

                    // other streams may specify the start or end endpoint of a range, therefore this operation can be additive
                    if (subqRangeDesc != null) {
                        if (subqRangeDesc.RangeInfo.Type.IsRange()) {
                            continue;
                        }

                        // see if we can make this additive by using a range
                        var relOpOther = (QueryGraphValueEntryRangeRelOpForge) subqRangeDesc.RangeInfo;
                        var relOpThis = (QueryGraphValueEntryRangeRelOpForge) rangeDesc;

                        var opsDesc = QueryGraphRangeUtil.GetCanConsolidate(relOpThis.Type, relOpOther.Type);
                        if (opsDesc != null) {
                            ExprNode start;
                            ExprNode end;
                            if (!opsDesc.IsReverse) {
                                start = relOpOther.Expression;
                                end = relOpThis.Expression;
                            }
                            else {
                                start = relOpThis.Expression;
                                end = relOpOther.Expression;
                            }

                            var allowRangeReversal = relOpOther.IsBetweenPart && relOpThis.IsBetweenPart;
                            var rangeIn = new QueryGraphValueEntryRangeInForge(
                                opsDesc.Type,
                                start,
                                end,
                                allowRangeReversal);

                            var indexedPropType = allStreamTypesZeroIndexed[0]
                                .GetPropertyType(rangeIndexProp)
                                .GetBoxedType();
                            var coercionType = indexedPropType;
                            var proposedType = CoercionUtil.GetCoercionTypeRangeIn(
                                indexedPropType,
                                rangeIn.ExprStart,
                                rangeIn.ExprEnd);
                            if (proposedType != null && proposedType != indexedPropType) {
                                coercionType = proposedType;
                            }

                            subqRangeDesc = new SubordPropRangeKeyForge(rangeIn, coercionType);
                            rangeProps.Put(rangeIndexProp, subqRangeDesc);
                        }

                        // ignore
                        continue;
                    }

                    // an existing entry has not been found
                    if (rangeDesc.Type.IsRange()) {
                        var rangeIn = (QueryGraphValueEntryRangeInForge) rangeDesc;
                        var indexedPropType =
                            allStreamTypesZeroIndexed[0].GetPropertyType(rangeIndexProp).GetBoxedType();
                        var coercionType = indexedPropType;
                        var proposedType = CoercionUtil.GetCoercionTypeRangeIn(
                            indexedPropType,
                            rangeIn.ExprStart,
                            rangeIn.ExprEnd);
                        if (proposedType != null && proposedType != indexedPropType) {
                            coercionType = proposedType;
                        }

                        subqRangeDesc = new SubordPropRangeKeyForge(rangeDesc, coercionType);
                    }
                    else {
                        var relOp = (QueryGraphValueEntryRangeRelOpForge) rangeDesc;
                        var keyPropType = relOp.Expression.Forge.EvaluationType;
                        var indexedPropType =
                            allStreamTypesZeroIndexed[0].GetPropertyType(rangeIndexProp).GetBoxedType();
                        var coercionType = indexedPropType;
                        if (keyPropType != indexedPropType) {
                            coercionType = keyPropType.GetCompareToCoercionType(indexedPropType);
                        }

                        subqRangeDesc = new SubordPropRangeKeyForge(rangeDesc, coercionType);
                    }

                    rangeProps.Put(rangeIndexProp, subqRangeDesc);
                }
            }

            SubordPropInKeywordSingleIndex inKeywordSingleIdxProp = null;
            SubordPropInKeywordMultiIndex inKeywordMultiIdxProp = null;
            if (joinProps.IsEmpty() && rangeProps.IsEmpty()) {
                for (var stream = 0; stream < outsideStreamCount; stream++) {
                    var lookupStream = stream + 1;
                    var queryGraphValue = queryGraph.GetGraphValue(lookupStream, 0);

                    var inkwSingles = queryGraphValue.InKeywordSingles;
                    if (inkwSingles.Indexed.Length != 0) {
                        ExprNode[] keys = inkwSingles.Key[0].KeyExprs;
                        var key = inkwSingles.Indexed[0];
                        if (inKeywordSingleIdxProp != null) {
                            continue;
                        }

                        var coercionType = keys[0].Forge.EvaluationType; // for in-comparison the same type is required
                        inKeywordSingleIdxProp = new SubordPropInKeywordSingleIndex(key, coercionType, keys);
                    }

                    var inkwMultis = queryGraphValue.InKeywordMulti;
                    if (!inkwMultis.IsEmpty()) {
                        QueryGraphValuePairInKWMultiIdx multi = inkwMultis[0];
                        inKeywordMultiIdxProp = new SubordPropInKeywordMultiIndex(
                            ExprNodeUtilityQuery.GetIdentResolvedPropertyNames(multi.Indexed),
                            multi.Indexed[0].Forge.EvaluationType,
                            multi.Key.KeyExpr);
                    }

                    if (inKeywordSingleIdxProp != null && inKeywordMultiIdxProp != null) {
                        inKeywordMultiIdxProp = null;
                    }
                }
            }

            return new SubordPropPlan(
                joinProps,
                rangeProps,
                inKeywordSingleIdxProp,
                inKeywordMultiIdxProp,
                customIndexOps);
        }
コード例 #5
0
        public void TestGetPlan()
        {
            OuterJoinDesc[] descList =
            {
                SupportOuterJoinDescFactory.MakeDesc(
                    container, "IntPrimitive", "s0", "IntBoxed", "s1", OuterJoinType.LEFT)
            };

            var queryGraph = new QueryGraphForge(2, null, false);
            var plan       = QueryPlanBuilder.GetPlan(
                typesPerStream,
                new OuterJoinDesc[0],
                queryGraph,
                null,
                new HistoricalViewableDesc(5),
                dependencyGraph,
                null,
                new StreamJoinAnalysisResultCompileTime(2),
                true,
                null,
                null);

            AssertPlan(plan.Forge);

            plan = QueryPlanBuilder.GetPlan(
                typesPerStream,
                descList,
                queryGraph,
                null,
                new HistoricalViewableDesc(5),
                dependencyGraph,
                null,
                new StreamJoinAnalysisResultCompileTime(2),
                true,
                null,
                null);
            AssertPlan(plan.Forge);

            FilterExprAnalyzer.Analyze(SupportExprNodeFactory.GetInstance(container).MakeEqualsNode(), queryGraph, false);
            plan = QueryPlanBuilder.GetPlan(
                typesPerStream,
                descList,
                queryGraph,
                null,
                new HistoricalViewableDesc(5),
                dependencyGraph,
                null,
                new StreamJoinAnalysisResultCompileTime(2),
                true,
                null,
                null);
            AssertPlan(plan.Forge);

            plan = QueryPlanBuilder.GetPlan(
                typesPerStream,
                new OuterJoinDesc[0],
                queryGraph,
                null,
                new HistoricalViewableDesc(5),
                dependencyGraph,
                null,
                new StreamJoinAnalysisResultCompileTime(2),
                true,
                null,
                null);
            AssertPlan(plan.Forge);
        }
コード例 #6
0
        /// <summary>
        /// Builds join tuple composer.
        /// </summary>
        /// <param name="statementName">Name of the statement.</param>
        /// <param name="statementId">The statement identifier.</param>
        /// <param name="outerJoinDescList">list of descriptors for outer join criteria</param>
        /// <param name="optionalFilterNode">filter tree for analysis to build indexes for fast access</param>
        /// <param name="streamTypes">types of streams</param>
        /// <param name="streamNames">names of streams</param>
        /// <param name="streamJoinAnalysisResult">The stream join analysis result.</param>
        /// <param name="queryPlanLogging">if set to <c>true</c> [query plan logging].</param>
        /// <param name="statementContext">The statement context.</param>
        /// <param name="historicalViewableDesc">The historical viewable desc.</param>
        /// <param name="exprEvaluatorContext">The expr evaluator context.</param>
        /// <param name="selectsRemoveStream">if set to <c>true</c> [selects remove stream].</param>
        /// <param name="hasAggregations">if set to <c>true</c> [has aggregations].</param>
        /// <param name="tableService">The table service.</param>
        /// <param name="isOnDemandQuery">if set to <c>true</c> [is on demand query].</param>
        /// <param name="allowIndexInit">if set to <c>true</c> [allow index initialize].</param>
        /// <returns>
        /// composer implementation
        /// </returns>
        /// <throws>com.espertech.esper.epl.expression.core.ExprValidationException is thrown to indicate thatvalidation of view use in joins failed.
        /// {D255958A-8513-4226-94B9-080D98F904A1}</throws>
        public static JoinSetComposerPrototype MakeComposerPrototype(string statementName, int statementId, OuterJoinDesc[] outerJoinDescList, ExprNode optionalFilterNode, EventType[] streamTypes, string[] streamNames, StreamJoinAnalysisResult streamJoinAnalysisResult, bool queryPlanLogging, StatementContext statementContext, HistoricalViewableDesc historicalViewableDesc, ExprEvaluatorContext exprEvaluatorContext, bool selectsRemoveStream, bool hasAggregations, TableService tableService, bool isOnDemandQuery, bool allowIndexInit)
        {
            // Determine if there is a historical stream, and what dependencies exist
            var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false);

            for (var i = 0; i < streamTypes.Length; i++)
            {
                if (historicalViewableDesc.Historical[i])
                {
                    var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i];
                    historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn);
                }
            }

            if (log.IsDebugEnabled)
            {
                log.Debug("Dependency graph: " + historicalDependencyGraph);
            }

            // Handle a join with a database or other historical data source for 2 streams
            if ((historicalViewableDesc.HasHistorical) && (streamTypes.Length == 2))
            {
                return(MakeComposerHistorical2Stream(outerJoinDescList, optionalFilterNode, streamTypes, historicalViewableDesc, queryPlanLogging, exprEvaluatorContext, statementContext, streamNames, allowIndexInit));
            }

            var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList);

            // Query graph for graph relationships between streams/historicals
            // For outer joins the query graph will just contain outer join relationships
            var hint       = ExcludePlanHint.GetHint(streamNames, statementContext);
            var queryGraph = new QueryGraph(streamTypes.Length, hint, false);

            if (outerJoinDescList.Length > 0)
            {
                OuterJoinAnalyzer.Analyze(outerJoinDescList, queryGraph);
                if (log.IsDebugEnabled)
                {
                    log.Debug(".makeComposer After outer join queryGraph=\n" + queryGraph);
                }
            }

            // Let the query graph reflect the where-clause
            if (optionalFilterNode != null)
            {
                // Analyze relationships between streams using the optional filter expression.
                // Relationships are properties in AND and EQUALS nodes of joins.
                FilterExprAnalyzer.Analyze(optionalFilterNode, queryGraph, isOuterJoins);
                if (log.IsDebugEnabled)
                {
                    log.Debug(".makeComposer After filter expression queryGraph=\n" + queryGraph);
                }

                // Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c)
                QueryGraph.FillEquivalentNav(streamTypes, queryGraph);
                if (log.IsDebugEnabled)
                {
                    log.Debug(".makeComposer After fill equiv. nav. queryGraph=\n" + queryGraph);
                }
            }

            // Historical index lists
            var historicalStreamIndexLists = new HistoricalStreamIndexList[streamTypes.Length];

            var queryPlan = QueryPlanBuilder.GetPlan(streamTypes, outerJoinDescList, queryGraph, streamNames,
                                                     historicalViewableDesc, historicalDependencyGraph, historicalStreamIndexLists,
                                                     streamJoinAnalysisResult, queryPlanLogging, statementContext.Annotations, exprEvaluatorContext);

            // remove unused indexes - consider all streams or all unidirectional
            var usedIndexes = new HashSet <TableLookupIndexReqKey>();
            var indexSpecs  = queryPlan.IndexSpecs;

            for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++)
            {
                var planNode = queryPlan.ExecNodeSpecs[streamNum];
                if (planNode != null)
                {
                    planNode.AddIndexes(usedIndexes);
                }
            }
            foreach (var indexSpec in indexSpecs)
            {
                if (indexSpec == null)
                {
                    continue;
                }
                var items      = indexSpec.Items;
                var indexNames = items.Keys.ToArray();
                foreach (var indexName in indexNames)
                {
                    if (!usedIndexes.Contains(indexName))
                    {
                        items.Remove(indexName);
                    }
                }
            }

            var hook = QueryPlanIndexHookUtil.GetHook(statementContext.Annotations);

            if (queryPlanLogging && (QueryPlanLog.IsInfoEnabled || hook != null))
            {
                QueryPlanLog.Info("Query plan: " + queryPlan.ToQueryPlan());
                if (hook != null)
                {
                    hook.Join(queryPlan);
                }
            }

            // register index-use references for tables
            if (!isOnDemandQuery)
            {
                foreach (var usedIndex in usedIndexes)
                {
                    if (usedIndex.TableName != null)
                    {
                        tableService.GetTableMetadata(usedIndex.TableName).AddIndexReference(usedIndex.Name, statementName);
                    }
                }
            }

            var joinRemoveStream = selectsRemoveStream || hasAggregations;

            return(new JoinSetComposerPrototypeImpl(
                       statementName,
                       statementId,
                       outerJoinDescList,
                       optionalFilterNode,
                       streamTypes,
                       streamNames,
                       streamJoinAnalysisResult,
                       statementContext.Annotations,
                       historicalViewableDesc,
                       exprEvaluatorContext,
                       indexSpecs,
                       queryPlan,
                       historicalStreamIndexLists,
                       joinRemoveStream,
                       isOuterJoins,
                       tableService,
                       statementContext.EventTableIndexService));
        }
コード例 #7
0
        /// <summary>
        /// Ctor.
        /// </summary>
        /// <param name="statementSpec">
        /// is a container for the definition of all statement constructs that may have been used in the
        /// statement, i.e. if defines the select clauses, insert into, outer joins etc.
        /// </param>
        /// <param name="services">is the service instances for dependency injection</param>
        /// <param name="statementContext">is statement-level information and statement services</param>
        /// <throws>ExprValidationException if the preparation failed</throws>
        public EPPreparedExecuteMethodQuery(
            StatementSpecCompiled statementSpec,
            EPServicesContext services,
            StatementContext statementContext)
        {
            var queryPlanLogging = services.ConfigSnapshot.EngineDefaults.Logging.IsEnableQueryPlan;

            if (queryPlanLogging)
            {
                QueryPlanLog.Info("Query plans for Fire-and-forget query '" + statementContext.Expression + "'");
            }

            _hasTableAccess = (statementSpec.TableNodes != null && statementSpec.TableNodes.Length > 0);
            foreach (var streamSpec in statementSpec.StreamSpecs)
            {
                _hasTableAccess |= streamSpec is TableQueryStreamSpec;
            }

            _statementSpec = statementSpec;
            _services      = services;

            EPPreparedExecuteMethodHelper.ValidateFAFQuery(statementSpec);

            var numStreams     = statementSpec.StreamSpecs.Length;
            var typesPerStream = new EventType[numStreams];
            var namesPerStream = new string[numStreams];

            _processors           = new FireAndForgetProcessor[numStreams];
            _agentInstanceContext = new AgentInstanceContext(statementContext, null, -1, null, null, statementContext.DefaultAgentInstanceScriptContext);

            // resolve types and processors
            for (var i = 0; i < numStreams; i++)
            {
                var streamSpec = statementSpec.StreamSpecs[i];
                _processors[i] = FireAndForgetProcessorFactory.ValidateResolveProcessor(streamSpec, services);

                string streamName = _processors[i].NamedWindowOrTableName;
                if (streamSpec.OptionalStreamName != null)
                {
                    streamName = streamSpec.OptionalStreamName;
                }
                namesPerStream[i] = streamName;
                typesPerStream[i] = _processors[i].EventTypeResultSetProcessor;
            }

            // compile filter to optimize access to named window
            var types           = new StreamTypeServiceImpl(typesPerStream, namesPerStream, new bool[numStreams], services.EngineURI, false);
            var excludePlanHint = ExcludePlanHint.GetHint(types.StreamNames, statementContext);

            _queryGraph = new QueryGraph(numStreams, excludePlanHint, false);

            if (statementSpec.FilterRootNode != null)
            {
                for (var i = 0; i < numStreams; i++)
                {
                    try
                    {
                        ExprEvaluatorContextStatement evaluatorContextStmt = new ExprEvaluatorContextStatement(statementContext, false);
                        ExprValidationContext         validationContext    = new ExprValidationContext(
                            statementContext.Container,
                            types,
                            statementContext.EngineImportService,
                            statementContext.StatementExtensionServicesContext, null,
                            statementContext.TimeProvider,
                            statementContext.VariableService,
                            statementContext.TableService, evaluatorContextStmt,
                            statementContext.EventAdapterService,
                            statementContext.StatementName,
                            statementContext.StatementId,
                            statementContext.Annotations,
                            statementContext.ContextDescriptor,
                            statementContext.ScriptingService,
                            false, false, true, false, null, true);
                        ExprNode validated = ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.FILTER, statementSpec.FilterRootNode, validationContext);
                        FilterExprAnalyzer.Analyze(validated, _queryGraph, false);
                    }
                    catch (Exception ex)
                    {
                        Log.Warn("Unexpected exception analyzing filter paths: " + ex.Message, ex);
                    }
                }
            }

            // obtain result set processor
            var isIStreamOnly = new bool[namesPerStream.Length];

            CompatExtensions.Fill(isIStreamOnly, true);
            StreamTypeService typeService = new StreamTypeServiceImpl(typesPerStream, namesPerStream, isIStreamOnly, services.EngineURI, true);

            EPStatementStartMethodHelperValidate.ValidateNodes(statementSpec, statementContext, typeService, null);

            var resultSetProcessorPrototype = ResultSetProcessorFactoryFactory.GetProcessorPrototype(statementSpec, statementContext, typeService, null, new bool[0], true, ContextPropertyRegistryImpl.EMPTY_REGISTRY, null, services.ConfigSnapshot, services.ResultSetProcessorHelperFactory, true, false);

            _resultSetProcessor = EPStatementStartMethodHelperAssignExpr.GetAssignResultSetProcessor(_agentInstanceContext, resultSetProcessorPrototype, false, null, true);

            if (statementSpec.SelectClauseSpec.IsDistinct)
            {
                if (_resultSetProcessor.ResultEventType is EventTypeSPI)
                {
                    _eventBeanReader = ((EventTypeSPI)_resultSetProcessor.ResultEventType).Reader;
                }
                if (_eventBeanReader == null)
                {
                    _eventBeanReader = new EventBeanReaderDefaultImpl(_resultSetProcessor.ResultEventType);
                }
            }

            // check context partition use
            if (statementSpec.OptionalContextName != null)
            {
                if (numStreams > 1)
                {
                    throw new ExprValidationException("Joins in runtime queries for context partitions are not supported");
                }
            }

            // plan joins or simple queries
            if (numStreams > 1)
            {
                var streamJoinAnalysisResult = new StreamJoinAnalysisResult(numStreams);
                CompatExtensions.Fill(streamJoinAnalysisResult.NamedWindow, true);
                for (var i = 0; i < numStreams; i++)
                {
                    var processorInstance = _processors[i].GetProcessorInstance(_agentInstanceContext);
                    if (_processors[i].IsVirtualDataWindow)
                    {
                        streamJoinAnalysisResult.ViewExternal[i] = agentInstanceContext => processorInstance.VirtualDataWindow;
                    }
                    var uniqueIndexes = _processors[i].GetUniqueIndexes(processorInstance);
                    streamJoinAnalysisResult.UniqueKeys[i] = uniqueIndexes;
                }

                var hasAggregations = !resultSetProcessorPrototype.AggregationServiceFactoryDesc.Expressions.IsEmpty();

                _joinSetComposerPrototype = JoinSetComposerPrototypeFactory.MakeComposerPrototype(null, -1,
                                                                                                  statementSpec.OuterJoinDescList, statementSpec.FilterRootNode, typesPerStream, namesPerStream,
                                                                                                  streamJoinAnalysisResult, queryPlanLogging, statementContext, new HistoricalViewableDesc(numStreams),
                                                                                                  _agentInstanceContext, false, hasAggregations, services.TableService, true,
                                                                                                  services.EventTableIndexService.AllowInitIndex(false));
            }
        }
コード例 #8
0
        public static JoinSetComposerPrototypeDesc MakeComposerPrototype(
            StatementSpecCompiled spec,
            StreamJoinAnalysisResultCompileTime joinAnalysisResult,
            StreamTypeService typeService,
            HistoricalViewableDesc historicalViewableDesc,
            bool isOnDemandQuery,
            bool hasAggregations,
            StatementRawInfo statementRawInfo,
            StatementCompileTimeServices compileTimeServices)
        {
            var streamTypes = typeService.EventTypes;
            var streamNames = typeService.StreamNames;
            var whereClause = spec.Raw.WhereClause;
            var queryPlanLogging = compileTimeServices.Configuration.Common.Logging.IsEnableQueryPlan;
            var additionalForgeables = new List<StmtClassForgeableFactory>();

            // Determine if there is a historical stream, and what dependencies exist
            var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false);
            for (var i = 0; i < streamTypes.Length; i++) {
                if (historicalViewableDesc.Historical[i]) {
                    var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i];
                    historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn);
                }
            }

            if (Log.IsDebugEnabled) {
                Log.Debug("Dependency graph: " + historicalDependencyGraph);
            }

            // Handle a join with a database or other historical data source for 2 streams
            var outerJoinDescs = OuterJoinDesc.ToArray(spec.Raw.OuterJoinDescList);
            if (historicalViewableDesc.IsHistorical && streamTypes.Length == 2) {
                var desc = MakeComposerHistorical2Stream(
                    outerJoinDescs,
                    whereClause,
                    streamTypes,
                    streamNames,
                    historicalViewableDesc,
                    queryPlanLogging,
                    statementRawInfo,
                    compileTimeServices);
                return new JoinSetComposerPrototypeDesc(desc.Forge, desc.AdditionalForgeables);
            }

            var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescs);

            // Query graph for graph relationships between streams/historicals
            // For outer joins the query graph will just contain outer join relationships
            var hint = ExcludePlanHint.GetHint(
                typeService.StreamNames,
                statementRawInfo,
                compileTimeServices);
            var queryGraph = new QueryGraphForge(streamTypes.Length, hint, false);
            if (outerJoinDescs.Length > 0) {
                OuterJoinAnalyzer.Analyze(outerJoinDescs, queryGraph);
                if (Log.IsDebugEnabled) {
                    Log.Debug(".makeComposer After outer join filterQueryGraph=\n" + queryGraph);
                }
            }

            // Let the query graph reflect the where-clause
            if (whereClause != null) {
                // Analyze relationships between streams using the optional filter expression.
                // Relationships are properties in AND and EQUALS nodes of joins.
                FilterExprAnalyzer.Analyze(whereClause, queryGraph, isOuterJoins);
                if (Log.IsDebugEnabled) {
                    Log.Debug(".makeComposer After filter expression filterQueryGraph=\n" + queryGraph);
                }

                // Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c)
                QueryGraphForge.FillEquivalentNav(streamTypes, queryGraph);
                if (Log.IsDebugEnabled) {
                    Log.Debug(".makeComposer After fill equiv. nav. filterQueryGraph=\n" + queryGraph);
                }
            }

            // Historical index lists
            var historicalStreamIndexLists =
                new HistoricalStreamIndexListForge[streamTypes.Length];

            var queryPlanDesc = QueryPlanBuilder.GetPlan(
                streamTypes,
                outerJoinDescs,
                queryGraph,
                typeService.StreamNames,
                historicalViewableDesc,
                historicalDependencyGraph,
                historicalStreamIndexLists,
                joinAnalysisResult,
                queryPlanLogging,
                statementRawInfo,
                compileTimeServices);
            QueryPlanForge queryPlan = queryPlanDesc.Forge;
            additionalForgeables.AddAll(queryPlanDesc.AdditionalForgeables);

            // remove unused indexes - consider all streams or all unidirectional
            var usedIndexes = new HashSet<TableLookupIndexReqKey>();
            var indexSpecs = queryPlan.IndexSpecs;
            for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) {
                var planNode = queryPlan.ExecNodeSpecs[streamNum];
                planNode?.AddIndexes(usedIndexes);
            }

            foreach (var indexSpec in indexSpecs) {
                if (indexSpec == null) {
                    continue;
                }

                var items = indexSpec.Items;
                var indexNames = items.Keys.ToArray();
                foreach (var indexName in indexNames) {
                    if (!usedIndexes.Contains(indexName)) {
                        items.Remove(indexName);
                    }
                }
            }

            // plan multikeys
            IList<StmtClassForgeableFactory> multikeyForgeables = PlanMultikeys(
                indexSpecs, statementRawInfo, compileTimeServices);
            additionalForgeables.AddAll(multikeyForgeables);

            QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook(
                spec.Annotations,
                compileTimeServices.ImportServiceCompileTime);
            if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) {
                QUERY_PLAN_LOG.Info("Query plan: " + queryPlan.ToQueryPlan());
                hook?.Join(queryPlan);
            }

            var selectsRemoveStream =
                spec.Raw.SelectStreamSelectorEnum.IsSelectsRStream() || spec.Raw.OutputLimitSpec != null;
            var joinRemoveStream = selectsRemoveStream || hasAggregations;

            ExprNode postJoinEvaluator;
            if (JoinSetComposerUtil.IsNonUnidirectionalNonSelf(
                isOuterJoins,
                joinAnalysisResult.IsUnidirectional,
                joinAnalysisResult.IsPureSelfJoin)) {
                postJoinEvaluator = GetFilterExpressionInclOnClause(
                    spec.Raw.WhereClause,
                    outerJoinDescs,
                    statementRawInfo,
                    compileTimeServices);
            }
            else {
                postJoinEvaluator = spec.Raw.WhereClause;
            }
            
            JoinSetComposerPrototypeGeneralForge forge = new JoinSetComposerPrototypeGeneralForge(
                typeService.EventTypes,
                postJoinEvaluator, 
                outerJoinDescs.Length > 0,
                queryPlan, 
                joinAnalysisResult, 
                typeService.StreamNames,
                joinRemoveStream, 
                historicalViewableDesc.IsHistorical);
            return new JoinSetComposerPrototypeDesc(forge, additionalForgeables);
        }
コード例 #9
0
        public FAFQueryMethodSelectDesc(
            StatementSpecCompiled statementSpec,
            Compilable compilable,
            StatementRawInfo statementRawInfo,
            StatementCompileTimeServices services)
        {
            Annotations = statementSpec.Annotations;
            ContextName = statementSpec.Raw.OptionalContextName;

            var queryPlanLogging = services.Configuration.Common.Logging.IsEnableQueryPlan;
            if (queryPlanLogging) {
                QUERY_PLAN_LOG.Info("Query plans for Fire-and-forget query '" + compilable.ToEPL() + "'");
            }

            HasTableAccess = statementSpec.TableAccessNodes != null && statementSpec.TableAccessNodes.Count > 0;
            foreach (var streamSpec in statementSpec.StreamSpecs) {
                HasTableAccess |= streamSpec is TableQueryStreamSpec;
            }

            HasTableAccess |= StatementLifecycleSvcUtil.IsSubqueryWithTable(
                statementSpec.SubselectNodes, services.TableCompileTimeResolver);
            IsDistinct = statementSpec.SelectClauseCompiled.IsDistinct;

            FAFQueryMethodHelper.ValidateFAFQuery(statementSpec);

            var numStreams = statementSpec.StreamSpecs.Length;
            var typesPerStream = new EventType[numStreams];
            var namesPerStream = new string[numStreams];
            var eventTypeNames = new string[numStreams];
            Processors = new FireAndForgetProcessorForge[numStreams];
            ConsumerFilters = new ExprNode[numStreams];

            // check context partition use
            if (statementSpec.Raw.OptionalContextName != null) {
                if (numStreams > 1) {
                    throw new ExprValidationException(
                        "Joins in runtime queries for context partitions are not supported");
                }
            }

            // resolve types and processors
            for (var i = 0; i < numStreams; i++) {
                var streamSpec = statementSpec.StreamSpecs[i];
                Processors[i] = FireAndForgetProcessorForgeFactory.ValidateResolveProcessor(streamSpec);
                if (numStreams > 1 && Processors[i].ContextName != null) {
                    throw new ExprValidationException(
                        "Joins against named windows that are under context are not supported");
                }

                var streamName = Processors[i].NamedWindowOrTableName;
                if (streamSpec.OptionalStreamName != null) {
                    streamName = streamSpec.OptionalStreamName;
                }

                namesPerStream[i] = streamName;
                typesPerStream[i] = Processors[i].EventTypeRspInputEvents;
                eventTypeNames[i] = typesPerStream[i].Name;

                IList<ExprNode> consumerFilterExprs;
                if (streamSpec is NamedWindowConsumerStreamSpec) {
                    var namedSpec = (NamedWindowConsumerStreamSpec) streamSpec;
                    consumerFilterExprs = namedSpec.FilterExpressions;
                }
                else {
                    var tableSpec = (TableQueryStreamSpec) streamSpec;
                    consumerFilterExprs = tableSpec.FilterExpressions;
                }

                ConsumerFilters[i] = ExprNodeUtilityMake.ConnectExpressionsByLogicalAndWhenNeeded(consumerFilterExprs);
            }

            // compile filter to optimize access to named window
            var optionalStreamsIfAny = OuterJoinAnalyzer.OptionalStreamsIfAny(statementSpec.Raw.OuterJoinDescList);
            var types = new StreamTypeServiceImpl(
                typesPerStream,
                namesPerStream,
                new bool[numStreams],
                false,
                optionalStreamsIfAny);
            var excludePlanHint = ExcludePlanHint.GetHint(types.StreamNames, statementRawInfo, services);
            QueryGraph = new QueryGraphForge(numStreams, excludePlanHint, false);
            if (statementSpec.Raw.WhereClause != null) {
                for (var i = 0; i < numStreams; i++) {
                    try {
                        var validationContext = new ExprValidationContextBuilder(types, statementRawInfo, services)
                            .WithAllowBindingConsumption(true)
                            .WithIsFilterExpression(true)
                            .Build();
                        var validated = ExprNodeUtilityValidate.GetValidatedSubtree(
                            ExprNodeOrigin.FILTER,
                            statementSpec.Raw.WhereClause,
                            validationContext);
                        FilterExprAnalyzer.Analyze(validated, QueryGraph, false);
                    }
                    catch (Exception ex) {
                        Log.Warn("Unexpected exception analyzing filter paths: " + ex.Message, ex);
                    }
                }
            }
            
            // handle subselects
            // first we create streams for subselects, if there are any
            var @base = new StatementBaseInfo(compilable, statementSpec, null, statementRawInfo, null);
            var subqueryNamedWindowConsumers = new List<NamedWindowConsumerStreamSpec>();
            SubSelectActivationDesc subSelectActivationDesc = SubSelectHelperActivations.CreateSubSelectActivation(
                EmptyList<FilterSpecCompiled>.Instance, subqueryNamedWindowConsumers, @base, services);
            IDictionary<ExprSubselectNode, SubSelectActivationPlan> subselectActivation = subSelectActivationDesc.Subselects;
            AdditionalForgeables.AddAll(subSelectActivationDesc.AdditionalForgeables);

            SubSelectHelperForgePlan subSelectForgePlan = SubSelectHelperForgePlanner.PlanSubSelect(
                @base, subselectActivation, namesPerStream, typesPerStream, eventTypeNames, services);
            SubselectForges = subSelectForgePlan.Subselects;
            AdditionalForgeables.AddAll(subSelectForgePlan.AdditionalForgeables);

            // obtain result set processor
            var isIStreamOnly = new bool[namesPerStream.Length];
            isIStreamOnly.Fill(true);
            StreamTypeService typeService = new StreamTypeServiceImpl(
                typesPerStream,
                namesPerStream,
                isIStreamOnly,
                true,
                optionalStreamsIfAny);
            WhereClause = EPStatementStartMethodHelperValidate.ValidateNodes(
                statementSpec.Raw,
                typeService,
                null,
                statementRawInfo,
                services);

            var resultSetSpec = new ResultSetSpec(statementSpec);
            ResultSetProcessor = ResultSetProcessorFactoryFactory.GetProcessorPrototype(
                resultSetSpec,
                typeService,
                null,
                new bool[0],
                true,
                null,
                true,
                false,
                statementRawInfo,
                services);
            AdditionalForgeables.AddAll(ResultSetProcessor.AdditionalForgeables);

            // plan table access
            TableAccessForges = ExprTableEvalHelperPlan.PlanTableAccess(statementSpec.Raw.TableExpressions);

            // plan joins or simple queries
            if (numStreams > 1) {
                var streamJoinAnalysisResult = new StreamJoinAnalysisResultCompileTime(numStreams);
                CompatExtensions.Fill(streamJoinAnalysisResult.NamedWindowsPerStream, (NamedWindowMetaData) null);
                for (var i = 0; i < numStreams; i++) {
                    var uniqueIndexes = Processors[i].UniqueIndexes;
                    streamJoinAnalysisResult.UniqueKeys[i] = uniqueIndexes;
                }

                var hasAggregations = ResultSetProcessor.ResultSetProcessorType.IsAggregated();
                var desc = JoinSetComposerPrototypeForgeFactory.MakeComposerPrototype(
                    statementSpec,
                    streamJoinAnalysisResult,
                    types,
                    new HistoricalViewableDesc(numStreams),
                    true,
                    hasAggregations,
                    statementRawInfo,
                    services);
                AdditionalForgeables.AddAll(desc.AdditionalForgeables);
                Joins = desc.Forge;
            }
            else {
                Joins = null;
            }
            
            var multiKeyPlan = MultiKeyPlanner.PlanMultiKeyDistinct(
                IsDistinct, ResultSetProcessor.ResultEventType, statementRawInfo, SerdeCompileTimeResolverNonHA.INSTANCE);
            AdditionalForgeables.AddAll(multiKeyPlan.MultiKeyForgeables);
            DistinctMultiKey = multiKeyPlan.ClassRef;
        }