private static IList<ExprNode> DecomposeCheckAggregation(IList<ExprNode> validatedNodes) { // Break a top-level AND into constituent expression nodes IList<ExprNode> constituents = new List<ExprNode>(); foreach (ExprNode validated in validatedNodes) { if (validated is ExprAndNode) { RecursiveAndConstituents(constituents, validated); } else { constituents.Add(validated); } // Ensure there is no aggregation nodes var aggregateExprNodes = new List<ExprAggregateNode>(); ExprAggregateNodeUtil.GetAggregatesBottomUp(validated, aggregateExprNodes); if (!aggregateExprNodes.IsEmpty()) { throw new ExprValidationException("Aggregation functions not allowed within filters"); } } return constituents; }
public static void ValidateNoSpecialsGroupByExpressions(ExprNode[] groupByNodes) { ExprNodeSubselectDeclaredDotVisitor visitorSubselects = new ExprNodeSubselectDeclaredDotVisitor(); ExprNodeGroupingVisitorWParent visitorGrouping = new ExprNodeGroupingVisitorWParent(); IList<ExprAggregateNode> aggNodesInGroupBy = new List<ExprAggregateNode>(1); foreach (ExprNode groupByNode in groupByNodes) { // no subselects groupByNode.Accept(visitorSubselects); if (visitorSubselects.Subselects.Count > 0) { throw new ExprValidationException("Subselects not allowed within group-by"); } // no special grouping-clauses groupByNode.Accept(visitorGrouping); if (!visitorGrouping.GroupingIdNodes.IsEmpty()) { throw ExprGroupingIdNode.MakeException("grouping_id"); } if (!visitorGrouping.GroupingNodes.IsEmpty()) { throw ExprGroupingIdNode.MakeException("grouping"); } // no aggregations allowed ExprAggregateNodeUtil.GetAggregatesBottomUp(groupByNode, aggNodesInGroupBy); if (!aggNodesInGroupBy.IsEmpty()) { throw new ExprValidationException("Group-by expressions cannot contain aggregate functions"); } } }
/// <summary> /// Returns processor for order-by clauses. /// </summary> /// <param name="selectionList">is a list of select expressions</param> /// <param name="groupByNodes">is a list of group-by expressions</param> /// <param name="orderByList">is a list of order-by expressions</param> /// <param name="rowLimitSpec">specification for row limit, or null if no row limit is defined</param> /// <param name="variableService">for retrieving variable state for use with row limiting</param> /// <param name="isSortUsingCollator">for string value sorting using compare or Collator</param> /// <param name="optionalContextName">Name of the optional context.</param> /// <returns>ordering processor instance</returns> /// <throws><seealso cref="ExprValidationException" /> when validation of expressions fails</throws> public static OrderByProcessorFactory GetProcessor( IList <SelectClauseExprCompiledSpec> selectionList, ExprNode[] groupByNodes, IList <OrderByItem> orderByList, RowLimitSpec rowLimitSpec, VariableService variableService, bool isSortUsingCollator, String optionalContextName) { // Get the order by expression nodes IList <ExprNode> orderByNodes = orderByList.Select(element => element.ExprNode).ToList(); // No order-by clause if (orderByList.IsEmpty()) { Log.Debug(".getProcessor Using no _orderByProcessor"); if (rowLimitSpec != null) { var rowLimitProcessorFactory = new RowLimitProcessorFactory(rowLimitSpec, variableService, optionalContextName); return(new OrderByProcessorRowLimitOnlyFactory(rowLimitProcessorFactory)); } return(null); } // Determine aggregate functions used in select, if any IList <ExprAggregateNode> selectAggNodes = new List <ExprAggregateNode>(); foreach (var element in selectionList) { ExprAggregateNodeUtil.GetAggregatesBottomUp(element.SelectExpression, selectAggNodes); } // Get all the aggregate functions occuring in the order-by clause IList <ExprAggregateNode> orderAggNodes = new List <ExprAggregateNode>(); foreach (var orderByNode in orderByNodes) { ExprAggregateNodeUtil.GetAggregatesBottomUp(orderByNode, orderAggNodes); } ValidateOrderByAggregates(selectAggNodes, orderAggNodes); // Tell the order-by processor whether to compute group-by // keys if they are not present var needsGroupByKeys = !selectionList.IsEmpty() && !orderAggNodes.IsEmpty(); Log.Debug(".getProcessor Using OrderByProcessorImpl"); var orderByProcessorFactory = new OrderByProcessorFactoryImpl(orderByList, groupByNodes, needsGroupByKeys, isSortUsingCollator); if (rowLimitSpec == null) { return(orderByProcessorFactory); } else { var rowLimitProcessorFactory = new RowLimitProcessorFactory(rowLimitSpec, variableService, optionalContextName); return(new OrderByProcessorOrderedLimitFactory(orderByProcessorFactory, rowLimitProcessorFactory)); } }
// Min/Max nodes can be either an aggregate or a per-row function depending on the number or arguments private static void HandleMinMax( string ident, EsperEPL2GrammarParser.LibFunctionArgsContext ctxArgs, IDictionary <ITree, ExprNode> astExprNodeMap) { // Determine min or max var childNodeText = ident; MinMaxTypeEnum minMaxTypeEnum; var filtered = childNodeText.StartsWith("f"); if (childNodeText.ToLowerInvariant().Equals("min") || childNodeText.ToLowerInvariant().Equals("fmin")) { minMaxTypeEnum = MinMaxTypeEnum.MIN; } else if (childNodeText.ToLowerInvariant().Equals("max") || childNodeText.ToLowerInvariant().Equals("fmax")) { minMaxTypeEnum = MinMaxTypeEnum.MAX; } else { throw ASTWalkException.From("Uncountered unrecognized min or max node '" + ident + "'"); } var args = Collections.GetEmptyList <ExprNode>(); if (ctxArgs != null && ctxArgs.libFunctionArgItem() != null) { args = ASTExprHelper.ExprCollectSubNodes(ctxArgs, 0, astExprNodeMap); } var numArgsPositional = ExprAggregateNodeUtil.CountPositionalArgs(args); var isDistinct = ctxArgs != null && ctxArgs.DISTINCT() != null; if (numArgsPositional > 1 && isDistinct && !filtered) { throw ASTWalkException.From( "The distinct keyword is not valid in per-row min and max " + "functions with multiple sub-expressions"); } ExprNode minMaxNode; if (!isDistinct && numArgsPositional > 1 && !filtered) { // use the row function minMaxNode = new ExprMinMaxRowNode(minMaxTypeEnum); } else { // use the aggregation function minMaxNode = new ExprMinMaxAggrNode(isDistinct, minMaxTypeEnum, filtered, false); } minMaxNode.AddChildNodes(args); astExprNodeMap.Put(ctxArgs, minMaxNode); }
internal static void ValidateNoAggregations(ExprNode exprNode, String errorMsg) { // Make sure there is no aggregation in the where clause var aggregateNodes = new List <ExprAggregateNode>(); ExprAggregateNodeUtil.GetAggregatesBottomUp(exprNode, aggregateNodes); if (!aggregateNodes.IsEmpty()) { throw new ExprValidationException(errorMsg); } }
public void TestGetAggregatesBottomUp() { /* * top (ag) * c1 c2 * c1_1 (ag) c1_2 (ag) c2_1 c2_2 * c2_1_1 (ag) c2_1_2 (ag) * */ ExprNode top = new SupportAggregateExprNode(null); ExprNode c1 = new SupportExprNode(null); ExprNode c2 = new SupportExprNode(null); top.AddChildNode(c1); top.AddChildNode(c2); ExprNode c1_1 = new SupportAggregateExprNode(null); ExprNode c1_2 = new SupportAggregateExprNode(null); c1.AddChildNode(c1_1); c1.AddChildNode(c1_2); c1_1.AddChildNode(new SupportExprNode(null)); c1_2.AddChildNode(new SupportExprNode(null)); ExprNode c2_1 = new SupportExprNode(null); ExprNode c2_2 = new SupportExprNode(null); c2.AddChildNode(c2_1); c2.AddChildNode(c2_2); c2_2.AddChildNode(new SupportExprNode(null)); ExprNode c2_1_1 = new SupportAggregateExprNode(null); ExprNode c2_1_2 = new SupportAggregateExprNode(null); c2_1.AddChildNode(c2_1_1); c2_1.AddChildNode(c2_1_2); List <ExprAggregateNode> aggregates = new List <ExprAggregateNode>(); ExprAggregateNodeUtil.GetAggregatesBottomUp(top, aggregates); Assert.AreEqual(5, aggregates.Count); Assert.AreSame(c2_1_1, aggregates[0]); Assert.AreSame(c2_1_2, aggregates[1]); Assert.AreSame(c1_1, aggregates[2]); Assert.AreSame(c1_2, aggregates[3]); Assert.AreSame(top, aggregates[4]); // Test no aggregates aggregates.Clear(); ExprAggregateNodeUtil.GetAggregatesBottomUp(new SupportExprNode(null), aggregates); Assert.IsTrue(aggregates.IsEmpty()); }
private static ExprNode HandleMinMaxNode( string chainFirstLowerCase, Chainable spec) { MinMaxTypeEnum minMaxTypeEnum; var filtered = chainFirstLowerCase.StartsWith("f"); if (chainFirstLowerCase.Equals("min") || chainFirstLowerCase.Equals("fmin")) { minMaxTypeEnum = MinMaxTypeEnum.MIN; } else if (chainFirstLowerCase.Equals("max") || chainFirstLowerCase.Equals("fmax")) { minMaxTypeEnum = MinMaxTypeEnum.MAX; } else { throw new ValidationException("Uncountered unrecognized min or max node '" + spec.GetRootNameOrEmptyString() + "'"); } var args = spec.GetParametersOrEmpty(); var distinct = spec.IsDistinct; var numArgsPositional = ExprAggregateNodeUtil.CountPositionalArgs(args); if (numArgsPositional > 1 && spec.IsDistinct && !filtered) { throw new ValidationException( "The distinct keyword is not valid in per-row min and max " + "functions with multiple sub-expressions"); } ExprNode minMaxNode; if (!distinct && numArgsPositional > 1 && !filtered) { // use the row function minMaxNode = new ExprMinMaxRowNode(minMaxTypeEnum); } else { // use the aggregation function minMaxNode = new ExprMinMaxAggrNode(distinct, minMaxTypeEnum, filtered, false); } minMaxNode.AddChildNodes(args); return(minMaxNode); }
/// <summary> /// Check if the expression is minimal: does not have a subselect, aggregation and does not need view resources /// </summary> /// <param name="expression">to inspect</param> /// <returns>null if minimal, otherwise name of offending sub-expression</returns> public static string IsMinimalExpression(ExprNode expression) { ExprNodeSubselectDeclaredDotVisitor subselectVisitor = new ExprNodeSubselectDeclaredDotVisitor(); expression.Accept(subselectVisitor); if (subselectVisitor.Subselects.Count > 0) { return "a subselect"; } ExprNodeViewResourceVisitor viewResourceVisitor = new ExprNodeViewResourceVisitor(); expression.Accept(viewResourceVisitor); if (viewResourceVisitor.ExprNodes.Count > 0) { return "a function that requires view resources (prior, prev)"; } IList<ExprAggregateNode> aggregateNodes = new List<ExprAggregateNode>(); ExprAggregateNodeUtil.GetAggregatesBottomUp(expression, aggregateNodes); if (!aggregateNodes.IsEmpty()) { return "an aggregation function"; } return null; }
/// <summary> /// Validate filter and join expression nodes. /// </summary> /// <param name="statementSpec">the compiled statement</param> /// <param name="statementContext">the statement services</param> /// <param name="typeService">the event types for streams</param> /// <param name="viewResourceDelegate">the delegate to verify expressions that use view resources</param> internal static void ValidateNodes( StatementSpecCompiled statementSpec, StatementContext statementContext, StreamTypeService typeService, ViewResourceDelegateUnverified viewResourceDelegate) { var methodResolutionService = statementContext.MethodResolutionService; var evaluatorContextStmt = new ExprEvaluatorContextStatement(statementContext, false); var intoTableName = statementSpec.IntoTableSpec == null ? null : statementSpec.IntoTableSpec.Name; if (statementSpec.FilterRootNode != null) { var optionalFilterNode = statementSpec.FilterRootNode; // Validate where clause, initializing nodes to the stream ids used try { var validationContext = new ExprValidationContext( typeService, methodResolutionService, viewResourceDelegate, statementContext.SchedulingService, statementContext.VariableService, statementContext.TableService, evaluatorContextStmt, statementContext.EventAdapterService, statementContext.StatementName, statementContext.StatementId, statementContext.Annotations, statementContext.ContextDescriptor, statementContext.ScriptingService, false, false, true, false, intoTableName, false); optionalFilterNode = ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.FILTER, optionalFilterNode, validationContext); if (optionalFilterNode.ExprEvaluator.ReturnType != typeof(bool) && optionalFilterNode.ExprEvaluator.ReturnType != typeof(bool?)) { throw new ExprValidationException("The where-clause filter expression must return a boolean value"); } statementSpec.FilterExprRootNode = optionalFilterNode; // Make sure there is no aggregation in the where clause var aggregateNodes = new List <ExprAggregateNode>(); ExprAggregateNodeUtil.GetAggregatesBottomUp(optionalFilterNode, aggregateNodes); if (!aggregateNodes.IsEmpty()) { throw new ExprValidationException("An aggregate function may not appear in a WHERE clause (use the HAVING clause)"); } } catch (ExprValidationException ex) { Log.Debug(".validateNodes Validation exception for filter=" + ExprNodeUtility.ToExpressionStringMinPrecedenceSafe(optionalFilterNode), ex); throw new EPStatementException("Error validating expression: " + ex.Message, ex, statementContext.Expression); } } if ((statementSpec.OutputLimitSpec != null) && ((statementSpec.OutputLimitSpec.WhenExpressionNode != null) || (statementSpec.OutputLimitSpec.AndAfterTerminateExpr != null))) { // Validate where clause, initializing nodes to the stream ids used try { var outputLimitType = OutputConditionExpressionFactory.GetBuiltInEventType(statementContext.EventAdapterService); StreamTypeService typeServiceOutputWhen = new StreamTypeServiceImpl(new EventType[] { outputLimitType }, new String[] { null }, new bool[] { true }, statementContext.EngineURI, false); var validationContext = new ExprValidationContext( typeServiceOutputWhen, methodResolutionService, null, statementContext.SchedulingService, statementContext.VariableService, statementContext.TableService, evaluatorContextStmt, statementContext.EventAdapterService, statementContext.StatementName, statementContext.StatementId, statementContext.Annotations, statementContext.ContextDescriptor, statementContext.ScriptingService, false, false, false, false, intoTableName, false); var outputLimitWhenNode = statementSpec.OutputLimitSpec.WhenExpressionNode; if (outputLimitWhenNode != null) { outputLimitWhenNode = ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.OUTPUTLIMIT, outputLimitWhenNode, validationContext); statementSpec.OutputLimitSpec.WhenExpressionNode = outputLimitWhenNode; if (TypeHelper.GetBoxedType(outputLimitWhenNode.ExprEvaluator.ReturnType) != typeof(bool?)) { throw new ExprValidationException("The when-trigger expression in the OUTPUT WHEN clause must return a boolean-type value"); } EPStatementStartMethodHelperValidate.ValidateNoAggregations(outputLimitWhenNode, "An aggregate function may not appear in a OUTPUT LIMIT clause"); } // validate and-terminate expression if provided if (statementSpec.OutputLimitSpec.AndAfterTerminateExpr != null) { if (statementSpec.OutputLimitSpec.RateType != OutputLimitRateType.WHEN_EXPRESSION && statementSpec.OutputLimitSpec.RateType != OutputLimitRateType.TERM) { throw new ExprValidationException("A terminated-and expression must be used with the OUTPUT WHEN clause"); } var validated = ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.OUTPUTLIMIT, statementSpec.OutputLimitSpec.AndAfterTerminateExpr, validationContext); statementSpec.OutputLimitSpec.AndAfterTerminateExpr = validated; if (validated.ExprEvaluator.ReturnType.GetBoxedType() != typeof(bool?)) { throw new ExprValidationException("The terminated-and expression must return a boolean-type value"); } EPStatementStartMethodHelperValidate.ValidateNoAggregations(validated, "An aggregate function may not appear in a terminated-and clause"); } // validate then-expression ValidateThenSetAssignments(statementSpec.OutputLimitSpec.ThenExpressions, validationContext); // validate after-terminated then-expression ValidateThenSetAssignments(statementSpec.OutputLimitSpec.AndAfterTerminateThenExpressions, validationContext); } catch (ExprValidationException ex) { throw new EPStatementException("Error validating expression: " + ex.Message, statementContext.Expression); } } for (var outerJoinCount = 0; outerJoinCount < statementSpec.OuterJoinDescList.Length; outerJoinCount++) { var outerJoinDesc = statementSpec.OuterJoinDescList[outerJoinCount]; // validate on-expression nodes, if provided if (outerJoinDesc.OptLeftNode != null) { var streamIdPair = ValidateOuterJoinPropertyPair(statementContext, outerJoinDesc.OptLeftNode, outerJoinDesc.OptRightNode, outerJoinCount, typeService, viewResourceDelegate); if (outerJoinDesc.AdditionalLeftNodes != null) { ISet <int?> streamSet = new HashSet <int?>(); streamSet.Add(streamIdPair.First); streamSet.Add(streamIdPair.Second); for (var i = 0; i < outerJoinDesc.AdditionalLeftNodes.Length; i++) { var streamIdPairAdd = ValidateOuterJoinPropertyPair(statementContext, outerJoinDesc.AdditionalLeftNodes[i], outerJoinDesc.AdditionalRightNodes[i], outerJoinCount, typeService, viewResourceDelegate); // make sure all additional properties point to the same two streams if ((!streamSet.Contains(streamIdPairAdd.First) || (!streamSet.Contains(streamIdPairAdd.Second)))) { const string message = "Outer join ON-clause columns must refer to properties of the same joined streams" + " when using multiple columns in the on-clause"; throw new EPStatementException("Error validating expression: " + message, statementContext.Expression); } } } } } }
/// <summary> /// Ctor. /// </summary> /// <param name="viewChain">views</param> /// <param name="matchRecognizeSpec">specification</param> /// <param name="agentInstanceContext">The agent instance context.</param> /// <param name="isUnbound">true for unbound stream</param> /// <param name="annotations">annotations</param> /// <exception cref="ExprValidationException"> /// Variable ' + defineItem.Identifier + ' has already been defined /// or /// An aggregate function may not appear in a DEFINE clause /// or /// Failed to validate condition expression for variable ' + defineItem.Identifier + ': + ex.Message /// or /// Aggregation functions in the measure-clause must only refer to properties of exactly one group variable returning multiple events /// or /// Aggregation functions in the measure-clause must refer to one or more properties of exactly one group variable returning multiple events /// or /// The measures clause requires that each expression utilizes the AS keyword to assign a column name /// </exception> /// <throws>ExprValidationException if validation fails</throws> public EventRowRegexNFAViewFactory( ViewFactoryChain viewChain, MatchRecognizeSpec matchRecognizeSpec, AgentInstanceContext agentInstanceContext, bool isUnbound, Attribute[] annotations, ConfigurationEngineDefaults.MatchRecognizeConfig matchRecognizeConfig) { var parentViewType = viewChain.EventType; _matchRecognizeSpec = matchRecognizeSpec; _isUnbound = isUnbound; _isIterateOnly = HintEnum.ITERATE_ONLY.GetHint(annotations) != null; _matchRecognizeConfig = matchRecognizeConfig; var statementContext = agentInstanceContext.StatementContext; // Expand repeats and permutations _expandedPatternNode = RegexPatternExpandUtil.Expand(matchRecognizeSpec.Pattern); // Determine single-row and multiple-row variables _variablesSingle = new LinkedHashSet <string>(); ISet <string> variablesMultiple = new LinkedHashSet <string>(); EventRowRegexHelper.RecursiveInspectVariables(_expandedPatternNode, false, _variablesSingle, variablesMultiple); // each variable gets associated with a stream number (multiple-row variables as well to hold the current event for the expression). var streamNum = 0; _variableStreams = new LinkedHashMap <string, Pair <int, bool> >(); foreach (var variableSingle in _variablesSingle) { _variableStreams.Put(variableSingle, new Pair <int, bool>(streamNum, false)); streamNum++; } foreach (var variableMultiple in variablesMultiple) { _variableStreams.Put(variableMultiple, new Pair <int, bool>(streamNum, true)); streamNum++; } // mapping of stream to variable _streamVariables = new SortedDictionary <int, string>(); foreach (var entry in _variableStreams) { _streamVariables.Put(entry.Value.First, entry.Key); } // determine visibility rules var visibility = EventRowRegexHelper.DetermineVisibility(_expandedPatternNode); // assemble all single-row variables for expression validation var allStreamNames = new string[_variableStreams.Count]; var allTypes = new EventType[_variableStreams.Count]; streamNum = 0; foreach (var variableSingle in _variablesSingle) { allStreamNames[streamNum] = variableSingle; allTypes[streamNum] = parentViewType; streamNum++; } foreach (var variableMultiple in variablesMultiple) { allStreamNames[streamNum] = variableMultiple; allTypes[streamNum] = parentViewType; streamNum++; } // determine type service for use with DEFINE // validate each DEFINE clause expression ISet <string> definedVariables = new HashSet <string>(); IList <ExprAggregateNode> aggregateNodes = new List <ExprAggregateNode>(); var exprEvaluatorContext = new ExprEvaluatorContextStatement(statementContext, false); _isExprRequiresMultimatchState = new bool[_variableStreams.Count]; for (var defineIndex = 0; defineIndex < matchRecognizeSpec.Defines.Count; defineIndex++) { var defineItem = matchRecognizeSpec.Defines[defineIndex]; if (definedVariables.Contains(defineItem.Identifier)) { throw new ExprValidationException("Variable '" + defineItem.Identifier + "' has already been defined"); } definedVariables.Add(defineItem.Identifier); // stream-type visibilities handled here var typeServiceDefines = EventRowRegexNFAViewFactoryHelper.BuildDefineStreamTypeServiceDefine(statementContext, _variableStreams, defineItem, visibility, parentViewType); var exprNodeResult = HandlePreviousFunctions(defineItem.Expression); var validationContext = new ExprValidationContext( typeServiceDefines, statementContext.EngineImportService, statementContext.StatementExtensionServicesContext, null, statementContext.SchedulingService, statementContext.VariableService, statementContext.TableService, exprEvaluatorContext, statementContext.EventAdapterService, statementContext.StatementName, statementContext.StatementId, statementContext.Annotations, statementContext.ContextDescriptor, statementContext.ScriptingService, true, false, true, false, null, false); ExprNode validated; try { // validate validated = ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.MATCHRECOGDEFINE, exprNodeResult, validationContext); // check aggregates defineItem.Expression = validated; ExprAggregateNodeUtil.GetAggregatesBottomUp(validated, aggregateNodes); if (!aggregateNodes.IsEmpty()) { throw new ExprValidationException("An aggregate function may not appear in a DEFINE clause"); } } catch (ExprValidationException ex) { throw new ExprValidationException("Failed to validate condition expression for variable '" + defineItem.Identifier + "': " + ex.Message, ex); } // determine access to event properties from multi-matches var visitor = new ExprNodeStreamRequiredVisitor(); validated.Accept(visitor); var streamsRequired = visitor.StreamsRequired; foreach (var streamRequired in streamsRequired) { if (streamRequired >= _variableStreams.Count) { var streamNumIdent = _variableStreams.Get(defineItem.Identifier).First; _isExprRequiresMultimatchState[streamNumIdent] = true; break; } } } _isDefineAsksMultimatches = CollectionUtil.IsAnySet(_isExprRequiresMultimatchState); _defineMultimatchEventBean = _isDefineAsksMultimatches ? EventRowRegexNFAViewFactoryHelper.GetDefineMultimatchBean(statementContext, _variableStreams, parentViewType) : null; // assign "prev" node indexes // Since an expression such as "prior(2, price), prior(8, price)" translates into {2, 8} the relative index is {0, 1}. // Map the expression-supplied index to a relative index var countPrev = 0; foreach (var entry in _callbacksPerIndex) { foreach (var callback in entry.Value) { callback.AssignedIndex = countPrev; } countPrev++; } // determine type service for use with MEASURE IDictionary <string, object> measureTypeDef = new LinkedHashMap <string, object>(); foreach (var variableSingle in _variablesSingle) { measureTypeDef.Put(variableSingle, parentViewType); } foreach (var variableMultiple in variablesMultiple) { measureTypeDef.Put(variableMultiple, new EventType[] { parentViewType }); } var outputEventTypeName = statementContext.StatementId + "_rowrecog"; _compositeEventType = (ObjectArrayEventType)statementContext.EventAdapterService.CreateAnonymousObjectArrayType(outputEventTypeName, measureTypeDef); StreamTypeService typeServiceMeasure = new StreamTypeServiceImpl(_compositeEventType, "MATCH_RECOGNIZE", true, statementContext.EngineURI); // find MEASURE clause aggregations var measureReferencesMultivar = false; IList <ExprAggregateNode> measureAggregateExprNodes = new List <ExprAggregateNode>(); foreach (var measureItem in matchRecognizeSpec.Measures) { ExprAggregateNodeUtil.GetAggregatesBottomUp(measureItem.Expr, measureAggregateExprNodes); } if (!measureAggregateExprNodes.IsEmpty()) { var isIStreamOnly = new bool[allStreamNames.Length]; CompatExtensions.Fill(isIStreamOnly, true); var typeServiceAggregateMeasure = new StreamTypeServiceImpl(allTypes, allStreamNames, isIStreamOnly, statementContext.EngineURI, false); var measureExprAggNodesPerStream = new Dictionary <int, IList <ExprAggregateNode> >(); foreach (var aggregateNode in measureAggregateExprNodes) { // validate absence of group-by aggregateNode.ValidatePositionals(); if (aggregateNode.OptionalLocalGroupBy != null) { throw new ExprValidationException("Match-recognize does not allow aggregation functions to specify a group-by"); } // validate node and params var count = 0; var visitor = new ExprNodeIdentifierVisitor(true); var validationContext = new ExprValidationContext( typeServiceAggregateMeasure, statementContext.EngineImportService, statementContext.StatementExtensionServicesContext, null, statementContext.SchedulingService, statementContext.VariableService, statementContext.TableService, exprEvaluatorContext, statementContext.EventAdapterService, statementContext.StatementName, statementContext.StatementId, statementContext.Annotations, statementContext.ContextDescriptor, statementContext.ScriptingService, false, false, true, false, null, false); for (int ii = 0; ii < aggregateNode.ChildNodes.Count; ii++) { var child = aggregateNode.ChildNodes[ii]; var validated = ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.MATCHRECOGMEASURE, child, validationContext); validated.Accept(visitor); aggregateNode.SetChildNode(count++, new ExprNodeValidated(validated)); } validationContext = new ExprValidationContext( typeServiceMeasure, statementContext.EngineImportService, statementContext.StatementExtensionServicesContext, null, statementContext.SchedulingService, statementContext.VariableService, statementContext.TableService, exprEvaluatorContext, statementContext.EventAdapterService, statementContext.StatementName, statementContext.StatementId, statementContext.Annotations, statementContext.ContextDescriptor, statementContext.ScriptingService, false, false, true, false, null, false); aggregateNode.Validate(validationContext); // verify properties used within the aggregation var aggregatedStreams = new HashSet <int>(); foreach (var pair in visitor.ExprProperties) { aggregatedStreams.Add(pair.First); } int?multipleVarStream = null; foreach (int streamNumAggregated in aggregatedStreams) { var variable = _streamVariables.Get(streamNumAggregated); if (variablesMultiple.Contains(variable)) { measureReferencesMultivar = true; if (multipleVarStream == null) { multipleVarStream = streamNumAggregated; continue; } throw new ExprValidationException("Aggregation functions in the measure-clause must only refer to properties of exactly one group variable returning multiple events"); } } if (multipleVarStream == null) { throw new ExprValidationException("Aggregation functions in the measure-clause must refer to one or more properties of exactly one group variable returning multiple events"); } var aggNodesForStream = measureExprAggNodesPerStream.Get(multipleVarStream.Value); if (aggNodesForStream == null) { aggNodesForStream = new List <ExprAggregateNode>(); measureExprAggNodesPerStream.Put(multipleVarStream.Value, aggNodesForStream); } aggNodesForStream.Add(aggregateNode); } var factoryDesc = AggregationServiceFactoryFactory.GetServiceMatchRecognize(_streamVariables.Count, measureExprAggNodesPerStream, typeServiceAggregateMeasure.EventTypes); _aggregationService = factoryDesc.AggregationServiceFactory.MakeService(agentInstanceContext); _aggregationExpressions = factoryDesc.Expressions; } else { _aggregationService = null; _aggregationExpressions = Collections.GetEmptyList <AggregationServiceAggExpressionDesc>(); } // validate each MEASURE clause expression IDictionary <string, object> rowTypeDef = new LinkedHashMap <string, object>(); var streamRefVisitor = new ExprNodeStreamUseCollectVisitor(); foreach (var measureItem in matchRecognizeSpec.Measures) { if (measureItem.Name == null) { throw new ExprValidationException("The measures clause requires that each expression utilizes the AS keyword to assign a column name"); } var validated = ValidateMeasureClause(measureItem.Expr, typeServiceMeasure, variablesMultiple, _variablesSingle, statementContext); measureItem.Expr = validated; rowTypeDef.Put(measureItem.Name, validated.ExprEvaluator.ReturnType); validated.Accept(streamRefVisitor); } // Determine if any of the multi-var streams are referenced in the measures (non-aggregated only) foreach (var @ref in streamRefVisitor.Referenced) { var rootPropName = @ref.RootPropertyNameIfAny; if (rootPropName != null) { if (variablesMultiple.Contains(rootPropName)) { measureReferencesMultivar = true; break; } } var streamRequired = @ref.StreamReferencedIfAny; if (streamRequired != null) { var streamVariable = _streamVariables.Get(streamRequired.Value); if (streamVariable != null) { var def = _variableStreams.Get(streamVariable); if (def != null && def.Second) { measureReferencesMultivar = true; break; } } } } _isCollectMultimatches = measureReferencesMultivar || _isDefineAsksMultimatches; // create rowevent type var rowEventTypeName = statementContext.StatementId + "_rowrecogrow"; _rowEventType = statementContext.EventAdapterService.CreateAnonymousMapType(rowEventTypeName, rowTypeDef, true); // validate partition-by expressions, if any if (!matchRecognizeSpec.PartitionByExpressions.IsEmpty()) { var typeServicePartition = new StreamTypeServiceImpl(parentViewType, "MATCH_RECOGNIZE_PARTITION", true, statementContext.EngineURI); var validated = new List <ExprNode>(); var validationContext = new ExprValidationContext( typeServicePartition, statementContext.EngineImportService, statementContext.StatementExtensionServicesContext, null, statementContext.SchedulingService, statementContext.VariableService, statementContext.TableService, exprEvaluatorContext, statementContext.EventAdapterService, statementContext.StatementName, statementContext.StatementId, statementContext.Annotations, statementContext.ContextDescriptor, statementContext.ScriptingService, false, false, true, false, null, false); foreach (var partitionExpr in matchRecognizeSpec.PartitionByExpressions) { validated.Add(ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.MATCHRECOGPARTITION, partitionExpr, validationContext)); } matchRecognizeSpec.PartitionByExpressions = validated; } // validate interval if present if (matchRecognizeSpec.Interval != null) { var validationContext = new ExprValidationContext( new StreamTypeServiceImpl(statementContext.EngineURI, false), statementContext.EngineImportService, statementContext.StatementExtensionServicesContext, null, statementContext.SchedulingService, statementContext.VariableService, statementContext.TableService, exprEvaluatorContext, statementContext.EventAdapterService, statementContext.StatementName, statementContext.StatementId, statementContext.Annotations, statementContext.ContextDescriptor, statementContext.ScriptingService, false, false, true, false, null, false); matchRecognizeSpec.Interval.Validate(validationContext); } }
private static void HandleSubselectSelectClauses( int subselectStreamNumber, StatementContext statementContext, ExprSubselectNode subselect, EventType outerEventType, string outerEventTypeName, string outerStreamName, IDictionary <string, Pair <EventType, string> > taggedEventTypes, IDictionary <string, Pair <EventType, string> > arrayEventTypes) { var statementSpec = subselect.StatementSpecCompiled; var filterStreamSpec = statementSpec.StreamSpecs[0]; ViewFactoryChain viewFactoryChain; string subselecteventTypeName = null; // construct view factory chain try { if (statementSpec.StreamSpecs[0] is FilterStreamSpecCompiled) { var filterStreamSpecCompiled = (FilterStreamSpecCompiled)statementSpec.StreamSpecs[0]; subselecteventTypeName = filterStreamSpecCompiled.FilterSpec.FilterForEventTypeName; // A child view is required to limit the stream if (filterStreamSpec.ViewSpecs.Length == 0) { throw new ExprValidationException("Subqueries require one or more views to limit the stream, consider declaring a length or time window"); } // Register filter, create view factories viewFactoryChain = statementContext.ViewService.CreateFactories(subselectStreamNumber, filterStreamSpecCompiled.FilterSpec.ResultEventType, filterStreamSpec.ViewSpecs, filterStreamSpec.Options, statementContext); subselect.RawEventType = viewFactoryChain.EventType; } else { var namedSpec = (NamedWindowConsumerStreamSpec)statementSpec.StreamSpecs[0]; var processor = statementContext.NamedWindowService.GetProcessor(namedSpec.WindowName); viewFactoryChain = statementContext.ViewService.CreateFactories(0, processor.NamedWindowType, namedSpec.ViewSpecs, namedSpec.Options, statementContext); subselecteventTypeName = namedSpec.WindowName; EPLValidationUtil.ValidateContextName(false, processor.NamedWindowName, processor.ContextName, statementContext.ContextName, true); } } catch (ViewProcessingException ex) { throw new ExprValidationException("Error validating subexpression: " + ex.Message, ex); } // the final event type var eventType = viewFactoryChain.EventType; // determine a stream name unless one was supplied var subexpressionStreamName = filterStreamSpec.OptionalStreamName; if (subexpressionStreamName == null) { subexpressionStreamName = "$subselect_" + subselectStreamNumber; } // Named windows don't allow data views if (filterStreamSpec is NamedWindowConsumerStreamSpec) { EPStatementStartMethodHelperValidate.ValidateNoDataWindowOnNamedWindow(viewFactoryChain.FactoryChain); } // Streams event types are the original stream types with the stream zero the subselect stream var namesAndTypes = new LinkedHashMap <string, Pair <EventType, string> >(); namesAndTypes.Put(subexpressionStreamName, new Pair <EventType, string>(eventType, subselecteventTypeName)); namesAndTypes.Put(outerStreamName, new Pair <EventType, string>(outerEventType, outerEventTypeName)); if (taggedEventTypes != null) { foreach (KeyValuePair <string, Pair <EventType, string> > entry in taggedEventTypes) { namesAndTypes.Put(entry.Key, new Pair <EventType, string>(entry.Value.First, entry.Value.Second)); } } if (arrayEventTypes != null) { foreach (KeyValuePair <string, Pair <EventType, string> > entry in arrayEventTypes) { namesAndTypes.Put(entry.Key, new Pair <EventType, string>(entry.Value.First, entry.Value.Second)); } } StreamTypeService subselectTypeService = new StreamTypeServiceImpl(namesAndTypes, statementContext.EngineURI, true, true); var viewResourceDelegateSubselect = new ViewResourceDelegateUnverified(); subselect.FilterSubqueryStreamTypes = subselectTypeService; // Validate select expression var selectClauseSpec = subselect.StatementSpecCompiled.SelectClauseSpec; if (selectClauseSpec.SelectExprList.Length > 0) { if (selectClauseSpec.SelectExprList.Length > 1) { throw new ExprValidationException("Subquery multi-column select is not allowed in this context."); } var element = selectClauseSpec.SelectExprList[0]; if (element is SelectClauseExprCompiledSpec) { // validate var compiled = (SelectClauseExprCompiledSpec)element; var selectExpression = compiled.SelectExpression; var evaluatorContextStmt = new ExprEvaluatorContextStatement(statementContext, false); var validationContext = new ExprValidationContext( subselectTypeService, statementContext.MethodResolutionService, viewResourceDelegateSubselect, statementContext.SchedulingService, statementContext.VariableService, statementContext.TableService, evaluatorContextStmt, statementContext.EventAdapterService, statementContext.StatementName, statementContext.StatementId, statementContext.Annotations, statementContext.ContextDescriptor, statementContext.ScriptingService, false, false, true, false, null, false); selectExpression = ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.SUBQUERYSELECT, selectExpression, validationContext); subselect.SelectClause = new ExprNode[] { selectExpression }; subselect.SelectAsNames = new string[] { compiled.AssignedName }; // handle aggregation var aggExprNodes = new List <ExprAggregateNode>(); ExprAggregateNodeUtil.GetAggregatesBottomUp(selectExpression, aggExprNodes); if (aggExprNodes.Count > 0) { // Other stream properties, if there is aggregation, cannot be under aggregation. foreach (var aggNode in aggExprNodes) { var propertiesNodesAggregated = ExprNodeUtility.GetExpressionProperties(aggNode, true); foreach (var pair in propertiesNodesAggregated) { if (pair.First != 0) { throw new ExprValidationException("Subselect aggregation function cannot aggregate across correlated properties"); } } } // This stream (stream 0) properties must either all be under aggregation, or all not be. var propertiesNotAggregated = ExprNodeUtility.GetExpressionProperties(selectExpression, false); foreach (var pair in propertiesNotAggregated) { if (pair.First == 0) { throw new ExprValidationException("Subselect properties must all be within aggregation functions"); } } } } } }
public static RowRecogPlan ValidateAndPlan( IContainer container, EventType parentEventType, bool unbound, StatementBaseInfo @base, StatementCompileTimeServices services) { var statementRawInfo = @base.StatementRawInfo; var matchRecognizeSpec = @base.StatementSpec.Raw.MatchRecognizeSpec; var annotations = statementRawInfo.Annotations; var iterateOnly = HintEnum.ITERATE_ONLY.GetHint(annotations) != null; var additionalForgeables = new List<StmtClassForgeableFactory>(); // Expanded pattern already there RowRecogExprNode expandedPatternNode = matchRecognizeSpec.Pattern; // Determine single-row and multiple-row variables var variablesSingle = new LinkedHashSet<string>(); var variablesMultiple = new LinkedHashSet<string>(); RowRecogHelper.RecursiveInspectVariables(expandedPatternNode, false, variablesSingle, variablesMultiple); // each variable gets associated with a stream number (multiple-row variables as well to hold the current event for the expression). var streamNum = 0; var variableStreams = new LinkedHashMap<string, Pair<int, bool>>(); foreach (var variableSingle in variablesSingle) { variableStreams.Put(variableSingle, new Pair<int, bool>(streamNum, false)); streamNum++; } foreach (var variableMultiple in variablesMultiple) { variableStreams.Put(variableMultiple, new Pair<int, bool>(streamNum, true)); streamNum++; } // mapping of stream to variable var streamVariables = new OrderedListDictionary<int, string>(); foreach (var entry in variableStreams) { streamVariables.Put(entry.Value.First, entry.Key); } // determine visibility rules var visibility = RowRecogHelper.DetermineVisibility(expandedPatternNode); // assemble all single-row variables for expression validation var allStreamNames = new string[variableStreams.Count]; var allTypes = new EventType[variableStreams.Count]; streamNum = 0; foreach (var variableSingle in variablesSingle) { allStreamNames[streamNum] = variableSingle; allTypes[streamNum] = parentEventType; streamNum++; } foreach (var variableMultiple in variablesMultiple) { allStreamNames[streamNum] = variableMultiple; allTypes[streamNum] = parentEventType; streamNum++; } // determine type service for use with DEFINE // validate each DEFINE clause expression ISet<string> definedVariables = new HashSet<string>(); IList<ExprAggregateNode> aggregateNodes = new List<ExprAggregateNode>(); var isExprRequiresMultimatchState = new bool[variableStreams.Count]; var previousNodes = new OrderedListDictionary<int, IList<ExprPreviousMatchRecognizeNode>>(); for (var defineIndex = 0; defineIndex < matchRecognizeSpec.Defines.Count; defineIndex++) { MatchRecognizeDefineItem defineItem = matchRecognizeSpec.Defines[defineIndex]; if (definedVariables.Contains(defineItem.Identifier)) { throw new ExprValidationException( "Variable '" + defineItem.Identifier + "' has already been defined"); } definedVariables.Add(defineItem.Identifier); // stream-type visibilities handled here var typeServiceDefines = BuildDefineStreamTypeServiceDefine( defineIndex, variableStreams, defineItem, visibility, parentEventType, statementRawInfo, services); var exprNodeResult = HandlePreviousFunctions(defineItem.Expression, previousNodes); var validationContext = new ExprValidationContextBuilder(typeServiceDefines, statementRawInfo, services) .WithAllowBindingConsumption(true) .WithDisablePropertyExpressionEventCollCache(true) .Build(); ExprNode validated; try { // validate validated = ExprNodeUtilityValidate.GetValidatedSubtree( ExprNodeOrigin.MATCHRECOGDEFINE, exprNodeResult, validationContext); // check aggregates defineItem.Expression = validated; ExprAggregateNodeUtil.GetAggregatesBottomUp(validated, aggregateNodes); if (!aggregateNodes.IsEmpty()) { throw new ExprValidationException("An aggregate function may not appear in a DEFINE clause"); } } catch (ExprValidationException ex) { throw new ExprValidationException( "Failed to validate condition expression for variable '" + defineItem.Identifier + "': " + ex.Message, ex); } // determine access to event properties from multi-matches var visitor = new ExprNodeStreamRequiredVisitor(); validated.Accept(visitor); var streamsRequired = visitor.StreamsRequired; foreach (var streamRequired in streamsRequired) { if (streamRequired >= variableStreams.Count) { var streamNumIdent = variableStreams.Get(defineItem.Identifier).First; isExprRequiresMultimatchState[streamNumIdent] = true; break; } } } var defineAsksMultimatches = CollectionUtil.IsAnySet(isExprRequiresMultimatchState); // determine type service for use with MEASURE IDictionary<string, object> measureTypeDef = new LinkedHashMap<string, object>(); foreach (var variableSingle in variablesSingle) { measureTypeDef.Put(variableSingle, parentEventType); } foreach (var variableMultiple in variablesMultiple) { measureTypeDef.Put(variableMultiple, new[] {parentEventType}); } var compositeTypeName = services.EventTypeNameGeneratorStatement.AnonymousRowrecogCompositeName; var compositeTypeMetadata = new EventTypeMetadata( compositeTypeName, @base.ModuleName, EventTypeTypeClass.MATCHRECOGDERIVED, EventTypeApplicationType.OBJECTARR, NameAccessModifier.TRANSIENT, EventTypeBusModifier.NONBUS, false, EventTypeIdPair.Unassigned()); var compositeEventType = BaseNestableEventUtil.MakeOATypeCompileTime( compositeTypeMetadata, measureTypeDef, null, null, null, null, services.BeanEventTypeFactoryPrivate, services.EventTypeCompileTimeResolver); services.EventTypeCompileTimeRegistry.NewType(compositeEventType); StreamTypeService compositeTypeServiceMeasure = new StreamTypeServiceImpl(compositeEventType, "MATCH_RECOGNIZE", true); // find MEASURE clause aggregations var measureReferencesMultivar = false; IList<ExprAggregateNode> measureAggregateExprNodes = new List<ExprAggregateNode>(); foreach (var measureItem in matchRecognizeSpec.Measures) { ExprAggregateNodeUtil.GetAggregatesBottomUp(measureItem.Expr, measureAggregateExprNodes); } AggregationServiceForgeDesc[] aggregationServices = null; if (!measureAggregateExprNodes.IsEmpty()) { aggregationServices = PlanAggregations( measureAggregateExprNodes, compositeTypeServiceMeasure, allStreamNames, allTypes, streamVariables, variablesMultiple, @base, services); foreach (AggregationServiceForgeDesc svc in aggregationServices) { if (svc != null) { additionalForgeables.AddAll(svc.AdditionalForgeables); } } } // validate each MEASURE clause expression IDictionary<string, object> rowTypeDef = new LinkedHashMap<string, object>(); var streamRefVisitor = new ExprNodeStreamUseCollectVisitor(); foreach (var measureItem in matchRecognizeSpec.Measures) { if (measureItem.Name == null) { throw new ExprValidationException( "The measures clause requires that each expression utilizes the AS keyword to assign a column name"); } var validated = ValidateMeasureClause( measureItem.Expr, compositeTypeServiceMeasure, variablesMultiple, variablesSingle, statementRawInfo, services); measureItem.Expr = validated; rowTypeDef.Put(measureItem.Name, validated.Forge.EvaluationType); validated.Accept(streamRefVisitor); } // Determine if any of the multi-var streams are referenced in the measures (non-aggregated only) foreach (var @ref in streamRefVisitor.Referenced) { var rootPropName = @ref.RootPropertyNameIfAny; if (rootPropName != null) { if (variablesMultiple.Contains(rootPropName)) { measureReferencesMultivar = true; break; } } var streamRequired = @ref.StreamReferencedIfAny; if (streamRequired != null) { var streamVariable = streamVariables.Get(streamRequired.Value); if (streamVariable != null) { var def = variableStreams.Get(streamVariable); if (def != null && def.Second) { measureReferencesMultivar = true; break; } } } } var collectMultimatches = measureReferencesMultivar || defineAsksMultimatches; // create rowevent type var rowTypeName = services.EventTypeNameGeneratorStatement.AnonymousRowrecogRowName; var rowTypeMetadata = new EventTypeMetadata( rowTypeName, @base.ModuleName, EventTypeTypeClass.MATCHRECOGDERIVED, EventTypeApplicationType.MAP, NameAccessModifier.TRANSIENT, EventTypeBusModifier.NONBUS, false, EventTypeIdPair.Unassigned()); var rowEventType = BaseNestableEventUtil.MakeMapTypeCompileTime( rowTypeMetadata, rowTypeDef, null, null, null, null, services.BeanEventTypeFactoryPrivate, services.EventTypeCompileTimeResolver); services.EventTypeCompileTimeRegistry.NewType(rowEventType); // validate partition-by expressions, if any ExprNode[] partitionBy; MultiKeyClassRef partitionMultiKey; if (!matchRecognizeSpec.PartitionByExpressions.IsEmpty()) { StreamTypeService typeServicePartition = new StreamTypeServiceImpl( parentEventType, "MATCH_RECOGNIZE_PARTITION", true); IList<ExprNode> validated = new List<ExprNode>(); var validationContext = new ExprValidationContextBuilder(typeServicePartition, statementRawInfo, services) .WithAllowBindingConsumption(true) .Build(); foreach (var partitionExpr in matchRecognizeSpec.PartitionByExpressions) { validated.Add( ExprNodeUtilityValidate.GetValidatedSubtree( ExprNodeOrigin.MATCHRECOGPARTITION, partitionExpr, validationContext)); } matchRecognizeSpec.PartitionByExpressions = validated; partitionBy = ExprNodeUtilityQuery.ToArray(validated); MultiKeyPlan multiKeyPlan = MultiKeyPlanner.PlanMultiKey(partitionBy, false, @base.StatementRawInfo, services.SerdeResolver); partitionMultiKey = multiKeyPlan.ClassRef; additionalForgeables.AddAll(multiKeyPlan.MultiKeyForgeables); } else { partitionBy = null; partitionMultiKey = null; } // validate interval if present if (matchRecognizeSpec.Interval != null) { var validationContext = new ExprValidationContextBuilder(new StreamTypeServiceImpl(false), statementRawInfo, services) .WithAllowBindingConsumption(true) .Build(); var validated = (ExprTimePeriod) ExprNodeUtilityValidate.GetValidatedSubtree( ExprNodeOrigin.MATCHRECOGINTERVAL, matchRecognizeSpec.Interval.TimePeriodExpr, validationContext); matchRecognizeSpec.Interval.TimePeriodExpr = validated; } // compile variable definition expressions IDictionary<string, ExprNode> variableDefinitions = new Dictionary<string, ExprNode>(); foreach (var defineItem in matchRecognizeSpec.Defines) { variableDefinitions.Put(defineItem.Identifier, defineItem.Expression); } // create evaluators var columnNames = new string[matchRecognizeSpec.Measures.Count]; var columnForges = new ExprNode[matchRecognizeSpec.Measures.Count]; var count = 0; foreach (var measureItem in matchRecognizeSpec.Measures) { columnNames[count] = measureItem.Name; columnForges[count] = measureItem.Expr; count++; } // build states var strand = RowRecogHelper.BuildStartStates( expandedPatternNode, variableDefinitions, variableStreams, isExprRequiresMultimatchState); var startStates = strand.StartStates.ToArray(); RowRecogNFAStateForge[] allStates = strand.AllStates.ToArray(); if (Log.IsInfoEnabled) { Log.Info("NFA tree:\n" + RowRecogNFAViewUtil.Print(startStates)); } // determine names of multimatching variables string[] multimatchVariablesArray; int[] multimatchStreamNumToVariable; int[] multimatchVariableToStreamNum; if (variablesSingle.Count == variableStreams.Count) { multimatchVariablesArray = new string[0]; multimatchStreamNumToVariable = new int[0]; multimatchVariableToStreamNum = new int[0]; } else { multimatchVariablesArray = new string[variableStreams.Count - variablesSingle.Count]; multimatchVariableToStreamNum = new int[multimatchVariablesArray.Length]; multimatchStreamNumToVariable = new int[variableStreams.Count]; CompatExtensions.Fill(multimatchStreamNumToVariable, -1); count = 0; foreach (var entry in variableStreams) { if (entry.Value.Second) { var index = count; multimatchVariablesArray[index] = entry.Key; multimatchVariableToStreamNum[index] = entry.Value.First; multimatchStreamNumToVariable[entry.Value.First] = index; count++; } } } var numEventsEventsPerStreamDefine = defineAsksMultimatches ? variableStreams.Count + 1 : variableStreams.Count; // determine interval-or-terminated var orTerminated = matchRecognizeSpec.Interval != null && matchRecognizeSpec.Interval.IsOrTerminated; TimePeriodComputeForge intervalCompute = null; if (matchRecognizeSpec.Interval != null) { intervalCompute = matchRecognizeSpec.Interval.TimePeriodExpr.TimePeriodComputeForge; } EventType multimatchEventType = null; if (defineAsksMultimatches) { multimatchEventType = GetDefineMultimatchEventType(variableStreams, parentEventType, @base, services); } // determine previous-access indexes and assign "prev" node indexes // Since an expression such as "prior(2, price), prior(8, price)" translates into {2, 8} the relative index is {0, 1}. // Map the expression-supplied index to a relative index int[] previousRandomAccessIndexes = null; if (!previousNodes.IsEmpty()) { previousRandomAccessIndexes = new int[previousNodes.Count]; var countPrev = 0; foreach (var entry in previousNodes) { previousRandomAccessIndexes[countPrev] = entry.Key; foreach (var callback in entry.Value) { callback.AssignedIndex = countPrev; } countPrev++; } } RowRecogDescForge forge = new RowRecogDescForge( parentEventType, rowEventType, compositeEventType, multimatchEventType, multimatchStreamNumToVariable, multimatchVariableToStreamNum, partitionBy, partitionMultiKey, variableStreams, matchRecognizeSpec.Interval != null, iterateOnly, unbound, orTerminated, collectMultimatches, defineAsksMultimatches, numEventsEventsPerStreamDefine, multimatchVariablesArray, startStates, allStates, matchRecognizeSpec.IsAllMatches, matchRecognizeSpec.Skip.Skip, columnForges, columnNames, intervalCompute, previousRandomAccessIndexes, aggregationServices); return new RowRecogPlan(forge, additionalForgeables); }
public static IList <StmtClassForgeableFactory> HandleSubselectSelectClauses( ExprSubselectNode subselect, EventType outerEventType, string outerEventTypeName, string outerStreamName, IDictionary <string, Pair <EventType, string> > taggedEventTypes, IDictionary <string, Pair <EventType, string> > arrayEventTypes, StatementRawInfo statementRawInfo, StatementCompileTimeServices services) { if (subselect.SubselectNumber == -1) { throw new IllegalStateException("Subselect is unassigned"); } var statementSpec = subselect.StatementSpecCompiled; var filterStreamSpec = statementSpec.StreamSpecs[0]; IList <ViewFactoryForge> viewForges; string subselecteventTypeName; IList <StmtClassForgeableFactory> additionalForgeables; // construct view factory chain EventType eventType; try { var args = new ViewFactoryForgeArgs( -1, true, subselect.SubselectNumber, StreamSpecOptions.DEFAULT, null, statementRawInfo, services); var streamSpec = statementSpec.StreamSpecs[0]; if (streamSpec is FilterStreamSpecCompiled) { var filterStreamSpecCompiled = (FilterStreamSpecCompiled)statementSpec.StreamSpecs[0]; subselecteventTypeName = filterStreamSpecCompiled.FilterSpecCompiled.FilterForEventTypeName; // A child view is required to limit the stream if (filterStreamSpec.ViewSpecs.Length == 0) { throw new ExprValidationException( "Subqueries require one or more views to limit the stream, consider declaring a length or time window"); } ViewFactoryForgeDesc viewForgeDesc = ViewFactoryForgeUtil.CreateForges( filterStreamSpecCompiled.ViewSpecs, args, filterStreamSpecCompiled.FilterSpecCompiled.ResultEventType); viewForges = viewForgeDesc.Forges; additionalForgeables = viewForgeDesc.MultikeyForges; // Register filter, create view factories eventType = viewForges.IsEmpty() ? filterStreamSpecCompiled.FilterSpecCompiled.ResultEventType : viewForges[viewForges.Count - 1].EventType; subselect.RawEventType = eventType; } else if (streamSpec is NamedWindowConsumerStreamSpec) { var namedSpec = (NamedWindowConsumerStreamSpec)statementSpec.StreamSpecs[0]; var namedWindow = namedSpec.NamedWindow; ViewFactoryForgeDesc viewForgeDesc = ViewFactoryForgeUtil.CreateForges(namedSpec.ViewSpecs, args, namedWindow.EventType); viewForges = viewForgeDesc.Forges; additionalForgeables = viewForgeDesc.MultikeyForges; var namedWindowName = namedWindow.EventType.Name; subselecteventTypeName = namedWindowName; EPLValidationUtil.ValidateContextName(false, namedWindowName, namedWindow.ContextName, statementRawInfo.ContextName, true); subselect.RawEventType = namedWindow.EventType; eventType = namedWindow.EventType; } else if (streamSpec is TableQueryStreamSpec) { var namedSpec = (TableQueryStreamSpec)statementSpec.StreamSpecs[0]; var table = namedSpec.Table; ViewFactoryForgeDesc viewForgeDesc = ViewFactoryForgeUtil.CreateForges(namedSpec.ViewSpecs, args, table.InternalEventType); viewForges = viewForgeDesc.Forges; additionalForgeables = viewForgeDesc.MultikeyForges; var namedWindowName = table.TableName; subselecteventTypeName = namedWindowName; EPLValidationUtil.ValidateContextName(false, namedWindowName, table.OptionalContextName, statementRawInfo.ContextName, true); subselect.RawEventType = table.InternalEventType; eventType = table.InternalEventType; } else { throw new IllegalStateException("Unexpected stream spec " + streamSpec); } } catch (ViewProcessingException ex) { throw new ExprValidationException("Failed to validate subexpression: " + ex.Message, ex); } // determine a stream name unless one was supplied var subexpressionStreamName = SubselectUtil.GetStreamName(filterStreamSpec.OptionalStreamName, subselect.SubselectNumber); // Named windows don't allow data views if (filterStreamSpec is NamedWindowConsumerStreamSpec | filterStreamSpec is TableQueryStreamSpec) { EPStatementStartMethodHelperValidate.ValidateNoDataWindowOnNamedWindow(viewForges); } // Streams event types are the original stream types with the stream zero the subselect stream var namesAndTypes = new LinkedHashMap <string, Pair <EventType, string> >(); namesAndTypes.Put(subexpressionStreamName, new Pair <EventType, string>(eventType, subselecteventTypeName)); namesAndTypes.Put(outerStreamName, new Pair <EventType, string>(outerEventType, outerEventTypeName)); if (taggedEventTypes != null) { foreach (var entry in taggedEventTypes) { namesAndTypes.Put(entry.Key, new Pair <EventType, string>(entry.Value.First, entry.Value.Second)); } } if (arrayEventTypes != null) { foreach (var entry in arrayEventTypes) { namesAndTypes.Put(entry.Key, new Pair <EventType, string>(entry.Value.First, entry.Value.Second)); } } StreamTypeService subselectTypeService = new StreamTypeServiceImpl(namesAndTypes, true, true); var viewResourceDelegateSubselect = new ViewResourceDelegateExpr(); subselect.FilterSubqueryStreamTypes = subselectTypeService; // Validate select expression var selectClauseSpec = subselect.StatementSpecCompiled.SelectClauseCompiled; if (selectClauseSpec.SelectExprList.Length > 0) { if (selectClauseSpec.SelectExprList.Length > 1) { throw new ExprValidationException("Subquery multi-column select is not allowed in this context."); } var element = selectClauseSpec.SelectExprList[0]; if (element is SelectClauseExprCompiledSpec) { // validate var compiled = (SelectClauseExprCompiledSpec)element; var selectExpression = compiled.SelectExpression; var validationContext = new ExprValidationContextBuilder(subselectTypeService, statementRawInfo, services) .WithViewResourceDelegate(viewResourceDelegateSubselect) .WithAllowBindingConsumption(true) .WithMemberName(new ExprValidationMemberNameQualifiedSubquery(subselect.SubselectNumber)) .Build(); selectExpression = ExprNodeUtilityValidate.GetValidatedSubtree(ExprNodeOrigin.SUBQUERYSELECT, selectExpression, validationContext); subselect.SelectClause = new ExprNode[] { selectExpression }; subselect.SelectAsNames = new string[] { compiled.AssignedName }; // handle aggregation var aggExprNodes = new List <ExprAggregateNode>(); ExprAggregateNodeUtil.GetAggregatesBottomUp(selectExpression, aggExprNodes); if (aggExprNodes.Count > 0) { // Other stream properties, if there is aggregation, cannot be under aggregation. foreach (var aggNode in aggExprNodes) { var propertiesNodesAggregated = ExprNodeUtilityQuery.GetExpressionProperties(aggNode, true); foreach (var pair in propertiesNodesAggregated) { if (pair.First != 0) { throw new ExprValidationException("Subselect aggregation function cannot aggregate across correlated properties"); } } } // This stream (stream 0) properties must either all be under aggregation, or all not be. var propertiesNotAggregated = ExprNodeUtilityQuery.GetExpressionProperties(selectExpression, false); foreach (var pair in propertiesNotAggregated) { if (pair.First == 0) { throw new ExprValidationException("Subselect properties must all be within aggregation functions"); } } } } } return(additionalForgeables); }
public void Attach(EventType parentEventType, StatementContext statementContext, ViewFactory optionalParentFactory, IList <ViewFactory> parentViewFactories) { _eventType = parentEventType; // define built-in fields var builtinTypeDef = ExpressionViewOAFieldEnumExtensions.AsMapOfTypes(_eventType); _builtinMapType = statementContext.EventAdapterService.CreateAnonymousObjectArrayType( statementContext.StatementId + "_exprview", builtinTypeDef); StreamTypeService streamTypeService = new StreamTypeServiceImpl(new EventType[] { _eventType, _builtinMapType }, new String[2], new bool[2], statementContext.EngineURI, false); // validate expression ExpiryExpression = ViewFactorySupport.ValidateExpr(ViewName, statementContext, ExpiryExpression, streamTypeService, 0); var summaryVisitor = new ExprNodeSummaryVisitor(); ExpiryExpression.Accept(summaryVisitor); if (summaryVisitor.HasSubselect || summaryVisitor.HasStreamSelect || summaryVisitor.HasPreviousPrior) { throw new ViewParameterException("Invalid expiry expression: Sub-select, previous or prior functions are not supported in this context"); } var returnType = ExpiryExpression.ExprEvaluator.ReturnType; if (returnType.GetBoxedType() != typeof(bool?)) { throw new ViewParameterException("Invalid return value for expiry expression, expected a bool return value but received " + returnType.GetParameterAsString()); } // determine variables used, if any var visitor = new ExprNodeVariableVisitor(); ExpiryExpression.Accept(visitor); VariableNames = visitor.VariableNames; // determine aggregation nodes, if any var aggregateNodes = new List <ExprAggregateNode>(); ExprAggregateNodeUtil.GetAggregatesBottomUp(ExpiryExpression, aggregateNodes); if (aggregateNodes.IsNotEmpty()) { try { AggregationServiceFactoryDesc = AggregationServiceFactoryFactory.GetService( Collections.GetEmptyList <ExprAggregateNode>(), Collections.GetEmptyMap <ExprNode, String>(), Collections.GetEmptyList <ExprDeclaredNode>(), null, aggregateNodes, Collections.GetEmptyList <ExprAggregateNode>(), Collections.GetEmptyList <ExprAggregateNodeGroupKey>(), false, statementContext.Annotations, statementContext.VariableService, false, false, null, null, AggregationServiceFactoryServiceImpl.DEFAULT_FACTORY, streamTypeService.EventTypes, statementContext.MethodResolutionService, null, statementContext.ContextName, null, null); } catch (ExprValidationException ex) { throw new ViewParameterException(ex.Message, ex); } } }
public static ExprNode ValidateNodes( StatementSpecRaw statementSpec, StreamTypeService typeService, ViewResourceDelegateExpr viewResourceDelegate, StatementRawInfo statementRawInfo, StatementCompileTimeServices compileTimeServices) { var intoTableName = statementSpec.IntoTableSpec == null ? null : statementSpec.IntoTableSpec.Name; ExprNode whereClauseValidated = null; if (statementSpec.WhereClause != null) { var whereClause = statementSpec.WhereClause; // Validate where clause, initializing nodes to the stream ids used try { var validationContext = new ExprValidationContextBuilder( typeService, statementRawInfo, compileTimeServices) .WithViewResourceDelegate(viewResourceDelegate) .WithAllowBindingConsumption(true) .WithIntoTableName(intoTableName) .Build(); whereClause = ExprNodeUtilityValidate.GetValidatedSubtree( ExprNodeOrigin.FILTER, whereClause, validationContext); if (whereClause.Forge.EvaluationType != typeof(bool) && whereClause.Forge.EvaluationType != typeof(bool?)) { throw new ExprValidationException( "The where-clause filter expression must return a boolean value"); } whereClauseValidated = whereClause; // Make sure there is no aggregation in the where clause IList<ExprAggregateNode> aggregateNodes = new List<ExprAggregateNode>(); ExprAggregateNodeUtil.GetAggregatesBottomUp(whereClause, aggregateNodes); if (!aggregateNodes.IsEmpty()) { throw new ExprValidationException( "An aggregate function may not appear in a WHERE clause (use the HAVING clause)"); } } catch (ExprValidationException ex) { throw new ExprValidationException("Failed to validate expression: " + ex.Message, ex); } } if ((statementSpec.OutputLimitSpec != null) && (statementSpec.OutputLimitSpec.WhenExpressionNode != null || statementSpec.OutputLimitSpec.AndAfterTerminateExpr != null || statementSpec.OutputLimitSpec.AndAfterTerminateThenExpressions != null)) { // Validate where clause, initializing nodes to the stream ids used EventType outputLimitType = OutputConditionExpressionTypeUtil.GetBuiltInEventType( statementRawInfo.ModuleName, compileTimeServices.BeanEventTypeFactoryPrivate); StreamTypeService typeServiceOutputWhen = new StreamTypeServiceImpl( new[] {outputLimitType}, new string[] {null}, new[] {true}, false, false); var validationContext = new ExprValidationContextBuilder( typeServiceOutputWhen, statementRawInfo, compileTimeServices) .WithIntoTableName(intoTableName) .Build(); var outputLimitWhenNode = statementSpec.OutputLimitSpec.WhenExpressionNode; if (outputLimitWhenNode != null) { outputLimitWhenNode = ExprNodeUtilityValidate.GetValidatedSubtree( ExprNodeOrigin.OUTPUTLIMIT, outputLimitWhenNode, validationContext); statementSpec.OutputLimitSpec.WhenExpressionNode = outputLimitWhenNode; if (outputLimitWhenNode.Forge.EvaluationType.GetBoxedType() != typeof(bool?)) { throw new ExprValidationException( "The when-trigger expression in the OUTPUT WHEN clause must return a boolean-type value"); } ValidateNoAggregations( outputLimitWhenNode, "An aggregate function may not appear in a OUTPUT LIMIT clause"); } // validate and-terminate expression if provided if (statementSpec.OutputLimitSpec.AndAfterTerminateExpr != null) { if (statementSpec.OutputLimitSpec.RateType != OutputLimitRateType.WHEN_EXPRESSION && statementSpec.OutputLimitSpec.RateType != OutputLimitRateType.TERM) { throw new ExprValidationException( "A terminated-and expression must be used with the OUTPUT WHEN clause"); } var validated = ExprNodeUtilityValidate.GetValidatedSubtree( ExprNodeOrigin.OUTPUTLIMIT, statementSpec.OutputLimitSpec.AndAfterTerminateExpr, validationContext); statementSpec.OutputLimitSpec.AndAfterTerminateExpr = validated; if (validated.Forge.EvaluationType.GetBoxedType() != typeof(bool?)) { throw new ExprValidationException( "The terminated-and expression must return a boolean-type value"); } ValidateNoAggregations( validated, "An aggregate function may not appear in a terminated-and clause"); } // validate then-expression ValidateThenSetAssignments(statementSpec.OutputLimitSpec.ThenExpressions, validationContext, false); // validate after-terminated then-expression ValidateThenSetAssignments(statementSpec.OutputLimitSpec.AndAfterTerminateThenExpressions, validationContext, false); } for (var outerJoinCount = 0; outerJoinCount < statementSpec.OuterJoinDescList.Count; outerJoinCount++) { var outerJoinDesc = statementSpec.OuterJoinDescList[outerJoinCount]; // validate on-expression nodes, if provided if (outerJoinDesc.OptLeftNode != null) { var streamIdPair = ValidateOuterJoinPropertyPair( outerJoinDesc.OptLeftNode, outerJoinDesc.OptRightNode, outerJoinCount, typeService, viewResourceDelegate, statementRawInfo, compileTimeServices ); if (outerJoinDesc.AdditionalLeftNodes != null) { ISet<int> streamSet = new HashSet<int>(); streamSet.Add(streamIdPair.First); streamSet.Add(streamIdPair.Second); for (var i = 0; i < outerJoinDesc.AdditionalLeftNodes.Length; i++) { var streamIdPairAdd = ValidateOuterJoinPropertyPair( outerJoinDesc.AdditionalLeftNodes[i], outerJoinDesc.AdditionalRightNodes[i], outerJoinCount, typeService, viewResourceDelegate, statementRawInfo, compileTimeServices ); // make sure all additional properties point to the same two streams if (!streamSet.Contains(streamIdPairAdd.First) || !streamSet.Contains(streamIdPairAdd.Second)) { var message = "Outer join ON-clause columns must refer to properties of the same joined streams" + " when using multiple columns in the on-clause"; throw new ExprValidationException("Failed to validate outer-join expression: " + message); } } } } } return whereClauseValidated; }
public static OrderByProcessorFactoryForge GetProcessor( IList<SelectClauseExprCompiledSpec> selectionList, IList<OrderByItem> orderByList, RowLimitSpec rowLimitSpec, VariableCompileTimeResolver variableCompileTimeResolver, bool isSortUsingCollator, string optionalContextName, OrderByElementForge[][] orderByRollup) { // Get the order by expression nodes IList<ExprNode> orderByNodes = new List<ExprNode>(); foreach (var element in orderByList) { orderByNodes.Add(element.ExprNode); } // No order-by clause if (orderByList.IsEmpty()) { Log.Debug(".getProcessor Using no OrderByProcessor"); if (rowLimitSpec != null) { var rowLimitProcessorFactory = new RowLimitProcessorFactoryForge( rowLimitSpec, variableCompileTimeResolver, optionalContextName); return new OrderByProcessorRowLimitOnlyForge(rowLimitProcessorFactory); } return null; } // Determine aggregate functions used in select, if any IList<ExprAggregateNode> selectAggNodes = new List<ExprAggregateNode>(); foreach (var element in selectionList) { ExprAggregateNodeUtil.GetAggregatesBottomUp(element.SelectExpression, selectAggNodes); } // Get all the aggregate functions occuring in the order-by clause IList<ExprAggregateNode> orderAggNodes = new List<ExprAggregateNode>(); foreach (var orderByNode in orderByNodes) { ExprAggregateNodeUtil.GetAggregatesBottomUp(orderByNode, orderAggNodes); } ValidateOrderByAggregates(selectAggNodes, orderAggNodes); // Tell the order-by processor whether to compute group-by // keys if they are not present var needsGroupByKeys = !selectionList.IsEmpty() && !orderAggNodes.IsEmpty(); Log.Debug(".getProcessor Using OrderByProcessorImpl"); var elements = ToElementArray(orderByList); var comparator = GetComparator(elements, isSortUsingCollator); var orderByProcessorForge = new OrderByProcessorForgeImpl( elements, needsGroupByKeys, orderByRollup, comparator); if (rowLimitSpec == null) { return orderByProcessorForge; } { var rowLimitProcessorFactory = new RowLimitProcessorFactoryForge( rowLimitSpec, variableCompileTimeResolver, optionalContextName); return new OrderByProcessorOrderedLimitForge(orderByProcessorForge, rowLimitProcessorFactory); } }