/// <summary> /// Inspect variables recursively. /// </summary> /// <param name="parent">parent regex expression node</param> /// <param name="isMultiple">if the variable in the stack is multiple of single</param> /// <param name="variablesSingle">single variables list</param> /// <param name="variablesMultiple">group variables list</param> public static void RecursiveInspectVariables(RowRegexExprNode parent, bool isMultiple, ISet <String> variablesSingle, ISet <String> variablesMultiple) { if (parent is RowRegexExprNodeNested) { var nested = (RowRegexExprNodeNested)parent; foreach (var child in parent.ChildNodes) { RecursiveInspectVariables(child, nested.NFAType.IsMultipleMatches() || isMultiple, variablesSingle, variablesMultiple); } } else if (parent is RowRegexExprNodeAlteration) { foreach (var childAlteration in parent.ChildNodes) { var singles = new LinkedHashSet <String>(); var multiples = new LinkedHashSet <String>(); RecursiveInspectVariables(childAlteration, isMultiple, singles, multiples); variablesMultiple.AddAll(multiples); variablesSingle.AddAll(singles); } variablesSingle.RemoveAll(variablesMultiple); } else if (parent is RowRegexExprNodeAtom) { var atom = (RowRegexExprNodeAtom)parent; var name = atom.Tag; if (variablesMultiple.Contains(name)) { return; } if (variablesSingle.Contains(name)) { variablesSingle.Remove(name); variablesMultiple.Add(name); return; } if (atom.NFAType.IsMultipleMatches()) { variablesMultiple.Add(name); return; } if (isMultiple) { variablesMultiple.Add(name); } else { variablesSingle.Add(name); } } else { foreach (var child in parent.ChildNodes) { RecursiveInspectVariables(child, isMultiple, variablesSingle, variablesMultiple); } } }
private static RowRegexExprNode Replace(RowRegexExprNode optionalParent, RowRegexExprNode originalNode, IList <RowRegexExprNode> expandedRepeat) { if (optionalParent == null) { var newParentNode = new RowRegexExprNodeConcatenation(); newParentNode.ChildNodes.AddAll(expandedRepeat); return(newParentNode); } // for nested nodes, use a concatenation instead if (optionalParent is RowRegexExprNodeNested || optionalParent is RowRegexExprNodeAlteration) { var concatenation = new RowRegexExprNodeConcatenation(); concatenation.ChildNodes.AddAll(expandedRepeat); optionalParent.ReplaceChildNode(originalNode, Collections.SingletonList <RowRegexExprNode>(concatenation)); } // concatenations are simply changed else { optionalParent.ReplaceChildNode(originalNode, expandedRepeat); } return(null); }
/// <summary> /// Build a list of start states from the parent node. /// </summary> /// <param name="parent">to build start state for</param> /// <param name="variableDefinitions">each variable and its expressions</param> /// <param name="variableStreams">variable name and its stream number</param> /// <param name="exprRequiresMultimatchState">State of the expr requires multimatch.</param> /// <returns>strand of regex state nodes</returns> public static RegexNFAStrandResult RecursiveBuildStartStates( RowRegexExprNode parent, IDictionary <String, ExprNode> variableDefinitions, IDictionary <String, Pair <int, bool> > variableStreams, bool[] exprRequiresMultimatchState) { var nodeNumStack = new Stack <int>(); var strand = RecursiveBuildStatesInternal( parent, variableDefinitions, variableStreams, nodeNumStack, exprRequiresMultimatchState); // add end state var end = new RegexNFAStateEnd(); foreach (var endStates in strand.EndStates) { endStates.AddState(end); } // assign node num as a counter var nodeNumberFlat = 0; foreach (var theBase in strand.AllStates) { theBase.NodeNumFlat = nodeNumberFlat++; } return(new RegexNFAStrandResult(new List <RegexNFAState>(strand.StartStates), strand.AllStates)); }
public static IDictionary <String, ISet <String> > DetermineVisibility(RowRegexExprNode pattern) { IDictionary <String, ISet <String> > map = new Dictionary <String, ISet <String> >(); var path = new ArrayDeque <RowRegexExprNode>(); RecursiveFindPatternAtoms(pattern, path, map); return(map); }
public RowRegexExprNode Copy( RowRegexExprNode nodeToCopy, RegexNFATypeEnum newType) { var atom = (RowRegexExprNodeAtom)nodeToCopy; return(new RowRegexExprNodeAtom(atom.Tag, newType, null)); }
public virtual void AcceptChildnodes(RowRegexExprNodeVisitor visitor, RowRegexExprNode parent, int level) { visitor.Visit(this, parent, level); foreach (RowRegexExprNode childNode in ChildNodes) { childNode.AcceptChildnodes(visitor, this, level + 1); } }
private static RowRegexExprNode CheckedCopy(RowRegexExprNode inner) { try { return((RowRegexExprNode)SerializableObjectCopier.Copy(inner)); } catch (Exception e) { throw new EPException("Failed to repeat nested match-recognize: " + e.Message, e); } }
public RowRegexExprNode Copy(RowRegexExprNode nodeToCopy, RegexNFATypeEnum newType) { var nested = (RowRegexExprNodeNested)nodeToCopy; var nestedCopy = new RowRegexExprNodeNested(newType, null); foreach (var inner in nested.ChildNodes) { var innerCopy = CheckedCopy(inner); nestedCopy.AddChildNode(innerCopy); } return(nestedCopy); }
private static void RecursiveCollectAtomsWExclude(RowRegexExprNode node, ISet <String> identifiers, String excludedTag) { if (node is RowRegexExprNodeAtom) { var atom = (RowRegexExprNodeAtom)node; if (!excludedTag.Equals(atom.Tag)) { identifiers.Add(atom.Tag); } } foreach (var child in node.ChildNodes) { RecursiveCollectAtomsWExclude(child, identifiers, excludedTag); } }
private static void RecursiveFindPatternAtoms(RowRegexExprNode parent, ArrayDeque <RowRegexExprNode> path, IDictionary <String, ISet <String> > map) { path.Add(parent); foreach (var child in parent.ChildNodes) { if (child is RowRegexExprNodeAtom) { HandleAtom((RowRegexExprNodeAtom)child, path, map); } else { RecursiveFindPatternAtoms(child, path, map); } } path.RemoveLast(); }
public void TestVisibilityAnalysis() { var patternTests = new String[][] { new [] { "A", "[]" }, new [] { "A B", "[B=[A]]" }, new [] { "A B*", "[B=[A]]" }, new [] { "A B B", "[B=[A]]" }, new [] { "A B A", "[A=[B], B=[A]]" }, new [] { "A B+ C", "[B=[A], C=[A, B]]" }, new [] { "(A B)+ C", "[B=[A], C=[A, B]]" }, new [] { "D (A B)+ (G H)? C", "[A=[D], B=[A, D], C=[A, B, D, G, H], G=[A, B, D], H=[A, B, D, G]]" }, new [] { "A B | A C", "[B=[A], C=[A]]" }, new [] { "(A B*) | (A+ C)", "[B=[A], C=[A]]" }, new [] { "A (B | C) D", "[B=[A], C=[A], D=[A, B, C]]" }, new [] { "(((A))) (((B))) (( C | (D E)))", "[B=[A], C=[A, B], D=[A, B], E=[A, B, D]]" }, new [] { "(A | B) C", "[C=[A, B]]" }, new [] { "(A | B) (C | A)", "[A=[B], C=[A, B]]" }, }; for (int i = 0; i < patternTests.Length; i++) { String pattern = patternTests[i][0]; String expected = patternTests[i][1]; String expression = "select * from MyEvent.win:keepall() match_recognize (" + " partition by string measures A.string as a_string pattern ( " + pattern + ") define A as (A.value = 1) )"; EPLTreeWalkerListener walker = SupportParserHelper.ParseAndWalkEPL(expression); StatementSpecRaw raw = walker.GetStatementSpec(); RowRegexExprNode parent = raw.MatchRecognizeSpec.Pattern; IDictionary <String, ISet <String> > visibility = EventRowRegexHelper.DetermineVisibility(parent); // sort, for comparing var visibilitySorted = new LinkedHashMap <String, IList <String> >(); var tagsSorted = new List <String>(visibility.Keys); tagsSorted.Sort(); foreach (string tag in tagsSorted) { var sorted = new List <String>(visibility.Get(tag)); sorted.Sort(); visibilitySorted.Put(tag, sorted); } Assert.AreEqual(expected, visibilitySorted.Render(), "Failed in :" + pattern); } }
private static RowRegexExprNodeAlteration ExpandPermute(RowRegexExprNodePermute permute) { var e = PermutationEnumerator.Create(permute.ChildNodes.Count); var parent = new RowRegexExprNodeAlteration(); foreach (int[] indexes in e) { var concat = new RowRegexExprNodeConcatenation(); parent.AddChildNode(concat); for (var i = 0; i < indexes.Length; i++) { RowRegexExprNode toCopy = permute.ChildNodes[indexes[i]]; var copy = CheckedCopy(toCopy); concat.AddChildNode(copy); } } return(parent); }
public virtual void ReplaceChildNode(RowRegexExprNode nodeToReplace, IList <RowRegexExprNode> replacementNodes) { var newChildNodes = new List <RowRegexExprNode>(ChildNodes.Count - 1 + replacementNodes.Count); foreach (RowRegexExprNode node in ChildNodes) { if (node != nodeToReplace) { newChildNodes.Add(node); } else { newChildNodes.AddRange(replacementNodes); } } ChildNodes = newChildNodes; }
public void TestVariableAnalysis() { var patternTests = new String[][] { new[] { "A", "[A]", "[]" }, new[] { "A B", "[A, B]", "[]" }, new[] { "A B*", "[A]", "[B]" }, new[] { "A B B", "[A]", "[B]" }, new[] { "A B A", "[B]", "[A]" }, new[] { "A B+ C", "[A, C]", "[B]" }, new[] { "A B?", "[A, B]", "[]" }, new[] { "(A B)* C", "[C]", "[A, B]" }, new[] { "D (A B)+ (G H)? C", "[D, G, H, C]", "[A, B]" }, new[] { "A B | A C", "[A, B, C]", "[]" }, new[] { "(A B*) | (A+ C)", "[C]", "[B, A]" }, new[] { "(A | B) | (C | A)", "[A, B, C]", "[]" }, }; for (int i = 0; i < patternTests.Length; i++) { String pattern = patternTests[i][0]; String expression = "select * from MyEvent.win:keepall() match_recognize (" + " partition by TheString measures A.TheString as a_string pattern ( " + pattern + ") define A as (A.value = 1) )"; EPLTreeWalkerListener walker = SupportParserHelper.ParseAndWalkEPL(expression); StatementSpecRaw raw = walker.GetStatementSpec(); RowRegexExprNode parent = raw.MatchRecognizeSpec.Pattern; var singles = new FIFOHashSet <String>(); var multiples = new FIFOHashSet <String>(); EventRowRegexHelper.RecursiveInspectVariables(parent, false, singles, multiples); String @out = "Failed in :" + pattern + " result is : single " + singles.Render() + " multiple " + multiples.Render(); Assert.AreEqual(patternTests[i][1], singles.Render(), @out); Assert.AreEqual(patternTests[i][2], multiples.Render(), @out); } }
public void Visit(RowRegexExprNode node, RowRegexExprNode optionalParent, int level) { if (node is RowRegexExprNodeAtom) { var atom = (RowRegexExprNodeAtom)node; if (atom.OptionalRepeat != null) { if (_atoms == null) { _atoms = new List <Pair <RowRegexExprNodeAtom, RowRegexExprNode> >(); } _atoms.Add(new Pair <RowRegexExprNodeAtom, RowRegexExprNode>(atom, optionalParent)); } } if (node is RowRegexExprNodeNested) { var nested = (RowRegexExprNodeNested)node; if (nested.OptionalRepeat != null) { if (_nesteds == null) { _nesteds = new List <RowRegexNestedDesc>(); } _nesteds.Add(new RowRegexNestedDesc(nested, optionalParent, level)); } } if (node is RowRegexExprNodePermute) { var permute = (RowRegexExprNodePermute)node; if (_permutes == null) { _permutes = new List <RowRegexPermuteDesc>(); } _permutes.Add(new RowRegexPermuteDesc(permute, optionalParent, level)); } }
public static RowRegexExprNode Expand(RowRegexExprNode pattern) { var visitor = new RowRegexExprNodeVisitorRepeat(); pattern.Accept(visitor); var newParentNode = pattern; // expand permutes var permutes = visitor.Permutes; permutes.SortInPlace((o1, o2) => { if (o1.Level > o2.Level) { return(-1); } return(o1.Level == o2.Level ? 0 : 1); }); foreach (var permute in permutes) { var alteration = ExpandPermute(permute.Permute); RowRegexExprNode optionalNewParent = Replace(permute.OptionalParent, permute.Permute, Collections.SingletonList <RowRegexExprNode>(alteration)); if (optionalNewParent != null) { newParentNode = optionalNewParent; } } // expand atoms var atomPairs = visitor.Atoms; foreach (var pair in atomPairs) { var atom = pair.First; var expandedRepeat = ExpandRepeat(atom, atom.OptionalRepeat, atom.NFAType, ATOM_HANDLER); var optionalNewParent = Replace(pair.Second, pair.First, expandedRepeat); if (optionalNewParent != null) { newParentNode = optionalNewParent; } } // expand nested var nestedPairs = visitor.Nesteds; nestedPairs.SortInPlace((o1, o2) => { if (o1.Level > o2.Level) { return(-1); } return(o1.Level == o2.Level ? 0 : 1); }); foreach (var pair in nestedPairs) { var nested = pair.Nested; var expandedRepeat = ExpandRepeat(nested, nested.OptionalRepeat, nested.NFAType, NESTED_HANDLER); var optionalNewParent = Replace(pair.OptionalParent, pair.Nested, expandedRepeat); if (optionalNewParent != null) { newParentNode = optionalNewParent; } } return(newParentNode); }
private static IList <RowRegexExprNode> ExpandRepeat(RowRegexExprNode node, RowRegexExprRepeatDesc repeat, RegexNFATypeEnum type, RowRegexExprNodeCopier copier) { var evaluateParams = new EvaluateParams(null, true, null); // handle single-bounds (no ranges) IList <RowRegexExprNode> repeated = new List <RowRegexExprNode>(); if (repeat.Single != null) { ValidateExpression(repeat.Single); int numRepeated = repeat.Single.ExprEvaluator.Evaluate(evaluateParams).AsInt(); ValidateRange(numRepeated, 1, int.MaxValue); for (var i = 0; i < numRepeated; i++) { var copy = copier.Copy(node, type); repeated.Add(copy); } return(repeated); } // evaluate bounds int?lower = null; int?upper = null; if (repeat.Lower != null) { ValidateExpression(repeat.Lower); lower = (int?)repeat.Lower.ExprEvaluator.Evaluate(evaluateParams); } if (repeat.Upper != null) { ValidateExpression(repeat.Upper); upper = (int?)repeat.Upper.ExprEvaluator.Evaluate(evaluateParams); } // handle range if (lower != null && upper != null) { ValidateRange(lower.Value, 1, int.MaxValue); ValidateRange(upper.Value, 1, int.MaxValue); ValidateRange(lower.Value, 1, upper.Value); for (var i = 0; i < lower; i++) { var copy = copier.Copy(node, type); repeated.Add(copy); } for (int i = lower.Value; i < upper; i++) { // make type optional var newType = type; if (type == RegexNFATypeEnum.SINGLE) { newType = RegexNFATypeEnum.ONE_OPTIONAL; } else if (type == RegexNFATypeEnum.ONE_TO_MANY) { newType = RegexNFATypeEnum.ZERO_TO_MANY; } else if (type == RegexNFATypeEnum.ONE_TO_MANY_RELUCTANT) { newType = RegexNFATypeEnum.ZERO_TO_MANY_RELUCTANT; } var copy = copier.Copy(node, newType); repeated.Add(copy); } return(repeated); } // handle lower-bounds only if (upper == null) { ValidateRange(lower.Value, 1, int.MaxValue); for (var i = 0; i < lower; i++) { var copyInner = copier.Copy(node, type); repeated.Add(copyInner); } // make type optional var newType = type; if (type == RegexNFATypeEnum.SINGLE) { newType = RegexNFATypeEnum.ZERO_TO_MANY; } else if (type == RegexNFATypeEnum.ONE_OPTIONAL) { newType = RegexNFATypeEnum.ZERO_TO_MANY; } else if (type == RegexNFATypeEnum.ONE_OPTIONAL_RELUCTANT) { newType = RegexNFATypeEnum.ZERO_TO_MANY_RELUCTANT; } else if (type == RegexNFATypeEnum.ONE_TO_MANY) { newType = RegexNFATypeEnum.ZERO_TO_MANY; } else if (type == RegexNFATypeEnum.ONE_TO_MANY_RELUCTANT) { newType = RegexNFATypeEnum.ZERO_TO_MANY_RELUCTANT; } var copy = copier.Copy(node, newType); repeated.Add(copy); return(repeated); } // handle upper-bounds only ValidateRange(upper.Value, 1, int.MaxValue); for (var i = 0; i < upper; i++) { // make type optional var newType = type; if (type == RegexNFATypeEnum.SINGLE) { newType = RegexNFATypeEnum.ONE_OPTIONAL; } else if (type == RegexNFATypeEnum.ONE_TO_MANY) { newType = RegexNFATypeEnum.ZERO_TO_MANY; } else if (type == RegexNFATypeEnum.ONE_TO_MANY_RELUCTANT) { newType = RegexNFATypeEnum.ZERO_TO_MANY_RELUCTANT; } var copy = copier.Copy(node, newType); repeated.Add(copy); } return(repeated); }
/// <summary> /// Ctor. /// </summary> /// <param name="viewChain">views</param> /// <param name="matchRecognizeSpec">specification</param> /// <param name="agentInstanceContext">The agent instance context.</param> /// <param name="isUnbound">true for unbound stream</param> /// <param name="annotations">annotations</param> /// <exception cref="ExprValidationException"> /// Variable ' + defineItem.Identifier + ' has already been defined /// or /// An aggregate function may not appear in a DEFINE clause /// or /// Failed to validate condition expression for variable ' + defineItem.Identifier + ': + ex.Message /// or /// Aggregation functions in the measure-clause must only refer to properties of exactly one group variable returning multiple events /// or /// Aggregation functions in the measure-clause must refer to one or more properties of exactly one group variable returning multiple events /// or /// The measures clause requires that each expression utilizes the AS keyword to assign a column name /// </exception> /// <throws>ExprValidationException if validation fails</throws> public EventRowRegexNFAViewFactory( ViewFactoryChain viewChain, MatchRecognizeSpec matchRecognizeSpec, AgentInstanceContext agentInstanceContext, bool isUnbound, Attribute[] annotations, ConfigurationEngineDefaults.MatchRecognizeConfig matchRecognizeConfig) { var parentViewType = viewChain.EventType; _matchRecognizeSpec = matchRecognizeSpec; _isUnbound = isUnbound; _isIterateOnly = HintEnum.ITERATE_ONLY.GetHint(annotations) != null; _matchRecognizeConfig = matchRecognizeConfig; var statementContext = agentInstanceContext.StatementContext; // Expand repeats and permutations _expandedPatternNode = RegexPatternExpandUtil.Expand(matchRecognizeSpec.Pattern); // Determine single-row and multiple-row variables _variablesSingle = new LinkedHashSet <string>(); ISet <string> variablesMultiple = new LinkedHashSet <string>(); EventRowRegexHelper.RecursiveInspectVariables(_expandedPatternNode, false, _variablesSingle, variablesMultiple); // each variable gets associated with a stream number (multiple-row variables as well to hold the current event for the expression). var streamNum = 0; _variableStreams = new LinkedHashMap <string, Pair <int, bool> >(); foreach (var variableSingle in _variablesSingle) { _variableStreams.Put(variableSingle, new Pair <int, bool>(streamNum, false)); streamNum++; } foreach (var variableMultiple in variablesMultiple) { _variableStreams.Put(variableMultiple, new Pair <int, bool>(streamNum, true)); streamNum++; } // mapping of stream to variable _streamVariables = new SortedDictionary <int, string>(); foreach (var entry in _variableStreams) { _streamVariables.Put(entry.Value.First, entry.Key); } // determine visibility rules var visibility = EventRowRegexHelper.DetermineVisibility(_expandedPatternNode); // assemble all single-row variables for expression validation var allStreamNames = new string[_variableStreams.Count]; var allTypes = new EventType[_variableStreams.Count]; streamNum = 0; foreach (var variableSingle in _variablesSingle) { allStreamNames[streamNum] = variableSingle; allTypes[streamNum] = parentViewType; streamNum++; } foreach (var variableMultiple in variablesMultiple) { allStreamNames[streamNum] = variableMultiple; allTypes[streamNum] = parentViewType; streamNum++; } // determine type service for use with DEFINE // validate each DEFINE clause expression ISet <string> definedVariables = new HashSet <string>(); IList <ExprAggregateNode> aggregateNodes = new List <ExprAggregateNode>(); var exprEvaluatorContext = new ExprEvaluatorContextStatement(statementContext, false); _isExprRequiresMultimatchState = new bool[_variableStreams.Count]; for (var defineIndex = 0; defineIndex < matchRecognizeSpec.Defines.Count; defineIndex++) { var defineItem = matchRecognizeSpec.Defines[defineIndex]; if (definedVariables.Contains(defineItem.Identifier)) { throw new ExprValidationException("Variable '" + defineItem.Identifier + "' has already been defined"); } definedVariables.Add(defineItem.Identifier); // stream-type visibilities handled here var typeServiceDefines = EventRowRegexNFAViewFactoryHelper.BuildDefineStreamTypeServiceDefine(statementContext, _variableStreams, defineItem, visibility, parentViewType); var exprNodeResult = HandlePreviousFunctions(defineItem.Expression); var validationContext = new ExprValidationContext( typeServiceDefines, statementContext.EngineImportService, statementContext.StatementExtensionServicesContext, null, statementContext.SchedulingService, statementContext.VariableService, statementContext.TableService, exprEvaluatorContext, statementContext.EventAdapterService, statementContext.StatementName, statementContext.StatementId, statementContext.Annotations, statementContext.ContextDescriptor, statementContext.ScriptingService, true, false, true, false, null, false); ExprNode validated; try { // validate validated = ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.MATCHRECOGDEFINE, exprNodeResult, validationContext); // check aggregates defineItem.Expression = validated; ExprAggregateNodeUtil.GetAggregatesBottomUp(validated, aggregateNodes); if (!aggregateNodes.IsEmpty()) { throw new ExprValidationException("An aggregate function may not appear in a DEFINE clause"); } } catch (ExprValidationException ex) { throw new ExprValidationException("Failed to validate condition expression for variable '" + defineItem.Identifier + "': " + ex.Message, ex); } // determine access to event properties from multi-matches var visitor = new ExprNodeStreamRequiredVisitor(); validated.Accept(visitor); var streamsRequired = visitor.StreamsRequired; foreach (var streamRequired in streamsRequired) { if (streamRequired >= _variableStreams.Count) { var streamNumIdent = _variableStreams.Get(defineItem.Identifier).First; _isExprRequiresMultimatchState[streamNumIdent] = true; break; } } } _isDefineAsksMultimatches = CollectionUtil.IsAnySet(_isExprRequiresMultimatchState); _defineMultimatchEventBean = _isDefineAsksMultimatches ? EventRowRegexNFAViewFactoryHelper.GetDefineMultimatchBean(statementContext, _variableStreams, parentViewType) : null; // assign "prev" node indexes // Since an expression such as "prior(2, price), prior(8, price)" translates into {2, 8} the relative index is {0, 1}. // Map the expression-supplied index to a relative index var countPrev = 0; foreach (var entry in _callbacksPerIndex) { foreach (var callback in entry.Value) { callback.AssignedIndex = countPrev; } countPrev++; } // determine type service for use with MEASURE IDictionary <string, object> measureTypeDef = new LinkedHashMap <string, object>(); foreach (var variableSingle in _variablesSingle) { measureTypeDef.Put(variableSingle, parentViewType); } foreach (var variableMultiple in variablesMultiple) { measureTypeDef.Put(variableMultiple, new EventType[] { parentViewType }); } var outputEventTypeName = statementContext.StatementId + "_rowrecog"; _compositeEventType = (ObjectArrayEventType)statementContext.EventAdapterService.CreateAnonymousObjectArrayType(outputEventTypeName, measureTypeDef); StreamTypeService typeServiceMeasure = new StreamTypeServiceImpl(_compositeEventType, "MATCH_RECOGNIZE", true, statementContext.EngineURI); // find MEASURE clause aggregations var measureReferencesMultivar = false; IList <ExprAggregateNode> measureAggregateExprNodes = new List <ExprAggregateNode>(); foreach (var measureItem in matchRecognizeSpec.Measures) { ExprAggregateNodeUtil.GetAggregatesBottomUp(measureItem.Expr, measureAggregateExprNodes); } if (!measureAggregateExprNodes.IsEmpty()) { var isIStreamOnly = new bool[allStreamNames.Length]; CompatExtensions.Fill(isIStreamOnly, true); var typeServiceAggregateMeasure = new StreamTypeServiceImpl(allTypes, allStreamNames, isIStreamOnly, statementContext.EngineURI, false); var measureExprAggNodesPerStream = new Dictionary <int, IList <ExprAggregateNode> >(); foreach (var aggregateNode in measureAggregateExprNodes) { // validate absence of group-by aggregateNode.ValidatePositionals(); if (aggregateNode.OptionalLocalGroupBy != null) { throw new ExprValidationException("Match-recognize does not allow aggregation functions to specify a group-by"); } // validate node and params var count = 0; var visitor = new ExprNodeIdentifierVisitor(true); var validationContext = new ExprValidationContext( typeServiceAggregateMeasure, statementContext.EngineImportService, statementContext.StatementExtensionServicesContext, null, statementContext.SchedulingService, statementContext.VariableService, statementContext.TableService, exprEvaluatorContext, statementContext.EventAdapterService, statementContext.StatementName, statementContext.StatementId, statementContext.Annotations, statementContext.ContextDescriptor, statementContext.ScriptingService, false, false, true, false, null, false); for (int ii = 0; ii < aggregateNode.ChildNodes.Count; ii++) { var child = aggregateNode.ChildNodes[ii]; var validated = ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.MATCHRECOGMEASURE, child, validationContext); validated.Accept(visitor); aggregateNode.SetChildNode(count++, new ExprNodeValidated(validated)); } validationContext = new ExprValidationContext( typeServiceMeasure, statementContext.EngineImportService, statementContext.StatementExtensionServicesContext, null, statementContext.SchedulingService, statementContext.VariableService, statementContext.TableService, exprEvaluatorContext, statementContext.EventAdapterService, statementContext.StatementName, statementContext.StatementId, statementContext.Annotations, statementContext.ContextDescriptor, statementContext.ScriptingService, false, false, true, false, null, false); aggregateNode.Validate(validationContext); // verify properties used within the aggregation var aggregatedStreams = new HashSet <int>(); foreach (var pair in visitor.ExprProperties) { aggregatedStreams.Add(pair.First); } int?multipleVarStream = null; foreach (int streamNumAggregated in aggregatedStreams) { var variable = _streamVariables.Get(streamNumAggregated); if (variablesMultiple.Contains(variable)) { measureReferencesMultivar = true; if (multipleVarStream == null) { multipleVarStream = streamNumAggregated; continue; } throw new ExprValidationException("Aggregation functions in the measure-clause must only refer to properties of exactly one group variable returning multiple events"); } } if (multipleVarStream == null) { throw new ExprValidationException("Aggregation functions in the measure-clause must refer to one or more properties of exactly one group variable returning multiple events"); } var aggNodesForStream = measureExprAggNodesPerStream.Get(multipleVarStream.Value); if (aggNodesForStream == null) { aggNodesForStream = new List <ExprAggregateNode>(); measureExprAggNodesPerStream.Put(multipleVarStream.Value, aggNodesForStream); } aggNodesForStream.Add(aggregateNode); } var factoryDesc = AggregationServiceFactoryFactory.GetServiceMatchRecognize(_streamVariables.Count, measureExprAggNodesPerStream, typeServiceAggregateMeasure.EventTypes); _aggregationService = factoryDesc.AggregationServiceFactory.MakeService(agentInstanceContext); _aggregationExpressions = factoryDesc.Expressions; } else { _aggregationService = null; _aggregationExpressions = Collections.GetEmptyList <AggregationServiceAggExpressionDesc>(); } // validate each MEASURE clause expression IDictionary <string, object> rowTypeDef = new LinkedHashMap <string, object>(); var streamRefVisitor = new ExprNodeStreamUseCollectVisitor(); foreach (var measureItem in matchRecognizeSpec.Measures) { if (measureItem.Name == null) { throw new ExprValidationException("The measures clause requires that each expression utilizes the AS keyword to assign a column name"); } var validated = ValidateMeasureClause(measureItem.Expr, typeServiceMeasure, variablesMultiple, _variablesSingle, statementContext); measureItem.Expr = validated; rowTypeDef.Put(measureItem.Name, validated.ExprEvaluator.ReturnType); validated.Accept(streamRefVisitor); } // Determine if any of the multi-var streams are referenced in the measures (non-aggregated only) foreach (var @ref in streamRefVisitor.Referenced) { var rootPropName = @ref.RootPropertyNameIfAny; if (rootPropName != null) { if (variablesMultiple.Contains(rootPropName)) { measureReferencesMultivar = true; break; } } var streamRequired = @ref.StreamReferencedIfAny; if (streamRequired != null) { var streamVariable = _streamVariables.Get(streamRequired.Value); if (streamVariable != null) { var def = _variableStreams.Get(streamVariable); if (def != null && def.Second) { measureReferencesMultivar = true; break; } } } } _isCollectMultimatches = measureReferencesMultivar || _isDefineAsksMultimatches; // create rowevent type var rowEventTypeName = statementContext.StatementId + "_rowrecogrow"; _rowEventType = statementContext.EventAdapterService.CreateAnonymousMapType(rowEventTypeName, rowTypeDef, true); // validate partition-by expressions, if any if (!matchRecognizeSpec.PartitionByExpressions.IsEmpty()) { var typeServicePartition = new StreamTypeServiceImpl(parentViewType, "MATCH_RECOGNIZE_PARTITION", true, statementContext.EngineURI); var validated = new List <ExprNode>(); var validationContext = new ExprValidationContext( typeServicePartition, statementContext.EngineImportService, statementContext.StatementExtensionServicesContext, null, statementContext.SchedulingService, statementContext.VariableService, statementContext.TableService, exprEvaluatorContext, statementContext.EventAdapterService, statementContext.StatementName, statementContext.StatementId, statementContext.Annotations, statementContext.ContextDescriptor, statementContext.ScriptingService, false, false, true, false, null, false); foreach (var partitionExpr in matchRecognizeSpec.PartitionByExpressions) { validated.Add(ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.MATCHRECOGPARTITION, partitionExpr, validationContext)); } matchRecognizeSpec.PartitionByExpressions = validated; } // validate interval if present if (matchRecognizeSpec.Interval != null) { var validationContext = new ExprValidationContext( new StreamTypeServiceImpl(statementContext.EngineURI, false), statementContext.EngineImportService, statementContext.StatementExtensionServicesContext, null, statementContext.SchedulingService, statementContext.VariableService, statementContext.TableService, exprEvaluatorContext, statementContext.EventAdapterService, statementContext.StatementName, statementContext.StatementId, statementContext.Annotations, statementContext.ContextDescriptor, statementContext.ScriptingService, false, false, true, false, null, false); matchRecognizeSpec.Interval.Validate(validationContext); } }
public RowRegexPermuteDesc(RowRegexExprNodePermute permute, RowRegexExprNode optionalParent, int level) { Permute = permute; OptionalParent = optionalParent; Level = level; }
public RowRegexNestedDesc(RowRegexExprNodeNested nested, RowRegexExprNode optionalParent, int level) { Nested = nested; OptionalParent = optionalParent; Level = level; }
private static RegexNFAStrand RecursiveBuildStatesInternal( RowRegexExprNode node, IDictionary <String, ExprNode> variableDefinitions, IDictionary <String, Pair <int, Boolean> > variableStreams, Stack <int> nodeNumStack, bool[] exprRequiresMultimatchState) { if (node is RowRegexExprNodeAlteration) { var nodeNum = 0; var cumulativeStartStates = new List <RegexNFAStateBase>(); var cumulativeStates = new List <RegexNFAStateBase>(); var cumulativeEndStates = new List <RegexNFAStateBase>(); var isPassthrough = false; foreach (var child in node.ChildNodes) { nodeNumStack.Push(nodeNum); var strand = RecursiveBuildStatesInternal(child, variableDefinitions, variableStreams, nodeNumStack, exprRequiresMultimatchState); nodeNumStack.Pop(); cumulativeStartStates.AddAll(strand.StartStates); cumulativeStates.AddAll(strand.AllStates); cumulativeEndStates.AddAll(strand.EndStates); if (strand.IsPassthrough) { isPassthrough = true; } nodeNum++; } return(new RegexNFAStrand(cumulativeStartStates, cumulativeEndStates, cumulativeStates, isPassthrough)); } else if (node is RowRegexExprNodeConcatenation) { var nodeNum = 0; var isPassthrough = true; var cumulativeStates = new List <RegexNFAStateBase>(); var strands = new RegexNFAStrand[node.ChildNodes.Count]; foreach (var child in node.ChildNodes) { nodeNumStack.Push(nodeNum); strands[nodeNum] = RecursiveBuildStatesInternal(child, variableDefinitions, variableStreams, nodeNumStack, exprRequiresMultimatchState); nodeNumStack.Pop(); cumulativeStates.AddAll(strands[nodeNum].AllStates); if (!strands[nodeNum].IsPassthrough) { isPassthrough = false; } nodeNum++; } // determine start states: all states until the first non-passthrough start state var startStates = new List <RegexNFAStateBase>(); for (var i = 0; i < strands.Length; i++) { startStates.AddAll(strands[i].StartStates); if (!strands[i].IsPassthrough) { break; } } // determine end states: all states from the back until the last non-passthrough end state var endStates = new List <RegexNFAStateBase>(); for (var i = strands.Length - 1; i >= 0; i--) { endStates.AddAll(strands[i].EndStates); if (!strands[i].IsPassthrough) { break; } } // hook up the end state of each strand with the start states of each next strand for (var i = strands.Length - 1; i >= 1; i--) { var current = strands[i]; for (var j = i - 1; j >= 0; j--) { var prior = strands[j]; foreach (var endState in prior.EndStates) { foreach (var startState in current.StartStates) { endState.AddState(startState); } } if (!prior.IsPassthrough) { break; } } } return(new RegexNFAStrand(startStates, endStates, cumulativeStates, isPassthrough)); } else if (node is RowRegexExprNodeNested) { var nested = (RowRegexExprNodeNested)node; nodeNumStack.Push(0); var strand = RecursiveBuildStatesInternal(node.ChildNodes[0], variableDefinitions, variableStreams, nodeNumStack, exprRequiresMultimatchState); nodeNumStack.Pop(); var isPassthrough = strand.IsPassthrough || nested.NFAType.IsOptional(); // if this is a repeating node then pipe back each end state to each begin state if (nested.NFAType.IsMultipleMatches()) { foreach (var endstate in strand.EndStates) { foreach (var startstate in strand.StartStates) { if (!endstate.NextStates.Contains(startstate)) { endstate.NextStates.Add(startstate); } } } } return(new RegexNFAStrand(strand.StartStates, strand.EndStates, strand.AllStates, isPassthrough)); } else { var atom = (RowRegexExprNodeAtom)node; // assign stream number for single-variables for most direct expression eval; multiple-variable gets -1 var streamNum = variableStreams.Get(atom.Tag).First; var multiple = variableStreams.Get(atom.Tag).Second; var expressionDef = variableDefinitions.Get(atom.Tag); var exprRequiresMultimatch = exprRequiresMultimatchState[streamNum]; RegexNFAStateBase nextState; if ((atom.NFAType == RegexNFATypeEnum.ZERO_TO_MANY) || (atom.NFAType == RegexNFATypeEnum.ZERO_TO_MANY_RELUCTANT)) { nextState = new RegexNFAStateZeroToMany(ToString(nodeNumStack), atom.Tag, streamNum, multiple, atom.NFAType.IsGreedy(), expressionDef, exprRequiresMultimatch); } else if ((atom.NFAType == RegexNFATypeEnum.ONE_TO_MANY) || (atom.NFAType == RegexNFATypeEnum.ONE_TO_MANY_RELUCTANT)) { nextState = new RegexNFAStateOneToMany(ToString(nodeNumStack), atom.Tag, streamNum, multiple, atom.NFAType.IsGreedy(), expressionDef, exprRequiresMultimatch); } else if ((atom.NFAType == RegexNFATypeEnum.ONE_OPTIONAL) || (atom.NFAType == RegexNFATypeEnum.ONE_OPTIONAL_RELUCTANT)) { nextState = new RegexNFAStateOneOptional(ToString(nodeNumStack), atom.Tag, streamNum, multiple, atom.NFAType.IsGreedy(), expressionDef, exprRequiresMultimatch); } else if (expressionDef == null) { nextState = new RegexNFAStateAnyOne(ToString(nodeNumStack), atom.Tag, streamNum, multiple); } else { nextState = new RegexNFAStateFilter(ToString(nodeNumStack), atom.Tag, streamNum, multiple, expressionDef, exprRequiresMultimatch); } return(new RegexNFAStrand( Collections.SingletonList(nextState), Collections.SingletonList(nextState), Collections.SingletonList(nextState), atom.NFAType.IsOptional())); } }
/// <summary> /// Adds a child node. /// </summary> /// <param name="childNode">is the child evaluation tree node to add</param> public void AddChildNode(RowRegexExprNode childNode) { ChildNodes.Add(childNode); }