/// <summary>
/// Inspect variables recursively.
/// </summary>
/// <param name="parent">parent regex expression node</param>
/// <param name="isMultiple">if the variable in the stack is multiple of single</param>
/// <param name="variablesSingle">single variables list</param>
/// <param name="variablesMultiple">group variables list</param>
public static void RecursiveInspectVariables(RowRegexExprNode parent, bool isMultiple, ISet <String> variablesSingle, ISet <String> variablesMultiple)
{
if (parent is RowRegexExprNodeNested)
{
var nested = (RowRegexExprNodeNested)parent;
foreach (var child in parent.ChildNodes)
{
RecursiveInspectVariables(child, nested.NFAType.IsMultipleMatches() || isMultiple, variablesSingle, variablesMultiple);
}
}
else if (parent is RowRegexExprNodeAlteration)
{
foreach (var childAlteration in parent.ChildNodes)
{
var singles = new LinkedHashSet <String>();
var multiples = new LinkedHashSet <String>();
RecursiveInspectVariables(childAlteration, isMultiple, singles, multiples);
variablesMultiple.AddAll(multiples);
variablesSingle.AddAll(singles);
}
variablesSingle.RemoveAll(variablesMultiple);
}
else if (parent is RowRegexExprNodeAtom)
{
var atom = (RowRegexExprNodeAtom)parent;
var name = atom.Tag;
if (variablesMultiple.Contains(name))
{
return;
}
if (variablesSingle.Contains(name))
{
variablesSingle.Remove(name);
variablesMultiple.Add(name);
return;
}
if (atom.NFAType.IsMultipleMatches())
{
variablesMultiple.Add(name);
return;
}
if (isMultiple)
{
variablesMultiple.Add(name);
}
else
{
variablesSingle.Add(name);
}
}
else
{
foreach (var child in parent.ChildNodes)
{
RecursiveInspectVariables(child, isMultiple, variablesSingle, variablesMultiple);
}
}
}
private static RowRegexExprNode Replace(RowRegexExprNode optionalParent, RowRegexExprNode originalNode, IList <RowRegexExprNode> expandedRepeat)
{
if (optionalParent == null)
{
var newParentNode = new RowRegexExprNodeConcatenation();
newParentNode.ChildNodes.AddAll(expandedRepeat);
return(newParentNode);
}
// for nested nodes, use a concatenation instead
if (optionalParent is RowRegexExprNodeNested ||
optionalParent is RowRegexExprNodeAlteration)
{
var concatenation = new RowRegexExprNodeConcatenation();
concatenation.ChildNodes.AddAll(expandedRepeat);
optionalParent.ReplaceChildNode(originalNode, Collections.SingletonList <RowRegexExprNode>(concatenation));
}
// concatenations are simply changed
else
{
optionalParent.ReplaceChildNode(originalNode, expandedRepeat);
}
return(null);
}
/// <summary>
/// Build a list of start states from the parent node.
/// </summary>
/// <param name="parent">to build start state for</param>
/// <param name="variableDefinitions">each variable and its expressions</param>
/// <param name="variableStreams">variable name and its stream number</param>
/// <param name="exprRequiresMultimatchState">State of the expr requires multimatch.</param>
/// <returns>strand of regex state nodes</returns>
public static RegexNFAStrandResult RecursiveBuildStartStates(
RowRegexExprNode parent,
IDictionary <String, ExprNode> variableDefinitions,
IDictionary <String, Pair <int, bool> > variableStreams,
bool[] exprRequiresMultimatchState)
{
var nodeNumStack = new Stack <int>();
var strand = RecursiveBuildStatesInternal(
parent,
variableDefinitions,
variableStreams,
nodeNumStack,
exprRequiresMultimatchState);
// add end state
var end = new RegexNFAStateEnd();
foreach (var endStates in strand.EndStates)
{
endStates.AddState(end);
}
// assign node num as a counter
var nodeNumberFlat = 0;
foreach (var theBase in strand.AllStates)
{
theBase.NodeNumFlat = nodeNumberFlat++;
}
return(new RegexNFAStrandResult(new List <RegexNFAState>(strand.StartStates), strand.AllStates));
}
public static IDictionary <String, ISet <String> > DetermineVisibility(RowRegexExprNode pattern)
{
IDictionary <String, ISet <String> > map = new Dictionary <String, ISet <String> >();
var path = new ArrayDeque <RowRegexExprNode>();
RecursiveFindPatternAtoms(pattern, path, map);
return(map);
}
public RowRegexExprNode Copy(
RowRegexExprNode nodeToCopy,
RegexNFATypeEnum newType)
{
var atom = (RowRegexExprNodeAtom)nodeToCopy;
return(new RowRegexExprNodeAtom(atom.Tag, newType, null));
}
public virtual void AcceptChildnodes(RowRegexExprNodeVisitor visitor, RowRegexExprNode parent, int level)
{
visitor.Visit(this, parent, level);
foreach (RowRegexExprNode childNode in ChildNodes)
{
childNode.AcceptChildnodes(visitor, this, level + 1);
}
}
private static RowRegexExprNode CheckedCopy(RowRegexExprNode inner)
{
try {
return((RowRegexExprNode)SerializableObjectCopier.Copy(inner));
}
catch (Exception e) {
throw new EPException("Failed to repeat nested match-recognize: " + e.Message, e);
}
}
public RowRegexExprNode Copy(RowRegexExprNode nodeToCopy, RegexNFATypeEnum newType)
{
var nested = (RowRegexExprNodeNested)nodeToCopy;
var nestedCopy = new RowRegexExprNodeNested(newType, null);
foreach (var inner in nested.ChildNodes)
{
var innerCopy = CheckedCopy(inner);
nestedCopy.AddChildNode(innerCopy);
}
return(nestedCopy);
}
private static void RecursiveCollectAtomsWExclude(RowRegexExprNode node, ISet <String> identifiers, String excludedTag)
{
if (node is RowRegexExprNodeAtom)
{
var atom = (RowRegexExprNodeAtom)node;
if (!excludedTag.Equals(atom.Tag))
{
identifiers.Add(atom.Tag);
}
}
foreach (var child in node.ChildNodes)
{
RecursiveCollectAtomsWExclude(child, identifiers, excludedTag);
}
}
private static void RecursiveFindPatternAtoms(RowRegexExprNode parent, ArrayDeque <RowRegexExprNode> path, IDictionary <String, ISet <String> > map)
{
path.Add(parent);
foreach (var child in parent.ChildNodes)
{
if (child is RowRegexExprNodeAtom)
{
HandleAtom((RowRegexExprNodeAtom)child, path, map);
}
else
{
RecursiveFindPatternAtoms(child, path, map);
}
}
path.RemoveLast();
}
public void TestVisibilityAnalysis()
{
var patternTests = new String[][]
{
new [] { "A", "[]" },
new [] { "A B", "[B=[A]]" },
new [] { "A B*", "[B=[A]]" },
new [] { "A B B", "[B=[A]]" },
new [] { "A B A", "[A=[B], B=[A]]" },
new [] { "A B+ C", "[B=[A], C=[A, B]]" },
new [] { "(A B)+ C", "[B=[A], C=[A, B]]" },
new [] { "D (A B)+ (G H)? C", "[A=[D], B=[A, D], C=[A, B, D, G, H], G=[A, B, D], H=[A, B, D, G]]" },
new [] { "A B | A C", "[B=[A], C=[A]]" },
new [] { "(A B*) | (A+ C)", "[B=[A], C=[A]]" },
new [] { "A (B | C) D", "[B=[A], C=[A], D=[A, B, C]]" },
new [] { "(((A))) (((B))) (( C | (D E)))", "[B=[A], C=[A, B], D=[A, B], E=[A, B, D]]" },
new [] { "(A | B) C", "[C=[A, B]]" },
new [] { "(A | B) (C | A)", "[A=[B], C=[A, B]]" },
};
for (int i = 0; i < patternTests.Length; i++)
{
String pattern = patternTests[i][0];
String expected = patternTests[i][1];
String expression = "select * from MyEvent.win:keepall() match_recognize (" +
" partition by string measures A.string as a_string pattern ( " + pattern + ") define A as (A.value = 1) )";
EPLTreeWalkerListener walker = SupportParserHelper.ParseAndWalkEPL(expression);
StatementSpecRaw raw = walker.GetStatementSpec();
RowRegexExprNode parent = raw.MatchRecognizeSpec.Pattern;
IDictionary <String, ISet <String> > visibility = EventRowRegexHelper.DetermineVisibility(parent);
// sort, for comparing
var visibilitySorted = new LinkedHashMap <String, IList <String> >();
var tagsSorted = new List <String>(visibility.Keys);
tagsSorted.Sort();
foreach (string tag in tagsSorted)
{
var sorted = new List <String>(visibility.Get(tag));
sorted.Sort();
visibilitySorted.Put(tag, sorted);
}
Assert.AreEqual(expected, visibilitySorted.Render(), "Failed in :" + pattern);
}
}
private static RowRegexExprNodeAlteration ExpandPermute(RowRegexExprNodePermute permute)
{
var e = PermutationEnumerator.Create(permute.ChildNodes.Count);
var parent = new RowRegexExprNodeAlteration();
foreach (int[] indexes in e)
{
var concat = new RowRegexExprNodeConcatenation();
parent.AddChildNode(concat);
for (var i = 0; i < indexes.Length; i++)
{
RowRegexExprNode toCopy = permute.ChildNodes[indexes[i]];
var copy = CheckedCopy(toCopy);
concat.AddChildNode(copy);
}
}
return(parent);
}
public virtual void ReplaceChildNode(RowRegexExprNode nodeToReplace, IList <RowRegexExprNode> replacementNodes)
{
var newChildNodes = new List <RowRegexExprNode>(ChildNodes.Count - 1 + replacementNodes.Count);
foreach (RowRegexExprNode node in ChildNodes)
{
if (node != nodeToReplace)
{
newChildNodes.Add(node);
}
else
{
newChildNodes.AddRange(replacementNodes);
}
}
ChildNodes = newChildNodes;
}
public void TestVariableAnalysis()
{
var patternTests = new String[][] {
new[] { "A", "[A]", "[]" },
new[] { "A B", "[A, B]", "[]" },
new[] { "A B*", "[A]", "[B]" },
new[] { "A B B", "[A]", "[B]" },
new[] { "A B A", "[B]", "[A]" },
new[] { "A B+ C", "[A, C]", "[B]" },
new[] { "A B?", "[A, B]", "[]" },
new[] { "(A B)* C", "[C]", "[A, B]" },
new[] { "D (A B)+ (G H)? C", "[D, G, H, C]", "[A, B]" },
new[] { "A B | A C", "[A, B, C]", "[]" },
new[] { "(A B*) | (A+ C)", "[C]", "[B, A]" },
new[] { "(A | B) | (C | A)", "[A, B, C]", "[]" },
};
for (int i = 0; i < patternTests.Length; i++)
{
String pattern = patternTests[i][0];
String expression = "select * from MyEvent.win:keepall() match_recognize (" +
" partition by TheString measures A.TheString as a_string pattern ( " + pattern + ") define A as (A.value = 1) )";
EPLTreeWalkerListener walker = SupportParserHelper.ParseAndWalkEPL(expression);
StatementSpecRaw raw = walker.GetStatementSpec();
RowRegexExprNode parent = raw.MatchRecognizeSpec.Pattern;
var singles = new FIFOHashSet <String>();
var multiples = new FIFOHashSet <String>();
EventRowRegexHelper.RecursiveInspectVariables(parent, false, singles, multiples);
String @out = "Failed in :" + pattern +
" result is : single " + singles.Render() +
" multiple " + multiples.Render();
Assert.AreEqual(patternTests[i][1], singles.Render(), @out);
Assert.AreEqual(patternTests[i][2], multiples.Render(), @out);
}
}
public void Visit(RowRegexExprNode node, RowRegexExprNode optionalParent, int level)
{
if (node is RowRegexExprNodeAtom)
{
var atom = (RowRegexExprNodeAtom)node;
if (atom.OptionalRepeat != null)
{
if (_atoms == null)
{
_atoms = new List <Pair <RowRegexExprNodeAtom, RowRegexExprNode> >();
}
_atoms.Add(new Pair <RowRegexExprNodeAtom, RowRegexExprNode>(atom, optionalParent));
}
}
if (node is RowRegexExprNodeNested)
{
var nested = (RowRegexExprNodeNested)node;
if (nested.OptionalRepeat != null)
{
if (_nesteds == null)
{
_nesteds = new List <RowRegexNestedDesc>();
}
_nesteds.Add(new RowRegexNestedDesc(nested, optionalParent, level));
}
}
if (node is RowRegexExprNodePermute)
{
var permute = (RowRegexExprNodePermute)node;
if (_permutes == null)
{
_permutes = new List <RowRegexPermuteDesc>();
}
_permutes.Add(new RowRegexPermuteDesc(permute, optionalParent, level));
}
}
public static RowRegexExprNode Expand(RowRegexExprNode pattern)
{
var visitor = new RowRegexExprNodeVisitorRepeat();
pattern.Accept(visitor);
var newParentNode = pattern;
// expand permutes
var permutes = visitor.Permutes;
permutes.SortInPlace((o1, o2) =>
{
if (o1.Level > o2.Level)
{
return(-1);
}
return(o1.Level == o2.Level ? 0 : 1);
});
foreach (var permute in permutes)
{
var alteration = ExpandPermute(permute.Permute);
RowRegexExprNode optionalNewParent = Replace(permute.OptionalParent, permute.Permute, Collections.SingletonList <RowRegexExprNode>(alteration));
if (optionalNewParent != null)
{
newParentNode = optionalNewParent;
}
}
// expand atoms
var atomPairs = visitor.Atoms;
foreach (var pair in atomPairs)
{
var atom = pair.First;
var expandedRepeat = ExpandRepeat(atom, atom.OptionalRepeat, atom.NFAType, ATOM_HANDLER);
var optionalNewParent = Replace(pair.Second, pair.First, expandedRepeat);
if (optionalNewParent != null)
{
newParentNode = optionalNewParent;
}
}
// expand nested
var nestedPairs = visitor.Nesteds;
nestedPairs.SortInPlace((o1, o2) =>
{
if (o1.Level > o2.Level)
{
return(-1);
}
return(o1.Level == o2.Level ? 0 : 1);
});
foreach (var pair in nestedPairs)
{
var nested = pair.Nested;
var expandedRepeat = ExpandRepeat(nested, nested.OptionalRepeat, nested.NFAType, NESTED_HANDLER);
var optionalNewParent = Replace(pair.OptionalParent, pair.Nested, expandedRepeat);
if (optionalNewParent != null)
{
newParentNode = optionalNewParent;
}
}
return(newParentNode);
}
private static IList <RowRegexExprNode> ExpandRepeat(RowRegexExprNode node,
RowRegexExprRepeatDesc repeat,
RegexNFATypeEnum type,
RowRegexExprNodeCopier copier)
{
var evaluateParams = new EvaluateParams(null, true, null);
// handle single-bounds (no ranges)
IList <RowRegexExprNode> repeated = new List <RowRegexExprNode>();
if (repeat.Single != null)
{
ValidateExpression(repeat.Single);
int numRepeated = repeat.Single.ExprEvaluator.Evaluate(evaluateParams).AsInt();
ValidateRange(numRepeated, 1, int.MaxValue);
for (var i = 0; i < numRepeated; i++)
{
var copy = copier.Copy(node, type);
repeated.Add(copy);
}
return(repeated);
}
// evaluate bounds
int?lower = null;
int?upper = null;
if (repeat.Lower != null)
{
ValidateExpression(repeat.Lower);
lower = (int?)repeat.Lower.ExprEvaluator.Evaluate(evaluateParams);
}
if (repeat.Upper != null)
{
ValidateExpression(repeat.Upper);
upper = (int?)repeat.Upper.ExprEvaluator.Evaluate(evaluateParams);
}
// handle range
if (lower != null && upper != null)
{
ValidateRange(lower.Value, 1, int.MaxValue);
ValidateRange(upper.Value, 1, int.MaxValue);
ValidateRange(lower.Value, 1, upper.Value);
for (var i = 0; i < lower; i++)
{
var copy = copier.Copy(node, type);
repeated.Add(copy);
}
for (int i = lower.Value; i < upper; i++)
{
// make type optional
var newType = type;
if (type == RegexNFATypeEnum.SINGLE)
{
newType = RegexNFATypeEnum.ONE_OPTIONAL;
}
else if (type == RegexNFATypeEnum.ONE_TO_MANY)
{
newType = RegexNFATypeEnum.ZERO_TO_MANY;
}
else if (type == RegexNFATypeEnum.ONE_TO_MANY_RELUCTANT)
{
newType = RegexNFATypeEnum.ZERO_TO_MANY_RELUCTANT;
}
var copy = copier.Copy(node, newType);
repeated.Add(copy);
}
return(repeated);
}
// handle lower-bounds only
if (upper == null)
{
ValidateRange(lower.Value, 1, int.MaxValue);
for (var i = 0; i < lower; i++)
{
var copyInner = copier.Copy(node, type);
repeated.Add(copyInner);
}
// make type optional
var newType = type;
if (type == RegexNFATypeEnum.SINGLE)
{
newType = RegexNFATypeEnum.ZERO_TO_MANY;
}
else if (type == RegexNFATypeEnum.ONE_OPTIONAL)
{
newType = RegexNFATypeEnum.ZERO_TO_MANY;
}
else if (type == RegexNFATypeEnum.ONE_OPTIONAL_RELUCTANT)
{
newType = RegexNFATypeEnum.ZERO_TO_MANY_RELUCTANT;
}
else if (type == RegexNFATypeEnum.ONE_TO_MANY)
{
newType = RegexNFATypeEnum.ZERO_TO_MANY;
}
else if (type == RegexNFATypeEnum.ONE_TO_MANY_RELUCTANT)
{
newType = RegexNFATypeEnum.ZERO_TO_MANY_RELUCTANT;
}
var copy = copier.Copy(node, newType);
repeated.Add(copy);
return(repeated);
}
// handle upper-bounds only
ValidateRange(upper.Value, 1, int.MaxValue);
for (var i = 0; i < upper; i++)
{
// make type optional
var newType = type;
if (type == RegexNFATypeEnum.SINGLE)
{
newType = RegexNFATypeEnum.ONE_OPTIONAL;
}
else if (type == RegexNFATypeEnum.ONE_TO_MANY)
{
newType = RegexNFATypeEnum.ZERO_TO_MANY;
}
else if (type == RegexNFATypeEnum.ONE_TO_MANY_RELUCTANT)
{
newType = RegexNFATypeEnum.ZERO_TO_MANY_RELUCTANT;
}
var copy = copier.Copy(node, newType);
repeated.Add(copy);
}
return(repeated);
}
/// <summary>
/// Ctor.
/// </summary>
/// <param name="viewChain">views</param>
/// <param name="matchRecognizeSpec">specification</param>
/// <param name="agentInstanceContext">The agent instance context.</param>
/// <param name="isUnbound">true for unbound stream</param>
/// <param name="annotations">annotations</param>
/// <exception cref="ExprValidationException">
/// Variable ' + defineItem.Identifier + ' has already been defined
/// or
/// An aggregate function may not appear in a DEFINE clause
/// or
/// Failed to validate condition expression for variable ' + defineItem.Identifier + ': + ex.Message
/// or
/// Aggregation functions in the measure-clause must only refer to properties of exactly one group variable returning multiple events
/// or
/// Aggregation functions in the measure-clause must refer to one or more properties of exactly one group variable returning multiple events
/// or
/// The measures clause requires that each expression utilizes the AS keyword to assign a column name
/// </exception>
/// <throws>ExprValidationException if validation fails</throws>
public EventRowRegexNFAViewFactory(
ViewFactoryChain viewChain,
MatchRecognizeSpec matchRecognizeSpec,
AgentInstanceContext agentInstanceContext,
bool isUnbound,
Attribute[] annotations,
ConfigurationEngineDefaults.MatchRecognizeConfig matchRecognizeConfig)
{
var parentViewType = viewChain.EventType;
_matchRecognizeSpec = matchRecognizeSpec;
_isUnbound = isUnbound;
_isIterateOnly = HintEnum.ITERATE_ONLY.GetHint(annotations) != null;
_matchRecognizeConfig = matchRecognizeConfig;
var statementContext = agentInstanceContext.StatementContext;
// Expand repeats and permutations
_expandedPatternNode = RegexPatternExpandUtil.Expand(matchRecognizeSpec.Pattern);
// Determine single-row and multiple-row variables
_variablesSingle = new LinkedHashSet <string>();
ISet <string> variablesMultiple = new LinkedHashSet <string>();
EventRowRegexHelper.RecursiveInspectVariables(_expandedPatternNode, false, _variablesSingle, variablesMultiple);
// each variable gets associated with a stream number (multiple-row variables as well to hold the current event for the expression).
var streamNum = 0;
_variableStreams = new LinkedHashMap <string, Pair <int, bool> >();
foreach (var variableSingle in _variablesSingle)
{
_variableStreams.Put(variableSingle, new Pair <int, bool>(streamNum, false));
streamNum++;
}
foreach (var variableMultiple in variablesMultiple)
{
_variableStreams.Put(variableMultiple, new Pair <int, bool>(streamNum, true));
streamNum++;
}
// mapping of stream to variable
_streamVariables = new SortedDictionary <int, string>();
foreach (var entry in _variableStreams)
{
_streamVariables.Put(entry.Value.First, entry.Key);
}
// determine visibility rules
var visibility = EventRowRegexHelper.DetermineVisibility(_expandedPatternNode);
// assemble all single-row variables for expression validation
var allStreamNames = new string[_variableStreams.Count];
var allTypes = new EventType[_variableStreams.Count];
streamNum = 0;
foreach (var variableSingle in _variablesSingle)
{
allStreamNames[streamNum] = variableSingle;
allTypes[streamNum] = parentViewType;
streamNum++;
}
foreach (var variableMultiple in variablesMultiple)
{
allStreamNames[streamNum] = variableMultiple;
allTypes[streamNum] = parentViewType;
streamNum++;
}
// determine type service for use with DEFINE
// validate each DEFINE clause expression
ISet <string> definedVariables = new HashSet <string>();
IList <ExprAggregateNode> aggregateNodes = new List <ExprAggregateNode>();
var exprEvaluatorContext = new ExprEvaluatorContextStatement(statementContext, false);
_isExprRequiresMultimatchState = new bool[_variableStreams.Count];
for (var defineIndex = 0; defineIndex < matchRecognizeSpec.Defines.Count; defineIndex++)
{
var defineItem = matchRecognizeSpec.Defines[defineIndex];
if (definedVariables.Contains(defineItem.Identifier))
{
throw new ExprValidationException("Variable '" + defineItem.Identifier + "' has already been defined");
}
definedVariables.Add(defineItem.Identifier);
// stream-type visibilities handled here
var typeServiceDefines = EventRowRegexNFAViewFactoryHelper.BuildDefineStreamTypeServiceDefine(statementContext, _variableStreams, defineItem, visibility, parentViewType);
var exprNodeResult = HandlePreviousFunctions(defineItem.Expression);
var validationContext = new ExprValidationContext(
typeServiceDefines,
statementContext.EngineImportService,
statementContext.StatementExtensionServicesContext, null,
statementContext.SchedulingService,
statementContext.VariableService,
statementContext.TableService, exprEvaluatorContext,
statementContext.EventAdapterService,
statementContext.StatementName,
statementContext.StatementId,
statementContext.Annotations,
statementContext.ContextDescriptor,
statementContext.ScriptingService,
true, false, true, false, null, false);
ExprNode validated;
try {
// validate
validated = ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.MATCHRECOGDEFINE, exprNodeResult, validationContext);
// check aggregates
defineItem.Expression = validated;
ExprAggregateNodeUtil.GetAggregatesBottomUp(validated, aggregateNodes);
if (!aggregateNodes.IsEmpty())
{
throw new ExprValidationException("An aggregate function may not appear in a DEFINE clause");
}
}
catch (ExprValidationException ex) {
throw new ExprValidationException("Failed to validate condition expression for variable '" + defineItem.Identifier + "': " + ex.Message, ex);
}
// determine access to event properties from multi-matches
var visitor = new ExprNodeStreamRequiredVisitor();
validated.Accept(visitor);
var streamsRequired = visitor.StreamsRequired;
foreach (var streamRequired in streamsRequired)
{
if (streamRequired >= _variableStreams.Count)
{
var streamNumIdent = _variableStreams.Get(defineItem.Identifier).First;
_isExprRequiresMultimatchState[streamNumIdent] = true;
break;
}
}
}
_isDefineAsksMultimatches = CollectionUtil.IsAnySet(_isExprRequiresMultimatchState);
_defineMultimatchEventBean = _isDefineAsksMultimatches ? EventRowRegexNFAViewFactoryHelper.GetDefineMultimatchBean(statementContext, _variableStreams, parentViewType) : null;
// assign "prev" node indexes
// Since an expression such as "prior(2, price), prior(8, price)" translates into {2, 8} the relative index is {0, 1}.
// Map the expression-supplied index to a relative index
var countPrev = 0;
foreach (var entry in _callbacksPerIndex)
{
foreach (var callback in entry.Value)
{
callback.AssignedIndex = countPrev;
}
countPrev++;
}
// determine type service for use with MEASURE
IDictionary <string, object> measureTypeDef = new LinkedHashMap <string, object>();
foreach (var variableSingle in _variablesSingle)
{
measureTypeDef.Put(variableSingle, parentViewType);
}
foreach (var variableMultiple in variablesMultiple)
{
measureTypeDef.Put(variableMultiple, new EventType[] { parentViewType });
}
var outputEventTypeName = statementContext.StatementId + "_rowrecog";
_compositeEventType = (ObjectArrayEventType)statementContext.EventAdapterService.CreateAnonymousObjectArrayType(outputEventTypeName, measureTypeDef);
StreamTypeService typeServiceMeasure = new StreamTypeServiceImpl(_compositeEventType, "MATCH_RECOGNIZE", true, statementContext.EngineURI);
// find MEASURE clause aggregations
var measureReferencesMultivar = false;
IList <ExprAggregateNode> measureAggregateExprNodes = new List <ExprAggregateNode>();
foreach (var measureItem in matchRecognizeSpec.Measures)
{
ExprAggregateNodeUtil.GetAggregatesBottomUp(measureItem.Expr, measureAggregateExprNodes);
}
if (!measureAggregateExprNodes.IsEmpty())
{
var isIStreamOnly = new bool[allStreamNames.Length];
CompatExtensions.Fill(isIStreamOnly, true);
var typeServiceAggregateMeasure = new StreamTypeServiceImpl(allTypes, allStreamNames, isIStreamOnly, statementContext.EngineURI, false);
var measureExprAggNodesPerStream = new Dictionary <int, IList <ExprAggregateNode> >();
foreach (var aggregateNode in measureAggregateExprNodes)
{
// validate absence of group-by
aggregateNode.ValidatePositionals();
if (aggregateNode.OptionalLocalGroupBy != null)
{
throw new ExprValidationException("Match-recognize does not allow aggregation functions to specify a group-by");
}
// validate node and params
var count = 0;
var visitor = new ExprNodeIdentifierVisitor(true);
var validationContext = new ExprValidationContext(
typeServiceAggregateMeasure,
statementContext.EngineImportService,
statementContext.StatementExtensionServicesContext, null,
statementContext.SchedulingService,
statementContext.VariableService,
statementContext.TableService,
exprEvaluatorContext,
statementContext.EventAdapterService,
statementContext.StatementName,
statementContext.StatementId,
statementContext.Annotations,
statementContext.ContextDescriptor,
statementContext.ScriptingService,
false, false, true, false, null, false);
for (int ii = 0; ii < aggregateNode.ChildNodes.Count; ii++)
{
var child = aggregateNode.ChildNodes[ii];
var validated = ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.MATCHRECOGMEASURE, child, validationContext);
validated.Accept(visitor);
aggregateNode.SetChildNode(count++, new ExprNodeValidated(validated));
}
validationContext = new ExprValidationContext(
typeServiceMeasure,
statementContext.EngineImportService,
statementContext.StatementExtensionServicesContext, null,
statementContext.SchedulingService,
statementContext.VariableService,
statementContext.TableService,
exprEvaluatorContext,
statementContext.EventAdapterService,
statementContext.StatementName,
statementContext.StatementId,
statementContext.Annotations,
statementContext.ContextDescriptor,
statementContext.ScriptingService,
false, false, true, false, null, false);
aggregateNode.Validate(validationContext);
// verify properties used within the aggregation
var aggregatedStreams = new HashSet <int>();
foreach (var pair in visitor.ExprProperties)
{
aggregatedStreams.Add(pair.First);
}
int?multipleVarStream = null;
foreach (int streamNumAggregated in aggregatedStreams)
{
var variable = _streamVariables.Get(streamNumAggregated);
if (variablesMultiple.Contains(variable))
{
measureReferencesMultivar = true;
if (multipleVarStream == null)
{
multipleVarStream = streamNumAggregated;
continue;
}
throw new ExprValidationException("Aggregation functions in the measure-clause must only refer to properties of exactly one group variable returning multiple events");
}
}
if (multipleVarStream == null)
{
throw new ExprValidationException("Aggregation functions in the measure-clause must refer to one or more properties of exactly one group variable returning multiple events");
}
var aggNodesForStream = measureExprAggNodesPerStream.Get(multipleVarStream.Value);
if (aggNodesForStream == null)
{
aggNodesForStream = new List <ExprAggregateNode>();
measureExprAggNodesPerStream.Put(multipleVarStream.Value, aggNodesForStream);
}
aggNodesForStream.Add(aggregateNode);
}
var factoryDesc = AggregationServiceFactoryFactory.GetServiceMatchRecognize(_streamVariables.Count, measureExprAggNodesPerStream, typeServiceAggregateMeasure.EventTypes);
_aggregationService = factoryDesc.AggregationServiceFactory.MakeService(agentInstanceContext);
_aggregationExpressions = factoryDesc.Expressions;
}
else
{
_aggregationService = null;
_aggregationExpressions = Collections.GetEmptyList <AggregationServiceAggExpressionDesc>();
}
// validate each MEASURE clause expression
IDictionary <string, object> rowTypeDef = new LinkedHashMap <string, object>();
var streamRefVisitor = new ExprNodeStreamUseCollectVisitor();
foreach (var measureItem in matchRecognizeSpec.Measures)
{
if (measureItem.Name == null)
{
throw new ExprValidationException("The measures clause requires that each expression utilizes the AS keyword to assign a column name");
}
var validated = ValidateMeasureClause(measureItem.Expr, typeServiceMeasure, variablesMultiple, _variablesSingle, statementContext);
measureItem.Expr = validated;
rowTypeDef.Put(measureItem.Name, validated.ExprEvaluator.ReturnType);
validated.Accept(streamRefVisitor);
}
// Determine if any of the multi-var streams are referenced in the measures (non-aggregated only)
foreach (var @ref in streamRefVisitor.Referenced)
{
var rootPropName = @ref.RootPropertyNameIfAny;
if (rootPropName != null)
{
if (variablesMultiple.Contains(rootPropName))
{
measureReferencesMultivar = true;
break;
}
}
var streamRequired = @ref.StreamReferencedIfAny;
if (streamRequired != null)
{
var streamVariable = _streamVariables.Get(streamRequired.Value);
if (streamVariable != null)
{
var def = _variableStreams.Get(streamVariable);
if (def != null && def.Second)
{
measureReferencesMultivar = true;
break;
}
}
}
}
_isCollectMultimatches = measureReferencesMultivar || _isDefineAsksMultimatches;
// create rowevent type
var rowEventTypeName = statementContext.StatementId + "_rowrecogrow";
_rowEventType = statementContext.EventAdapterService.CreateAnonymousMapType(rowEventTypeName, rowTypeDef, true);
// validate partition-by expressions, if any
if (!matchRecognizeSpec.PartitionByExpressions.IsEmpty())
{
var typeServicePartition = new StreamTypeServiceImpl(parentViewType, "MATCH_RECOGNIZE_PARTITION", true, statementContext.EngineURI);
var validated = new List <ExprNode>();
var validationContext = new ExprValidationContext(
typeServicePartition,
statementContext.EngineImportService,
statementContext.StatementExtensionServicesContext, null,
statementContext.SchedulingService, statementContext.VariableService, statementContext.TableService,
exprEvaluatorContext, statementContext.EventAdapterService, statementContext.StatementName,
statementContext.StatementId, statementContext.Annotations, statementContext.ContextDescriptor,
statementContext.ScriptingService,
false, false, true, false, null, false);
foreach (var partitionExpr in matchRecognizeSpec.PartitionByExpressions)
{
validated.Add(ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.MATCHRECOGPARTITION, partitionExpr, validationContext));
}
matchRecognizeSpec.PartitionByExpressions = validated;
}
// validate interval if present
if (matchRecognizeSpec.Interval != null)
{
var validationContext =
new ExprValidationContext(
new StreamTypeServiceImpl(statementContext.EngineURI, false),
statementContext.EngineImportService,
statementContext.StatementExtensionServicesContext, null,
statementContext.SchedulingService,
statementContext.VariableService, statementContext.TableService, exprEvaluatorContext,
statementContext.EventAdapterService, statementContext.StatementName, statementContext.StatementId,
statementContext.Annotations, statementContext.ContextDescriptor, statementContext.ScriptingService,
false, false, true, false, null, false);
matchRecognizeSpec.Interval.Validate(validationContext);
}
}
public RowRegexPermuteDesc(RowRegexExprNodePermute permute, RowRegexExprNode optionalParent, int level)
{
Permute = permute;
OptionalParent = optionalParent;
Level = level;
}
public RowRegexNestedDesc(RowRegexExprNodeNested nested, RowRegexExprNode optionalParent, int level)
{
Nested = nested;
OptionalParent = optionalParent;
Level = level;
}
private static RegexNFAStrand RecursiveBuildStatesInternal(
RowRegexExprNode node,
IDictionary <String, ExprNode> variableDefinitions,
IDictionary <String, Pair <int, Boolean> > variableStreams,
Stack <int> nodeNumStack,
bool[] exprRequiresMultimatchState)
{
if (node is RowRegexExprNodeAlteration)
{
var nodeNum = 0;
var cumulativeStartStates = new List <RegexNFAStateBase>();
var cumulativeStates = new List <RegexNFAStateBase>();
var cumulativeEndStates = new List <RegexNFAStateBase>();
var isPassthrough = false;
foreach (var child in node.ChildNodes)
{
nodeNumStack.Push(nodeNum);
var strand = RecursiveBuildStatesInternal(child,
variableDefinitions,
variableStreams,
nodeNumStack,
exprRequiresMultimatchState);
nodeNumStack.Pop();
cumulativeStartStates.AddAll(strand.StartStates);
cumulativeStates.AddAll(strand.AllStates);
cumulativeEndStates.AddAll(strand.EndStates);
if (strand.IsPassthrough)
{
isPassthrough = true;
}
nodeNum++;
}
return(new RegexNFAStrand(cumulativeStartStates, cumulativeEndStates, cumulativeStates, isPassthrough));
}
else if (node is RowRegexExprNodeConcatenation)
{
var nodeNum = 0;
var isPassthrough = true;
var cumulativeStates = new List <RegexNFAStateBase>();
var strands = new RegexNFAStrand[node.ChildNodes.Count];
foreach (var child in node.ChildNodes)
{
nodeNumStack.Push(nodeNum);
strands[nodeNum] = RecursiveBuildStatesInternal(child,
variableDefinitions,
variableStreams,
nodeNumStack,
exprRequiresMultimatchState);
nodeNumStack.Pop();
cumulativeStates.AddAll(strands[nodeNum].AllStates);
if (!strands[nodeNum].IsPassthrough)
{
isPassthrough = false;
}
nodeNum++;
}
// determine start states: all states until the first non-passthrough start state
var startStates = new List <RegexNFAStateBase>();
for (var i = 0; i < strands.Length; i++)
{
startStates.AddAll(strands[i].StartStates);
if (!strands[i].IsPassthrough)
{
break;
}
}
// determine end states: all states from the back until the last non-passthrough end state
var endStates = new List <RegexNFAStateBase>();
for (var i = strands.Length - 1; i >= 0; i--)
{
endStates.AddAll(strands[i].EndStates);
if (!strands[i].IsPassthrough)
{
break;
}
}
// hook up the end state of each strand with the start states of each next strand
for (var i = strands.Length - 1; i >= 1; i--)
{
var current = strands[i];
for (var j = i - 1; j >= 0; j--)
{
var prior = strands[j];
foreach (var endState in prior.EndStates)
{
foreach (var startState in current.StartStates)
{
endState.AddState(startState);
}
}
if (!prior.IsPassthrough)
{
break;
}
}
}
return(new RegexNFAStrand(startStates, endStates, cumulativeStates, isPassthrough));
}
else if (node is RowRegexExprNodeNested)
{
var nested = (RowRegexExprNodeNested)node;
nodeNumStack.Push(0);
var strand = RecursiveBuildStatesInternal(node.ChildNodes[0],
variableDefinitions,
variableStreams,
nodeNumStack,
exprRequiresMultimatchState);
nodeNumStack.Pop();
var isPassthrough = strand.IsPassthrough || nested.NFAType.IsOptional();
// if this is a repeating node then pipe back each end state to each begin state
if (nested.NFAType.IsMultipleMatches())
{
foreach (var endstate in strand.EndStates)
{
foreach (var startstate in strand.StartStates)
{
if (!endstate.NextStates.Contains(startstate))
{
endstate.NextStates.Add(startstate);
}
}
}
}
return(new RegexNFAStrand(strand.StartStates, strand.EndStates, strand.AllStates, isPassthrough));
}
else
{
var atom = (RowRegexExprNodeAtom)node;
// assign stream number for single-variables for most direct expression eval; multiple-variable gets -1
var streamNum = variableStreams.Get(atom.Tag).First;
var multiple = variableStreams.Get(atom.Tag).Second;
var expressionDef = variableDefinitions.Get(atom.Tag);
var exprRequiresMultimatch = exprRequiresMultimatchState[streamNum];
RegexNFAStateBase nextState;
if ((atom.NFAType == RegexNFATypeEnum.ZERO_TO_MANY) || (atom.NFAType == RegexNFATypeEnum.ZERO_TO_MANY_RELUCTANT))
{
nextState = new RegexNFAStateZeroToMany(ToString(nodeNumStack), atom.Tag, streamNum, multiple, atom.NFAType.IsGreedy(), expressionDef, exprRequiresMultimatch);
}
else if ((atom.NFAType == RegexNFATypeEnum.ONE_TO_MANY) || (atom.NFAType == RegexNFATypeEnum.ONE_TO_MANY_RELUCTANT))
{
nextState = new RegexNFAStateOneToMany(ToString(nodeNumStack), atom.Tag, streamNum, multiple, atom.NFAType.IsGreedy(), expressionDef, exprRequiresMultimatch);
}
else if ((atom.NFAType == RegexNFATypeEnum.ONE_OPTIONAL) || (atom.NFAType == RegexNFATypeEnum.ONE_OPTIONAL_RELUCTANT))
{
nextState = new RegexNFAStateOneOptional(ToString(nodeNumStack), atom.Tag, streamNum, multiple, atom.NFAType.IsGreedy(), expressionDef, exprRequiresMultimatch);
}
else if (expressionDef == null)
{
nextState = new RegexNFAStateAnyOne(ToString(nodeNumStack), atom.Tag, streamNum, multiple);
}
else
{
nextState = new RegexNFAStateFilter(ToString(nodeNumStack), atom.Tag, streamNum, multiple, expressionDef, exprRequiresMultimatch);
}
return(new RegexNFAStrand(
Collections.SingletonList(nextState),
Collections.SingletonList(nextState),
Collections.SingletonList(nextState),
atom.NFAType.IsOptional()));
}
}
/// <summary>
/// Adds a child node.
/// </summary>
/// <param name="childNode">is the child evaluation tree node to add</param>
public void AddChildNode(RowRegexExprNode childNode)
{
ChildNodes.Add(childNode);
}
