private static IList<ExprNode> DecomposeCheckAggregation(IList<ExprNode> validatedNodes)
{
// Break a top-level AND into constituent expression nodes
IList<ExprNode> constituents = new List<ExprNode>();
foreach (ExprNode validated in validatedNodes)
{
if (validated is ExprAndNode)
{
RecursiveAndConstituents(constituents, validated);
}
else
{
constituents.Add(validated);
}
// Ensure there is no aggregation nodes
var aggregateExprNodes = new List<ExprAggregateNode>();
ExprAggregateNodeUtil.GetAggregatesBottomUp(validated, aggregateExprNodes);
if (!aggregateExprNodes.IsEmpty())
{
throw new ExprValidationException("Aggregation functions not allowed within filters");
}
}
return constituents;
}
public static void ValidateNoSpecialsGroupByExpressions(ExprNode[] groupByNodes)
{
ExprNodeSubselectDeclaredDotVisitor visitorSubselects = new ExprNodeSubselectDeclaredDotVisitor();
ExprNodeGroupingVisitorWParent visitorGrouping = new ExprNodeGroupingVisitorWParent();
IList<ExprAggregateNode> aggNodesInGroupBy = new List<ExprAggregateNode>(1);
foreach (ExprNode groupByNode in groupByNodes) {
// no subselects
groupByNode.Accept(visitorSubselects);
if (visitorSubselects.Subselects.Count > 0) {
throw new ExprValidationException("Subselects not allowed within group-by");
}
// no special grouping-clauses
groupByNode.Accept(visitorGrouping);
if (!visitorGrouping.GroupingIdNodes.IsEmpty()) {
throw ExprGroupingIdNode.MakeException("grouping_id");
}
if (!visitorGrouping.GroupingNodes.IsEmpty()) {
throw ExprGroupingIdNode.MakeException("grouping");
}
// no aggregations allowed
ExprAggregateNodeUtil.GetAggregatesBottomUp(groupByNode, aggNodesInGroupBy);
if (!aggNodesInGroupBy.IsEmpty()) {
throw new ExprValidationException("Group-by expressions cannot contain aggregate functions");
}
}
}
/// <summary>
/// Returns processor for order-by clauses.
/// </summary>
/// <param name="selectionList">is a list of select expressions</param>
/// <param name="groupByNodes">is a list of group-by expressions</param>
/// <param name="orderByList">is a list of order-by expressions</param>
/// <param name="rowLimitSpec">specification for row limit, or null if no row limit is defined</param>
/// <param name="variableService">for retrieving variable state for use with row limiting</param>
/// <param name="isSortUsingCollator">for string value sorting using compare or Collator</param>
/// <param name="optionalContextName">Name of the optional context.</param>
/// <returns>ordering processor instance</returns>
/// <throws><seealso cref="ExprValidationException" /> when validation of expressions fails</throws>
public static OrderByProcessorFactory GetProcessor(
IList <SelectClauseExprCompiledSpec> selectionList,
ExprNode[] groupByNodes,
IList <OrderByItem> orderByList,
RowLimitSpec rowLimitSpec,
VariableService variableService,
bool isSortUsingCollator,
String optionalContextName)
{
// Get the order by expression nodes
IList <ExprNode> orderByNodes = orderByList.Select(element => element.ExprNode).ToList();
// No order-by clause
if (orderByList.IsEmpty())
{
Log.Debug(".getProcessor Using no _orderByProcessor");
if (rowLimitSpec != null)
{
var rowLimitProcessorFactory = new RowLimitProcessorFactory(rowLimitSpec, variableService, optionalContextName);
return(new OrderByProcessorRowLimitOnlyFactory(rowLimitProcessorFactory));
}
return(null);
}
// Determine aggregate functions used in select, if any
IList <ExprAggregateNode> selectAggNodes = new List <ExprAggregateNode>();
foreach (var element in selectionList)
{
ExprAggregateNodeUtil.GetAggregatesBottomUp(element.SelectExpression, selectAggNodes);
}
// Get all the aggregate functions occuring in the order-by clause
IList <ExprAggregateNode> orderAggNodes = new List <ExprAggregateNode>();
foreach (var orderByNode in orderByNodes)
{
ExprAggregateNodeUtil.GetAggregatesBottomUp(orderByNode, orderAggNodes);
}
ValidateOrderByAggregates(selectAggNodes, orderAggNodes);
// Tell the order-by processor whether to compute group-by
// keys if they are not present
var needsGroupByKeys = !selectionList.IsEmpty() && !orderAggNodes.IsEmpty();
Log.Debug(".getProcessor Using OrderByProcessorImpl");
var orderByProcessorFactory = new OrderByProcessorFactoryImpl(orderByList, groupByNodes, needsGroupByKeys, isSortUsingCollator);
if (rowLimitSpec == null)
{
return(orderByProcessorFactory);
}
else
{
var rowLimitProcessorFactory = new RowLimitProcessorFactory(rowLimitSpec, variableService, optionalContextName);
return(new OrderByProcessorOrderedLimitFactory(orderByProcessorFactory, rowLimitProcessorFactory));
}
}
// Min/Max nodes can be either an aggregate or a per-row function depending on the number or arguments
private static void HandleMinMax(
string ident,
EsperEPL2GrammarParser.LibFunctionArgsContext ctxArgs,
IDictionary <ITree, ExprNode> astExprNodeMap)
{
// Determine min or max
var childNodeText = ident;
MinMaxTypeEnum minMaxTypeEnum;
var filtered = childNodeText.StartsWith("f");
if (childNodeText.ToLowerInvariant().Equals("min") || childNodeText.ToLowerInvariant().Equals("fmin"))
{
minMaxTypeEnum = MinMaxTypeEnum.MIN;
}
else if (childNodeText.ToLowerInvariant().Equals("max") || childNodeText.ToLowerInvariant().Equals("fmax"))
{
minMaxTypeEnum = MinMaxTypeEnum.MAX;
}
else
{
throw ASTWalkException.From("Uncountered unrecognized min or max node '" + ident + "'");
}
var args = Collections.GetEmptyList <ExprNode>();
if (ctxArgs != null && ctxArgs.libFunctionArgItem() != null)
{
args = ASTExprHelper.ExprCollectSubNodes(ctxArgs, 0, astExprNodeMap);
}
var numArgsPositional = ExprAggregateNodeUtil.CountPositionalArgs(args);
var isDistinct = ctxArgs != null && ctxArgs.DISTINCT() != null;
if (numArgsPositional > 1 && isDistinct && !filtered)
{
throw ASTWalkException.From(
"The distinct keyword is not valid in per-row min and max " +
"functions with multiple sub-expressions");
}
ExprNode minMaxNode;
if (!isDistinct && numArgsPositional > 1 && !filtered)
{
// use the row function
minMaxNode = new ExprMinMaxRowNode(minMaxTypeEnum);
}
else
{
// use the aggregation function
minMaxNode = new ExprMinMaxAggrNode(isDistinct, minMaxTypeEnum, filtered, false);
}
minMaxNode.AddChildNodes(args);
astExprNodeMap.Put(ctxArgs, minMaxNode);
}
internal static void ValidateNoAggregations(ExprNode exprNode, String errorMsg)
{
// Make sure there is no aggregation in the where clause
var aggregateNodes = new List <ExprAggregateNode>();
ExprAggregateNodeUtil.GetAggregatesBottomUp(exprNode, aggregateNodes);
if (!aggregateNodes.IsEmpty())
{
throw new ExprValidationException(errorMsg);
}
}
public void TestGetAggregatesBottomUp()
{
/*
* top (ag)
* c1 c2
* c1_1 (ag) c1_2 (ag) c2_1 c2_2
* c2_1_1 (ag) c2_1_2 (ag)
*
*/
ExprNode top = new SupportAggregateExprNode(null);
ExprNode c1 = new SupportExprNode(null);
ExprNode c2 = new SupportExprNode(null);
top.AddChildNode(c1);
top.AddChildNode(c2);
ExprNode c1_1 = new SupportAggregateExprNode(null);
ExprNode c1_2 = new SupportAggregateExprNode(null);
c1.AddChildNode(c1_1);
c1.AddChildNode(c1_2);
c1_1.AddChildNode(new SupportExprNode(null));
c1_2.AddChildNode(new SupportExprNode(null));
ExprNode c2_1 = new SupportExprNode(null);
ExprNode c2_2 = new SupportExprNode(null);
c2.AddChildNode(c2_1);
c2.AddChildNode(c2_2);
c2_2.AddChildNode(new SupportExprNode(null));
ExprNode c2_1_1 = new SupportAggregateExprNode(null);
ExprNode c2_1_2 = new SupportAggregateExprNode(null);
c2_1.AddChildNode(c2_1_1);
c2_1.AddChildNode(c2_1_2);
List <ExprAggregateNode> aggregates = new List <ExprAggregateNode>();
ExprAggregateNodeUtil.GetAggregatesBottomUp(top, aggregates);
Assert.AreEqual(5, aggregates.Count);
Assert.AreSame(c2_1_1, aggregates[0]);
Assert.AreSame(c2_1_2, aggregates[1]);
Assert.AreSame(c1_1, aggregates[2]);
Assert.AreSame(c1_2, aggregates[3]);
Assert.AreSame(top, aggregates[4]);
// Test no aggregates
aggregates.Clear();
ExprAggregateNodeUtil.GetAggregatesBottomUp(new SupportExprNode(null), aggregates);
Assert.IsTrue(aggregates.IsEmpty());
}
private static ExprNode HandleMinMaxNode(
string chainFirstLowerCase,
Chainable spec)
{
MinMaxTypeEnum minMaxTypeEnum;
var filtered = chainFirstLowerCase.StartsWith("f");
if (chainFirstLowerCase.Equals("min") || chainFirstLowerCase.Equals("fmin"))
{
minMaxTypeEnum = MinMaxTypeEnum.MIN;
}
else if (chainFirstLowerCase.Equals("max") || chainFirstLowerCase.Equals("fmax"))
{
minMaxTypeEnum = MinMaxTypeEnum.MAX;
}
else
{
throw new ValidationException("Uncountered unrecognized min or max node '" + spec.GetRootNameOrEmptyString() + "'");
}
var args = spec.GetParametersOrEmpty();
var distinct = spec.IsDistinct;
var numArgsPositional = ExprAggregateNodeUtil.CountPositionalArgs(args);
if (numArgsPositional > 1 && spec.IsDistinct && !filtered)
{
throw new ValidationException(
"The distinct keyword is not valid in per-row min and max " +
"functions with multiple sub-expressions");
}
ExprNode minMaxNode;
if (!distinct && numArgsPositional > 1 && !filtered)
{
// use the row function
minMaxNode = new ExprMinMaxRowNode(minMaxTypeEnum);
}
else
{
// use the aggregation function
minMaxNode = new ExprMinMaxAggrNode(distinct, minMaxTypeEnum, filtered, false);
}
minMaxNode.AddChildNodes(args);
return(minMaxNode);
}
/// <summary>
/// Check if the expression is minimal: does not have a subselect, aggregation and does not need view resources
/// </summary>
/// <param name="expression">to inspect</param>
/// <returns>null if minimal, otherwise name of offending sub-expression</returns>
public static string IsMinimalExpression(ExprNode expression)
{
ExprNodeSubselectDeclaredDotVisitor subselectVisitor = new ExprNodeSubselectDeclaredDotVisitor();
expression.Accept(subselectVisitor);
if (subselectVisitor.Subselects.Count > 0) {
return "a subselect";
}
ExprNodeViewResourceVisitor viewResourceVisitor = new ExprNodeViewResourceVisitor();
expression.Accept(viewResourceVisitor);
if (viewResourceVisitor.ExprNodes.Count > 0) {
return "a function that requires view resources (prior, prev)";
}
IList<ExprAggregateNode> aggregateNodes = new List<ExprAggregateNode>();
ExprAggregateNodeUtil.GetAggregatesBottomUp(expression, aggregateNodes);
if (!aggregateNodes.IsEmpty()) {
return "an aggregation function";
}
return null;
}
/// <summary>
/// Validate filter and join expression nodes.
/// </summary>
/// <param name="statementSpec">the compiled statement</param>
/// <param name="statementContext">the statement services</param>
/// <param name="typeService">the event types for streams</param>
/// <param name="viewResourceDelegate">the delegate to verify expressions that use view resources</param>
internal static void ValidateNodes(
StatementSpecCompiled statementSpec,
StatementContext statementContext,
StreamTypeService typeService,
ViewResourceDelegateUnverified viewResourceDelegate)
{
var methodResolutionService = statementContext.MethodResolutionService;
var evaluatorContextStmt = new ExprEvaluatorContextStatement(statementContext, false);
var intoTableName = statementSpec.IntoTableSpec == null ? null : statementSpec.IntoTableSpec.Name;
if (statementSpec.FilterRootNode != null)
{
var optionalFilterNode = statementSpec.FilterRootNode;
// Validate where clause, initializing nodes to the stream ids used
try
{
var validationContext = new ExprValidationContext(
typeService,
methodResolutionService,
viewResourceDelegate,
statementContext.SchedulingService,
statementContext.VariableService,
statementContext.TableService,
evaluatorContextStmt,
statementContext.EventAdapterService,
statementContext.StatementName,
statementContext.StatementId,
statementContext.Annotations,
statementContext.ContextDescriptor,
statementContext.ScriptingService,
false, false, true, false,
intoTableName, false);
optionalFilterNode = ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.FILTER, optionalFilterNode, validationContext);
if (optionalFilterNode.ExprEvaluator.ReturnType != typeof(bool) && optionalFilterNode.ExprEvaluator.ReturnType != typeof(bool?))
{
throw new ExprValidationException("The where-clause filter expression must return a boolean value");
}
statementSpec.FilterExprRootNode = optionalFilterNode;
// Make sure there is no aggregation in the where clause
var aggregateNodes = new List <ExprAggregateNode>();
ExprAggregateNodeUtil.GetAggregatesBottomUp(optionalFilterNode, aggregateNodes);
if (!aggregateNodes.IsEmpty())
{
throw new ExprValidationException("An aggregate function may not appear in a WHERE clause (use the HAVING clause)");
}
}
catch (ExprValidationException ex)
{
Log.Debug(".validateNodes Validation exception for filter=" + ExprNodeUtility.ToExpressionStringMinPrecedenceSafe(optionalFilterNode), ex);
throw new EPStatementException("Error validating expression: " + ex.Message, ex, statementContext.Expression);
}
}
if ((statementSpec.OutputLimitSpec != null) && ((statementSpec.OutputLimitSpec.WhenExpressionNode != null) || (statementSpec.OutputLimitSpec.AndAfterTerminateExpr != null)))
{
// Validate where clause, initializing nodes to the stream ids used
try
{
var outputLimitType = OutputConditionExpressionFactory.GetBuiltInEventType(statementContext.EventAdapterService);
StreamTypeService typeServiceOutputWhen = new StreamTypeServiceImpl(new EventType[] { outputLimitType }, new String[] { null }, new bool[] { true }, statementContext.EngineURI, false);
var validationContext = new ExprValidationContext(
typeServiceOutputWhen,
methodResolutionService, null,
statementContext.SchedulingService,
statementContext.VariableService,
statementContext.TableService,
evaluatorContextStmt,
statementContext.EventAdapterService,
statementContext.StatementName,
statementContext.StatementId,
statementContext.Annotations,
statementContext.ContextDescriptor,
statementContext.ScriptingService,
false, false, false, false,
intoTableName, false);
var outputLimitWhenNode = statementSpec.OutputLimitSpec.WhenExpressionNode;
if (outputLimitWhenNode != null)
{
outputLimitWhenNode = ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.OUTPUTLIMIT, outputLimitWhenNode, validationContext);
statementSpec.OutputLimitSpec.WhenExpressionNode = outputLimitWhenNode;
if (TypeHelper.GetBoxedType(outputLimitWhenNode.ExprEvaluator.ReturnType) != typeof(bool?))
{
throw new ExprValidationException("The when-trigger expression in the OUTPUT WHEN clause must return a boolean-type value");
}
EPStatementStartMethodHelperValidate.ValidateNoAggregations(outputLimitWhenNode, "An aggregate function may not appear in a OUTPUT LIMIT clause");
}
// validate and-terminate expression if provided
if (statementSpec.OutputLimitSpec.AndAfterTerminateExpr != null)
{
if (statementSpec.OutputLimitSpec.RateType != OutputLimitRateType.WHEN_EXPRESSION && statementSpec.OutputLimitSpec.RateType != OutputLimitRateType.TERM)
{
throw new ExprValidationException("A terminated-and expression must be used with the OUTPUT WHEN clause");
}
var validated = ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.OUTPUTLIMIT, statementSpec.OutputLimitSpec.AndAfterTerminateExpr, validationContext);
statementSpec.OutputLimitSpec.AndAfterTerminateExpr = validated;
if (validated.ExprEvaluator.ReturnType.GetBoxedType() != typeof(bool?))
{
throw new ExprValidationException("The terminated-and expression must return a boolean-type value");
}
EPStatementStartMethodHelperValidate.ValidateNoAggregations(validated, "An aggregate function may not appear in a terminated-and clause");
}
// validate then-expression
ValidateThenSetAssignments(statementSpec.OutputLimitSpec.ThenExpressions, validationContext);
// validate after-terminated then-expression
ValidateThenSetAssignments(statementSpec.OutputLimitSpec.AndAfterTerminateThenExpressions, validationContext);
}
catch (ExprValidationException ex)
{
throw new EPStatementException("Error validating expression: " + ex.Message, statementContext.Expression);
}
}
for (var outerJoinCount = 0; outerJoinCount < statementSpec.OuterJoinDescList.Length; outerJoinCount++)
{
var outerJoinDesc = statementSpec.OuterJoinDescList[outerJoinCount];
// validate on-expression nodes, if provided
if (outerJoinDesc.OptLeftNode != null)
{
var streamIdPair = ValidateOuterJoinPropertyPair(statementContext, outerJoinDesc.OptLeftNode, outerJoinDesc.OptRightNode, outerJoinCount,
typeService, viewResourceDelegate);
if (outerJoinDesc.AdditionalLeftNodes != null)
{
ISet <int?> streamSet = new HashSet <int?>();
streamSet.Add(streamIdPair.First);
streamSet.Add(streamIdPair.Second);
for (var i = 0; i < outerJoinDesc.AdditionalLeftNodes.Length; i++)
{
var streamIdPairAdd = ValidateOuterJoinPropertyPair(statementContext, outerJoinDesc.AdditionalLeftNodes[i], outerJoinDesc.AdditionalRightNodes[i], outerJoinCount,
typeService, viewResourceDelegate);
// make sure all additional properties point to the same two streams
if ((!streamSet.Contains(streamIdPairAdd.First) || (!streamSet.Contains(streamIdPairAdd.Second))))
{
const string message = "Outer join ON-clause columns must refer to properties of the same joined streams" +
" when using multiple columns in the on-clause";
throw new EPStatementException("Error validating expression: " + message, statementContext.Expression);
}
}
}
}
}
}
/// <summary>
/// Ctor.
/// </summary>
/// <param name="viewChain">views</param>
/// <param name="matchRecognizeSpec">specification</param>
/// <param name="agentInstanceContext">The agent instance context.</param>
/// <param name="isUnbound">true for unbound stream</param>
/// <param name="annotations">annotations</param>
/// <exception cref="ExprValidationException">
/// Variable ' + defineItem.Identifier + ' has already been defined
/// or
/// An aggregate function may not appear in a DEFINE clause
/// or
/// Failed to validate condition expression for variable ' + defineItem.Identifier + ': + ex.Message
/// or
/// Aggregation functions in the measure-clause must only refer to properties of exactly one group variable returning multiple events
/// or
/// Aggregation functions in the measure-clause must refer to one or more properties of exactly one group variable returning multiple events
/// or
/// The measures clause requires that each expression utilizes the AS keyword to assign a column name
/// </exception>
/// <throws>ExprValidationException if validation fails</throws>
public EventRowRegexNFAViewFactory(
ViewFactoryChain viewChain,
MatchRecognizeSpec matchRecognizeSpec,
AgentInstanceContext agentInstanceContext,
bool isUnbound,
Attribute[] annotations,
ConfigurationEngineDefaults.MatchRecognizeConfig matchRecognizeConfig)
{
var parentViewType = viewChain.EventType;
_matchRecognizeSpec = matchRecognizeSpec;
_isUnbound = isUnbound;
_isIterateOnly = HintEnum.ITERATE_ONLY.GetHint(annotations) != null;
_matchRecognizeConfig = matchRecognizeConfig;
var statementContext = agentInstanceContext.StatementContext;
// Expand repeats and permutations
_expandedPatternNode = RegexPatternExpandUtil.Expand(matchRecognizeSpec.Pattern);
// Determine single-row and multiple-row variables
_variablesSingle = new LinkedHashSet <string>();
ISet <string> variablesMultiple = new LinkedHashSet <string>();
EventRowRegexHelper.RecursiveInspectVariables(_expandedPatternNode, false, _variablesSingle, variablesMultiple);
// each variable gets associated with a stream number (multiple-row variables as well to hold the current event for the expression).
var streamNum = 0;
_variableStreams = new LinkedHashMap <string, Pair <int, bool> >();
foreach (var variableSingle in _variablesSingle)
{
_variableStreams.Put(variableSingle, new Pair <int, bool>(streamNum, false));
streamNum++;
}
foreach (var variableMultiple in variablesMultiple)
{
_variableStreams.Put(variableMultiple, new Pair <int, bool>(streamNum, true));
streamNum++;
}
// mapping of stream to variable
_streamVariables = new SortedDictionary <int, string>();
foreach (var entry in _variableStreams)
{
_streamVariables.Put(entry.Value.First, entry.Key);
}
// determine visibility rules
var visibility = EventRowRegexHelper.DetermineVisibility(_expandedPatternNode);
// assemble all single-row variables for expression validation
var allStreamNames = new string[_variableStreams.Count];
var allTypes = new EventType[_variableStreams.Count];
streamNum = 0;
foreach (var variableSingle in _variablesSingle)
{
allStreamNames[streamNum] = variableSingle;
allTypes[streamNum] = parentViewType;
streamNum++;
}
foreach (var variableMultiple in variablesMultiple)
{
allStreamNames[streamNum] = variableMultiple;
allTypes[streamNum] = parentViewType;
streamNum++;
}
// determine type service for use with DEFINE
// validate each DEFINE clause expression
ISet <string> definedVariables = new HashSet <string>();
IList <ExprAggregateNode> aggregateNodes = new List <ExprAggregateNode>();
var exprEvaluatorContext = new ExprEvaluatorContextStatement(statementContext, false);
_isExprRequiresMultimatchState = new bool[_variableStreams.Count];
for (var defineIndex = 0; defineIndex < matchRecognizeSpec.Defines.Count; defineIndex++)
{
var defineItem = matchRecognizeSpec.Defines[defineIndex];
if (definedVariables.Contains(defineItem.Identifier))
{
throw new ExprValidationException("Variable '" + defineItem.Identifier + "' has already been defined");
}
definedVariables.Add(defineItem.Identifier);
// stream-type visibilities handled here
var typeServiceDefines = EventRowRegexNFAViewFactoryHelper.BuildDefineStreamTypeServiceDefine(statementContext, _variableStreams, defineItem, visibility, parentViewType);
var exprNodeResult = HandlePreviousFunctions(defineItem.Expression);
var validationContext = new ExprValidationContext(
typeServiceDefines,
statementContext.EngineImportService,
statementContext.StatementExtensionServicesContext, null,
statementContext.SchedulingService,
statementContext.VariableService,
statementContext.TableService, exprEvaluatorContext,
statementContext.EventAdapterService,
statementContext.StatementName,
statementContext.StatementId,
statementContext.Annotations,
statementContext.ContextDescriptor,
statementContext.ScriptingService,
true, false, true, false, null, false);
ExprNode validated;
try {
// validate
validated = ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.MATCHRECOGDEFINE, exprNodeResult, validationContext);
// check aggregates
defineItem.Expression = validated;
ExprAggregateNodeUtil.GetAggregatesBottomUp(validated, aggregateNodes);
if (!aggregateNodes.IsEmpty())
{
throw new ExprValidationException("An aggregate function may not appear in a DEFINE clause");
}
}
catch (ExprValidationException ex) {
throw new ExprValidationException("Failed to validate condition expression for variable '" + defineItem.Identifier + "': " + ex.Message, ex);
}
// determine access to event properties from multi-matches
var visitor = new ExprNodeStreamRequiredVisitor();
validated.Accept(visitor);
var streamsRequired = visitor.StreamsRequired;
foreach (var streamRequired in streamsRequired)
{
if (streamRequired >= _variableStreams.Count)
{
var streamNumIdent = _variableStreams.Get(defineItem.Identifier).First;
_isExprRequiresMultimatchState[streamNumIdent] = true;
break;
}
}
}
_isDefineAsksMultimatches = CollectionUtil.IsAnySet(_isExprRequiresMultimatchState);
_defineMultimatchEventBean = _isDefineAsksMultimatches ? EventRowRegexNFAViewFactoryHelper.GetDefineMultimatchBean(statementContext, _variableStreams, parentViewType) : null;
// assign "prev" node indexes
// Since an expression such as "prior(2, price), prior(8, price)" translates into {2, 8} the relative index is {0, 1}.
// Map the expression-supplied index to a relative index
var countPrev = 0;
foreach (var entry in _callbacksPerIndex)
{
foreach (var callback in entry.Value)
{
callback.AssignedIndex = countPrev;
}
countPrev++;
}
// determine type service for use with MEASURE
IDictionary <string, object> measureTypeDef = new LinkedHashMap <string, object>();
foreach (var variableSingle in _variablesSingle)
{
measureTypeDef.Put(variableSingle, parentViewType);
}
foreach (var variableMultiple in variablesMultiple)
{
measureTypeDef.Put(variableMultiple, new EventType[] { parentViewType });
}
var outputEventTypeName = statementContext.StatementId + "_rowrecog";
_compositeEventType = (ObjectArrayEventType)statementContext.EventAdapterService.CreateAnonymousObjectArrayType(outputEventTypeName, measureTypeDef);
StreamTypeService typeServiceMeasure = new StreamTypeServiceImpl(_compositeEventType, "MATCH_RECOGNIZE", true, statementContext.EngineURI);
// find MEASURE clause aggregations
var measureReferencesMultivar = false;
IList <ExprAggregateNode> measureAggregateExprNodes = new List <ExprAggregateNode>();
foreach (var measureItem in matchRecognizeSpec.Measures)
{
ExprAggregateNodeUtil.GetAggregatesBottomUp(measureItem.Expr, measureAggregateExprNodes);
}
if (!measureAggregateExprNodes.IsEmpty())
{
var isIStreamOnly = new bool[allStreamNames.Length];
CompatExtensions.Fill(isIStreamOnly, true);
var typeServiceAggregateMeasure = new StreamTypeServiceImpl(allTypes, allStreamNames, isIStreamOnly, statementContext.EngineURI, false);
var measureExprAggNodesPerStream = new Dictionary <int, IList <ExprAggregateNode> >();
foreach (var aggregateNode in measureAggregateExprNodes)
{
// validate absence of group-by
aggregateNode.ValidatePositionals();
if (aggregateNode.OptionalLocalGroupBy != null)
{
throw new ExprValidationException("Match-recognize does not allow aggregation functions to specify a group-by");
}
// validate node and params
var count = 0;
var visitor = new ExprNodeIdentifierVisitor(true);
var validationContext = new ExprValidationContext(
typeServiceAggregateMeasure,
statementContext.EngineImportService,
statementContext.StatementExtensionServicesContext, null,
statementContext.SchedulingService,
statementContext.VariableService,
statementContext.TableService,
exprEvaluatorContext,
statementContext.EventAdapterService,
statementContext.StatementName,
statementContext.StatementId,
statementContext.Annotations,
statementContext.ContextDescriptor,
statementContext.ScriptingService,
false, false, true, false, null, false);
for (int ii = 0; ii < aggregateNode.ChildNodes.Count; ii++)
{
var child = aggregateNode.ChildNodes[ii];
var validated = ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.MATCHRECOGMEASURE, child, validationContext);
validated.Accept(visitor);
aggregateNode.SetChildNode(count++, new ExprNodeValidated(validated));
}
validationContext = new ExprValidationContext(
typeServiceMeasure,
statementContext.EngineImportService,
statementContext.StatementExtensionServicesContext, null,
statementContext.SchedulingService,
statementContext.VariableService,
statementContext.TableService,
exprEvaluatorContext,
statementContext.EventAdapterService,
statementContext.StatementName,
statementContext.StatementId,
statementContext.Annotations,
statementContext.ContextDescriptor,
statementContext.ScriptingService,
false, false, true, false, null, false);
aggregateNode.Validate(validationContext);
// verify properties used within the aggregation
var aggregatedStreams = new HashSet <int>();
foreach (var pair in visitor.ExprProperties)
{
aggregatedStreams.Add(pair.First);
}
int?multipleVarStream = null;
foreach (int streamNumAggregated in aggregatedStreams)
{
var variable = _streamVariables.Get(streamNumAggregated);
if (variablesMultiple.Contains(variable))
{
measureReferencesMultivar = true;
if (multipleVarStream == null)
{
multipleVarStream = streamNumAggregated;
continue;
}
throw new ExprValidationException("Aggregation functions in the measure-clause must only refer to properties of exactly one group variable returning multiple events");
}
}
if (multipleVarStream == null)
{
throw new ExprValidationException("Aggregation functions in the measure-clause must refer to one or more properties of exactly one group variable returning multiple events");
}
var aggNodesForStream = measureExprAggNodesPerStream.Get(multipleVarStream.Value);
if (aggNodesForStream == null)
{
aggNodesForStream = new List <ExprAggregateNode>();
measureExprAggNodesPerStream.Put(multipleVarStream.Value, aggNodesForStream);
}
aggNodesForStream.Add(aggregateNode);
}
var factoryDesc = AggregationServiceFactoryFactory.GetServiceMatchRecognize(_streamVariables.Count, measureExprAggNodesPerStream, typeServiceAggregateMeasure.EventTypes);
_aggregationService = factoryDesc.AggregationServiceFactory.MakeService(agentInstanceContext);
_aggregationExpressions = factoryDesc.Expressions;
}
else
{
_aggregationService = null;
_aggregationExpressions = Collections.GetEmptyList <AggregationServiceAggExpressionDesc>();
}
// validate each MEASURE clause expression
IDictionary <string, object> rowTypeDef = new LinkedHashMap <string, object>();
var streamRefVisitor = new ExprNodeStreamUseCollectVisitor();
foreach (var measureItem in matchRecognizeSpec.Measures)
{
if (measureItem.Name == null)
{
throw new ExprValidationException("The measures clause requires that each expression utilizes the AS keyword to assign a column name");
}
var validated = ValidateMeasureClause(measureItem.Expr, typeServiceMeasure, variablesMultiple, _variablesSingle, statementContext);
measureItem.Expr = validated;
rowTypeDef.Put(measureItem.Name, validated.ExprEvaluator.ReturnType);
validated.Accept(streamRefVisitor);
}
// Determine if any of the multi-var streams are referenced in the measures (non-aggregated only)
foreach (var @ref in streamRefVisitor.Referenced)
{
var rootPropName = @ref.RootPropertyNameIfAny;
if (rootPropName != null)
{
if (variablesMultiple.Contains(rootPropName))
{
measureReferencesMultivar = true;
break;
}
}
var streamRequired = @ref.StreamReferencedIfAny;
if (streamRequired != null)
{
var streamVariable = _streamVariables.Get(streamRequired.Value);
if (streamVariable != null)
{
var def = _variableStreams.Get(streamVariable);
if (def != null && def.Second)
{
measureReferencesMultivar = true;
break;
}
}
}
}
_isCollectMultimatches = measureReferencesMultivar || _isDefineAsksMultimatches;
// create rowevent type
var rowEventTypeName = statementContext.StatementId + "_rowrecogrow";
_rowEventType = statementContext.EventAdapterService.CreateAnonymousMapType(rowEventTypeName, rowTypeDef, true);
// validate partition-by expressions, if any
if (!matchRecognizeSpec.PartitionByExpressions.IsEmpty())
{
var typeServicePartition = new StreamTypeServiceImpl(parentViewType, "MATCH_RECOGNIZE_PARTITION", true, statementContext.EngineURI);
var validated = new List <ExprNode>();
var validationContext = new ExprValidationContext(
typeServicePartition,
statementContext.EngineImportService,
statementContext.StatementExtensionServicesContext, null,
statementContext.SchedulingService, statementContext.VariableService, statementContext.TableService,
exprEvaluatorContext, statementContext.EventAdapterService, statementContext.StatementName,
statementContext.StatementId, statementContext.Annotations, statementContext.ContextDescriptor,
statementContext.ScriptingService,
false, false, true, false, null, false);
foreach (var partitionExpr in matchRecognizeSpec.PartitionByExpressions)
{
validated.Add(ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.MATCHRECOGPARTITION, partitionExpr, validationContext));
}
matchRecognizeSpec.PartitionByExpressions = validated;
}
// validate interval if present
if (matchRecognizeSpec.Interval != null)
{
var validationContext =
new ExprValidationContext(
new StreamTypeServiceImpl(statementContext.EngineURI, false),
statementContext.EngineImportService,
statementContext.StatementExtensionServicesContext, null,
statementContext.SchedulingService,
statementContext.VariableService, statementContext.TableService, exprEvaluatorContext,
statementContext.EventAdapterService, statementContext.StatementName, statementContext.StatementId,
statementContext.Annotations, statementContext.ContextDescriptor, statementContext.ScriptingService,
false, false, true, false, null, false);
matchRecognizeSpec.Interval.Validate(validationContext);
}
}
private static void HandleSubselectSelectClauses(
int subselectStreamNumber,
StatementContext statementContext,
ExprSubselectNode subselect,
EventType outerEventType,
string outerEventTypeName,
string outerStreamName,
IDictionary <string, Pair <EventType, string> > taggedEventTypes,
IDictionary <string, Pair <EventType, string> > arrayEventTypes)
{
var statementSpec = subselect.StatementSpecCompiled;
var filterStreamSpec = statementSpec.StreamSpecs[0];
ViewFactoryChain viewFactoryChain;
string subselecteventTypeName = null;
// construct view factory chain
try {
if (statementSpec.StreamSpecs[0] is FilterStreamSpecCompiled)
{
var filterStreamSpecCompiled = (FilterStreamSpecCompiled)statementSpec.StreamSpecs[0];
subselecteventTypeName = filterStreamSpecCompiled.FilterSpec.FilterForEventTypeName;
// A child view is required to limit the stream
if (filterStreamSpec.ViewSpecs.Length == 0)
{
throw new ExprValidationException("Subqueries require one or more views to limit the stream, consider declaring a length or time window");
}
// Register filter, create view factories
viewFactoryChain = statementContext.ViewService.CreateFactories(subselectStreamNumber, filterStreamSpecCompiled.FilterSpec.ResultEventType, filterStreamSpec.ViewSpecs, filterStreamSpec.Options, statementContext);
subselect.RawEventType = viewFactoryChain.EventType;
}
else
{
var namedSpec = (NamedWindowConsumerStreamSpec)statementSpec.StreamSpecs[0];
var processor = statementContext.NamedWindowService.GetProcessor(namedSpec.WindowName);
viewFactoryChain = statementContext.ViewService.CreateFactories(0, processor.NamedWindowType, namedSpec.ViewSpecs, namedSpec.Options, statementContext);
subselecteventTypeName = namedSpec.WindowName;
EPLValidationUtil.ValidateContextName(false, processor.NamedWindowName, processor.ContextName, statementContext.ContextName, true);
}
}
catch (ViewProcessingException ex) {
throw new ExprValidationException("Error validating subexpression: " + ex.Message, ex);
}
// the final event type
var eventType = viewFactoryChain.EventType;
// determine a stream name unless one was supplied
var subexpressionStreamName = filterStreamSpec.OptionalStreamName;
if (subexpressionStreamName == null)
{
subexpressionStreamName = "$subselect_" + subselectStreamNumber;
}
// Named windows don't allow data views
if (filterStreamSpec is NamedWindowConsumerStreamSpec)
{
EPStatementStartMethodHelperValidate.ValidateNoDataWindowOnNamedWindow(viewFactoryChain.FactoryChain);
}
// Streams event types are the original stream types with the stream zero the subselect stream
var namesAndTypes = new LinkedHashMap <string, Pair <EventType, string> >();
namesAndTypes.Put(subexpressionStreamName, new Pair <EventType, string>(eventType, subselecteventTypeName));
namesAndTypes.Put(outerStreamName, new Pair <EventType, string>(outerEventType, outerEventTypeName));
if (taggedEventTypes != null)
{
foreach (KeyValuePair <string, Pair <EventType, string> > entry in taggedEventTypes)
{
namesAndTypes.Put(entry.Key, new Pair <EventType, string>(entry.Value.First, entry.Value.Second));
}
}
if (arrayEventTypes != null)
{
foreach (KeyValuePair <string, Pair <EventType, string> > entry in arrayEventTypes)
{
namesAndTypes.Put(entry.Key, new Pair <EventType, string>(entry.Value.First, entry.Value.Second));
}
}
StreamTypeService subselectTypeService = new StreamTypeServiceImpl(namesAndTypes, statementContext.EngineURI, true, true);
var viewResourceDelegateSubselect = new ViewResourceDelegateUnverified();
subselect.FilterSubqueryStreamTypes = subselectTypeService;
// Validate select expression
var selectClauseSpec = subselect.StatementSpecCompiled.SelectClauseSpec;
if (selectClauseSpec.SelectExprList.Length > 0)
{
if (selectClauseSpec.SelectExprList.Length > 1)
{
throw new ExprValidationException("Subquery multi-column select is not allowed in this context.");
}
var element = selectClauseSpec.SelectExprList[0];
if (element is SelectClauseExprCompiledSpec)
{
// validate
var compiled = (SelectClauseExprCompiledSpec)element;
var selectExpression = compiled.SelectExpression;
var evaluatorContextStmt = new ExprEvaluatorContextStatement(statementContext, false);
var validationContext = new ExprValidationContext(
subselectTypeService, statementContext.MethodResolutionService, viewResourceDelegateSubselect,
statementContext.SchedulingService, statementContext.VariableService, statementContext.TableService,
evaluatorContextStmt, statementContext.EventAdapterService, statementContext.StatementName,
statementContext.StatementId, statementContext.Annotations, statementContext.ContextDescriptor,
statementContext.ScriptingService,
false, false, true, false, null, false);
selectExpression = ExprNodeUtility.GetValidatedSubtree(ExprNodeOrigin.SUBQUERYSELECT, selectExpression, validationContext);
subselect.SelectClause = new ExprNode[] { selectExpression };
subselect.SelectAsNames = new string[] { compiled.AssignedName };
// handle aggregation
var aggExprNodes = new List <ExprAggregateNode>();
ExprAggregateNodeUtil.GetAggregatesBottomUp(selectExpression, aggExprNodes);
if (aggExprNodes.Count > 0)
{
// Other stream properties, if there is aggregation, cannot be under aggregation.
foreach (var aggNode in aggExprNodes)
{
var propertiesNodesAggregated = ExprNodeUtility.GetExpressionProperties(aggNode, true);
foreach (var pair in propertiesNodesAggregated)
{
if (pair.First != 0)
{
throw new ExprValidationException("Subselect aggregation function cannot aggregate across correlated properties");
}
}
}
// This stream (stream 0) properties must either all be under aggregation, or all not be.
var propertiesNotAggregated = ExprNodeUtility.GetExpressionProperties(selectExpression, false);
foreach (var pair in propertiesNotAggregated)
{
if (pair.First == 0)
{
throw new ExprValidationException("Subselect properties must all be within aggregation functions");
}
}
}
}
}
}
public static RowRecogPlan ValidateAndPlan(
IContainer container,
EventType parentEventType,
bool unbound,
StatementBaseInfo @base,
StatementCompileTimeServices services)
{
var statementRawInfo = @base.StatementRawInfo;
var matchRecognizeSpec = @base.StatementSpec.Raw.MatchRecognizeSpec;
var annotations = statementRawInfo.Annotations;
var iterateOnly = HintEnum.ITERATE_ONLY.GetHint(annotations) != null;
var additionalForgeables = new List<StmtClassForgeableFactory>();
// Expanded pattern already there
RowRecogExprNode expandedPatternNode = matchRecognizeSpec.Pattern;
// Determine single-row and multiple-row variables
var variablesSingle = new LinkedHashSet<string>();
var variablesMultiple = new LinkedHashSet<string>();
RowRecogHelper.RecursiveInspectVariables(expandedPatternNode, false, variablesSingle, variablesMultiple);
// each variable gets associated with a stream number (multiple-row variables as well to hold the current event for the expression).
var streamNum = 0;
var variableStreams = new LinkedHashMap<string, Pair<int, bool>>();
foreach (var variableSingle in variablesSingle) {
variableStreams.Put(variableSingle, new Pair<int, bool>(streamNum, false));
streamNum++;
}
foreach (var variableMultiple in variablesMultiple) {
variableStreams.Put(variableMultiple, new Pair<int, bool>(streamNum, true));
streamNum++;
}
// mapping of stream to variable
var streamVariables = new OrderedListDictionary<int, string>();
foreach (var entry in variableStreams) {
streamVariables.Put(entry.Value.First, entry.Key);
}
// determine visibility rules
var visibility = RowRecogHelper.DetermineVisibility(expandedPatternNode);
// assemble all single-row variables for expression validation
var allStreamNames = new string[variableStreams.Count];
var allTypes = new EventType[variableStreams.Count];
streamNum = 0;
foreach (var variableSingle in variablesSingle) {
allStreamNames[streamNum] = variableSingle;
allTypes[streamNum] = parentEventType;
streamNum++;
}
foreach (var variableMultiple in variablesMultiple) {
allStreamNames[streamNum] = variableMultiple;
allTypes[streamNum] = parentEventType;
streamNum++;
}
// determine type service for use with DEFINE
// validate each DEFINE clause expression
ISet<string> definedVariables = new HashSet<string>();
IList<ExprAggregateNode> aggregateNodes = new List<ExprAggregateNode>();
var isExprRequiresMultimatchState = new bool[variableStreams.Count];
var previousNodes = new OrderedListDictionary<int, IList<ExprPreviousMatchRecognizeNode>>();
for (var defineIndex = 0; defineIndex < matchRecognizeSpec.Defines.Count; defineIndex++) {
MatchRecognizeDefineItem defineItem = matchRecognizeSpec.Defines[defineIndex];
if (definedVariables.Contains(defineItem.Identifier)) {
throw new ExprValidationException(
"Variable '" + defineItem.Identifier + "' has already been defined");
}
definedVariables.Add(defineItem.Identifier);
// stream-type visibilities handled here
var typeServiceDefines = BuildDefineStreamTypeServiceDefine(
defineIndex,
variableStreams,
defineItem,
visibility,
parentEventType,
statementRawInfo,
services);
var exprNodeResult = HandlePreviousFunctions(defineItem.Expression, previousNodes);
var validationContext = new ExprValidationContextBuilder(typeServiceDefines, statementRawInfo, services)
.WithAllowBindingConsumption(true)
.WithDisablePropertyExpressionEventCollCache(true)
.Build();
ExprNode validated;
try {
// validate
validated = ExprNodeUtilityValidate.GetValidatedSubtree(
ExprNodeOrigin.MATCHRECOGDEFINE,
exprNodeResult,
validationContext);
// check aggregates
defineItem.Expression = validated;
ExprAggregateNodeUtil.GetAggregatesBottomUp(validated, aggregateNodes);
if (!aggregateNodes.IsEmpty()) {
throw new ExprValidationException("An aggregate function may not appear in a DEFINE clause");
}
}
catch (ExprValidationException ex) {
throw new ExprValidationException(
"Failed to validate condition expression for variable '" +
defineItem.Identifier +
"': " +
ex.Message,
ex);
}
// determine access to event properties from multi-matches
var visitor = new ExprNodeStreamRequiredVisitor();
validated.Accept(visitor);
var streamsRequired = visitor.StreamsRequired;
foreach (var streamRequired in streamsRequired) {
if (streamRequired >= variableStreams.Count) {
var streamNumIdent = variableStreams.Get(defineItem.Identifier).First;
isExprRequiresMultimatchState[streamNumIdent] = true;
break;
}
}
}
var defineAsksMultimatches = CollectionUtil.IsAnySet(isExprRequiresMultimatchState);
// determine type service for use with MEASURE
IDictionary<string, object> measureTypeDef = new LinkedHashMap<string, object>();
foreach (var variableSingle in variablesSingle) {
measureTypeDef.Put(variableSingle, parentEventType);
}
foreach (var variableMultiple in variablesMultiple) {
measureTypeDef.Put(variableMultiple, new[] {parentEventType});
}
var compositeTypeName = services.EventTypeNameGeneratorStatement.AnonymousRowrecogCompositeName;
var compositeTypeMetadata = new EventTypeMetadata(
compositeTypeName,
@base.ModuleName,
EventTypeTypeClass.MATCHRECOGDERIVED,
EventTypeApplicationType.OBJECTARR,
NameAccessModifier.TRANSIENT,
EventTypeBusModifier.NONBUS,
false,
EventTypeIdPair.Unassigned());
var compositeEventType = BaseNestableEventUtil.MakeOATypeCompileTime(
compositeTypeMetadata,
measureTypeDef,
null,
null,
null,
null,
services.BeanEventTypeFactoryPrivate,
services.EventTypeCompileTimeResolver);
services.EventTypeCompileTimeRegistry.NewType(compositeEventType);
StreamTypeService compositeTypeServiceMeasure =
new StreamTypeServiceImpl(compositeEventType, "MATCH_RECOGNIZE", true);
// find MEASURE clause aggregations
var measureReferencesMultivar = false;
IList<ExprAggregateNode> measureAggregateExprNodes = new List<ExprAggregateNode>();
foreach (var measureItem in matchRecognizeSpec.Measures) {
ExprAggregateNodeUtil.GetAggregatesBottomUp(measureItem.Expr, measureAggregateExprNodes);
}
AggregationServiceForgeDesc[] aggregationServices = null;
if (!measureAggregateExprNodes.IsEmpty()) {
aggregationServices = PlanAggregations(
measureAggregateExprNodes,
compositeTypeServiceMeasure,
allStreamNames,
allTypes,
streamVariables,
variablesMultiple,
@base,
services);
foreach (AggregationServiceForgeDesc svc in aggregationServices) {
if (svc != null) {
additionalForgeables.AddAll(svc.AdditionalForgeables);
}
}
}
// validate each MEASURE clause expression
IDictionary<string, object> rowTypeDef = new LinkedHashMap<string, object>();
var streamRefVisitor = new ExprNodeStreamUseCollectVisitor();
foreach (var measureItem in matchRecognizeSpec.Measures) {
if (measureItem.Name == null) {
throw new ExprValidationException(
"The measures clause requires that each expression utilizes the AS keyword to assign a column name");
}
var validated = ValidateMeasureClause(
measureItem.Expr,
compositeTypeServiceMeasure,
variablesMultiple,
variablesSingle,
statementRawInfo,
services);
measureItem.Expr = validated;
rowTypeDef.Put(measureItem.Name, validated.Forge.EvaluationType);
validated.Accept(streamRefVisitor);
}
// Determine if any of the multi-var streams are referenced in the measures (non-aggregated only)
foreach (var @ref in streamRefVisitor.Referenced) {
var rootPropName = @ref.RootPropertyNameIfAny;
if (rootPropName != null) {
if (variablesMultiple.Contains(rootPropName)) {
measureReferencesMultivar = true;
break;
}
}
var streamRequired = @ref.StreamReferencedIfAny;
if (streamRequired != null) {
var streamVariable = streamVariables.Get(streamRequired.Value);
if (streamVariable != null) {
var def = variableStreams.Get(streamVariable);
if (def != null && def.Second) {
measureReferencesMultivar = true;
break;
}
}
}
}
var collectMultimatches = measureReferencesMultivar || defineAsksMultimatches;
// create rowevent type
var rowTypeName = services.EventTypeNameGeneratorStatement.AnonymousRowrecogRowName;
var rowTypeMetadata = new EventTypeMetadata(
rowTypeName,
@base.ModuleName,
EventTypeTypeClass.MATCHRECOGDERIVED,
EventTypeApplicationType.MAP,
NameAccessModifier.TRANSIENT,
EventTypeBusModifier.NONBUS,
false,
EventTypeIdPair.Unassigned());
var rowEventType = BaseNestableEventUtil.MakeMapTypeCompileTime(
rowTypeMetadata,
rowTypeDef,
null,
null,
null,
null,
services.BeanEventTypeFactoryPrivate,
services.EventTypeCompileTimeResolver);
services.EventTypeCompileTimeRegistry.NewType(rowEventType);
// validate partition-by expressions, if any
ExprNode[] partitionBy;
MultiKeyClassRef partitionMultiKey;
if (!matchRecognizeSpec.PartitionByExpressions.IsEmpty()) {
StreamTypeService typeServicePartition = new StreamTypeServiceImpl(
parentEventType,
"MATCH_RECOGNIZE_PARTITION",
true);
IList<ExprNode> validated = new List<ExprNode>();
var validationContext =
new ExprValidationContextBuilder(typeServicePartition, statementRawInfo, services)
.WithAllowBindingConsumption(true)
.Build();
foreach (var partitionExpr in matchRecognizeSpec.PartitionByExpressions) {
validated.Add(
ExprNodeUtilityValidate.GetValidatedSubtree(
ExprNodeOrigin.MATCHRECOGPARTITION,
partitionExpr,
validationContext));
}
matchRecognizeSpec.PartitionByExpressions = validated;
partitionBy = ExprNodeUtilityQuery.ToArray(validated);
MultiKeyPlan multiKeyPlan = MultiKeyPlanner.PlanMultiKey(partitionBy, false, @base.StatementRawInfo, services.SerdeResolver);
partitionMultiKey = multiKeyPlan.ClassRef;
additionalForgeables.AddAll(multiKeyPlan.MultiKeyForgeables);
}
else {
partitionBy = null;
partitionMultiKey = null;
}
// validate interval if present
if (matchRecognizeSpec.Interval != null) {
var validationContext =
new ExprValidationContextBuilder(new StreamTypeServiceImpl(false), statementRawInfo, services)
.WithAllowBindingConsumption(true)
.Build();
var validated = (ExprTimePeriod) ExprNodeUtilityValidate.GetValidatedSubtree(
ExprNodeOrigin.MATCHRECOGINTERVAL,
matchRecognizeSpec.Interval.TimePeriodExpr,
validationContext);
matchRecognizeSpec.Interval.TimePeriodExpr = validated;
}
// compile variable definition expressions
IDictionary<string, ExprNode> variableDefinitions = new Dictionary<string, ExprNode>();
foreach (var defineItem in matchRecognizeSpec.Defines) {
variableDefinitions.Put(defineItem.Identifier, defineItem.Expression);
}
// create evaluators
var columnNames = new string[matchRecognizeSpec.Measures.Count];
var columnForges = new ExprNode[matchRecognizeSpec.Measures.Count];
var count = 0;
foreach (var measureItem in matchRecognizeSpec.Measures) {
columnNames[count] = measureItem.Name;
columnForges[count] = measureItem.Expr;
count++;
}
// build states
var strand = RowRecogHelper.BuildStartStates(
expandedPatternNode,
variableDefinitions,
variableStreams,
isExprRequiresMultimatchState);
var startStates = strand.StartStates.ToArray();
RowRecogNFAStateForge[] allStates = strand.AllStates.ToArray();
if (Log.IsInfoEnabled) {
Log.Info("NFA tree:\n" + RowRecogNFAViewUtil.Print(startStates));
}
// determine names of multimatching variables
string[] multimatchVariablesArray;
int[] multimatchStreamNumToVariable;
int[] multimatchVariableToStreamNum;
if (variablesSingle.Count == variableStreams.Count) {
multimatchVariablesArray = new string[0];
multimatchStreamNumToVariable = new int[0];
multimatchVariableToStreamNum = new int[0];
}
else {
multimatchVariablesArray = new string[variableStreams.Count - variablesSingle.Count];
multimatchVariableToStreamNum = new int[multimatchVariablesArray.Length];
multimatchStreamNumToVariable = new int[variableStreams.Count];
CompatExtensions.Fill(multimatchStreamNumToVariable, -1);
count = 0;
foreach (var entry in variableStreams) {
if (entry.Value.Second) {
var index = count;
multimatchVariablesArray[index] = entry.Key;
multimatchVariableToStreamNum[index] = entry.Value.First;
multimatchStreamNumToVariable[entry.Value.First] = index;
count++;
}
}
}
var numEventsEventsPerStreamDefine =
defineAsksMultimatches ? variableStreams.Count + 1 : variableStreams.Count;
// determine interval-or-terminated
var orTerminated = matchRecognizeSpec.Interval != null && matchRecognizeSpec.Interval.IsOrTerminated;
TimePeriodComputeForge intervalCompute = null;
if (matchRecognizeSpec.Interval != null) {
intervalCompute = matchRecognizeSpec.Interval.TimePeriodExpr.TimePeriodComputeForge;
}
EventType multimatchEventType = null;
if (defineAsksMultimatches) {
multimatchEventType = GetDefineMultimatchEventType(variableStreams, parentEventType, @base, services);
}
// determine previous-access indexes and assign "prev" node indexes
// Since an expression such as "prior(2, price), prior(8, price)" translates into {2, 8} the relative index is {0, 1}.
// Map the expression-supplied index to a relative index
int[] previousRandomAccessIndexes = null;
if (!previousNodes.IsEmpty()) {
previousRandomAccessIndexes = new int[previousNodes.Count];
var countPrev = 0;
foreach (var entry in previousNodes) {
previousRandomAccessIndexes[countPrev] = entry.Key;
foreach (var callback in entry.Value) {
callback.AssignedIndex = countPrev;
}
countPrev++;
}
}
RowRecogDescForge forge = new RowRecogDescForge(
parentEventType,
rowEventType,
compositeEventType,
multimatchEventType,
multimatchStreamNumToVariable,
multimatchVariableToStreamNum,
partitionBy,
partitionMultiKey,
variableStreams,
matchRecognizeSpec.Interval != null,
iterateOnly,
unbound,
orTerminated,
collectMultimatches,
defineAsksMultimatches,
numEventsEventsPerStreamDefine,
multimatchVariablesArray,
startStates,
allStates,
matchRecognizeSpec.IsAllMatches,
matchRecognizeSpec.Skip.Skip,
columnForges,
columnNames,
intervalCompute,
previousRandomAccessIndexes,
aggregationServices);
return new RowRecogPlan(forge, additionalForgeables);
}
public static IList <StmtClassForgeableFactory> HandleSubselectSelectClauses(
ExprSubselectNode subselect,
EventType outerEventType,
string outerEventTypeName,
string outerStreamName,
IDictionary <string, Pair <EventType, string> > taggedEventTypes,
IDictionary <string, Pair <EventType, string> > arrayEventTypes,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
if (subselect.SubselectNumber == -1)
{
throw new IllegalStateException("Subselect is unassigned");
}
var statementSpec = subselect.StatementSpecCompiled;
var filterStreamSpec = statementSpec.StreamSpecs[0];
IList <ViewFactoryForge> viewForges;
string subselecteventTypeName;
IList <StmtClassForgeableFactory> additionalForgeables;
// construct view factory chain
EventType eventType;
try {
var args = new ViewFactoryForgeArgs(
-1,
true,
subselect.SubselectNumber,
StreamSpecOptions.DEFAULT,
null,
statementRawInfo,
services);
var streamSpec = statementSpec.StreamSpecs[0];
if (streamSpec is FilterStreamSpecCompiled)
{
var filterStreamSpecCompiled = (FilterStreamSpecCompiled)statementSpec.StreamSpecs[0];
subselecteventTypeName = filterStreamSpecCompiled.FilterSpecCompiled.FilterForEventTypeName;
// A child view is required to limit the stream
if (filterStreamSpec.ViewSpecs.Length == 0)
{
throw new ExprValidationException(
"Subqueries require one or more views to limit the stream, consider declaring a length or time window");
}
ViewFactoryForgeDesc viewForgeDesc = ViewFactoryForgeUtil.CreateForges(
filterStreamSpecCompiled.ViewSpecs,
args,
filterStreamSpecCompiled.FilterSpecCompiled.ResultEventType);
viewForges = viewForgeDesc.Forges;
additionalForgeables = viewForgeDesc.MultikeyForges;
// Register filter, create view factories
eventType = viewForges.IsEmpty() ? filterStreamSpecCompiled.FilterSpecCompiled.ResultEventType : viewForges[viewForges.Count - 1].EventType;
subselect.RawEventType = eventType;
}
else if (streamSpec is NamedWindowConsumerStreamSpec)
{
var namedSpec = (NamedWindowConsumerStreamSpec)statementSpec.StreamSpecs[0];
var namedWindow = namedSpec.NamedWindow;
ViewFactoryForgeDesc viewForgeDesc = ViewFactoryForgeUtil.CreateForges(namedSpec.ViewSpecs, args, namedWindow.EventType);
viewForges = viewForgeDesc.Forges;
additionalForgeables = viewForgeDesc.MultikeyForges;
var namedWindowName = namedWindow.EventType.Name;
subselecteventTypeName = namedWindowName;
EPLValidationUtil.ValidateContextName(false, namedWindowName, namedWindow.ContextName, statementRawInfo.ContextName, true);
subselect.RawEventType = namedWindow.EventType;
eventType = namedWindow.EventType;
}
else if (streamSpec is TableQueryStreamSpec)
{
var namedSpec = (TableQueryStreamSpec)statementSpec.StreamSpecs[0];
var table = namedSpec.Table;
ViewFactoryForgeDesc viewForgeDesc = ViewFactoryForgeUtil.CreateForges(namedSpec.ViewSpecs, args, table.InternalEventType);
viewForges = viewForgeDesc.Forges;
additionalForgeables = viewForgeDesc.MultikeyForges;
var namedWindowName = table.TableName;
subselecteventTypeName = namedWindowName;
EPLValidationUtil.ValidateContextName(false, namedWindowName, table.OptionalContextName, statementRawInfo.ContextName, true);
subselect.RawEventType = table.InternalEventType;
eventType = table.InternalEventType;
}
else
{
throw new IllegalStateException("Unexpected stream spec " + streamSpec);
}
}
catch (ViewProcessingException ex) {
throw new ExprValidationException("Failed to validate subexpression: " + ex.Message, ex);
}
// determine a stream name unless one was supplied
var subexpressionStreamName = SubselectUtil.GetStreamName(filterStreamSpec.OptionalStreamName, subselect.SubselectNumber);
// Named windows don't allow data views
if (filterStreamSpec is NamedWindowConsumerStreamSpec | filterStreamSpec is TableQueryStreamSpec)
{
EPStatementStartMethodHelperValidate.ValidateNoDataWindowOnNamedWindow(viewForges);
}
// Streams event types are the original stream types with the stream zero the subselect stream
var namesAndTypes = new LinkedHashMap <string, Pair <EventType, string> >();
namesAndTypes.Put(subexpressionStreamName, new Pair <EventType, string>(eventType, subselecteventTypeName));
namesAndTypes.Put(outerStreamName, new Pair <EventType, string>(outerEventType, outerEventTypeName));
if (taggedEventTypes != null)
{
foreach (var entry in taggedEventTypes)
{
namesAndTypes.Put(entry.Key, new Pair <EventType, string>(entry.Value.First, entry.Value.Second));
}
}
if (arrayEventTypes != null)
{
foreach (var entry in arrayEventTypes)
{
namesAndTypes.Put(entry.Key, new Pair <EventType, string>(entry.Value.First, entry.Value.Second));
}
}
StreamTypeService subselectTypeService = new StreamTypeServiceImpl(namesAndTypes, true, true);
var viewResourceDelegateSubselect = new ViewResourceDelegateExpr();
subselect.FilterSubqueryStreamTypes = subselectTypeService;
// Validate select expression
var selectClauseSpec = subselect.StatementSpecCompiled.SelectClauseCompiled;
if (selectClauseSpec.SelectExprList.Length > 0)
{
if (selectClauseSpec.SelectExprList.Length > 1)
{
throw new ExprValidationException("Subquery multi-column select is not allowed in this context.");
}
var element = selectClauseSpec.SelectExprList[0];
if (element is SelectClauseExprCompiledSpec)
{
// validate
var compiled = (SelectClauseExprCompiledSpec)element;
var selectExpression = compiled.SelectExpression;
var validationContext = new ExprValidationContextBuilder(subselectTypeService, statementRawInfo, services)
.WithViewResourceDelegate(viewResourceDelegateSubselect)
.WithAllowBindingConsumption(true)
.WithMemberName(new ExprValidationMemberNameQualifiedSubquery(subselect.SubselectNumber))
.Build();
selectExpression = ExprNodeUtilityValidate.GetValidatedSubtree(ExprNodeOrigin.SUBQUERYSELECT, selectExpression, validationContext);
subselect.SelectClause = new ExprNode[] { selectExpression };
subselect.SelectAsNames = new string[] { compiled.AssignedName };
// handle aggregation
var aggExprNodes = new List <ExprAggregateNode>();
ExprAggregateNodeUtil.GetAggregatesBottomUp(selectExpression, aggExprNodes);
if (aggExprNodes.Count > 0)
{
// Other stream properties, if there is aggregation, cannot be under aggregation.
foreach (var aggNode in aggExprNodes)
{
var propertiesNodesAggregated = ExprNodeUtilityQuery.GetExpressionProperties(aggNode, true);
foreach (var pair in propertiesNodesAggregated)
{
if (pair.First != 0)
{
throw new ExprValidationException("Subselect aggregation function cannot aggregate across correlated properties");
}
}
}
// This stream (stream 0) properties must either all be under aggregation, or all not be.
var propertiesNotAggregated = ExprNodeUtilityQuery.GetExpressionProperties(selectExpression, false);
foreach (var pair in propertiesNotAggregated)
{
if (pair.First == 0)
{
throw new ExprValidationException("Subselect properties must all be within aggregation functions");
}
}
}
}
}
return(additionalForgeables);
}
public void Attach(EventType parentEventType, StatementContext statementContext, ViewFactory optionalParentFactory, IList <ViewFactory> parentViewFactories)
{
_eventType = parentEventType;
// define built-in fields
var builtinTypeDef = ExpressionViewOAFieldEnumExtensions.AsMapOfTypes(_eventType);
_builtinMapType = statementContext.EventAdapterService.CreateAnonymousObjectArrayType(
statementContext.StatementId + "_exprview", builtinTypeDef);
StreamTypeService streamTypeService = new StreamTypeServiceImpl(new EventType[] { _eventType, _builtinMapType }, new String[2], new bool[2], statementContext.EngineURI, false);
// validate expression
ExpiryExpression = ViewFactorySupport.ValidateExpr(ViewName, statementContext, ExpiryExpression, streamTypeService, 0);
var summaryVisitor = new ExprNodeSummaryVisitor();
ExpiryExpression.Accept(summaryVisitor);
if (summaryVisitor.HasSubselect || summaryVisitor.HasStreamSelect || summaryVisitor.HasPreviousPrior)
{
throw new ViewParameterException("Invalid expiry expression: Sub-select, previous or prior functions are not supported in this context");
}
var returnType = ExpiryExpression.ExprEvaluator.ReturnType;
if (returnType.GetBoxedType() != typeof(bool?))
{
throw new ViewParameterException("Invalid return value for expiry expression, expected a bool return value but received " + returnType.GetParameterAsString());
}
// determine variables used, if any
var visitor = new ExprNodeVariableVisitor();
ExpiryExpression.Accept(visitor);
VariableNames = visitor.VariableNames;
// determine aggregation nodes, if any
var aggregateNodes = new List <ExprAggregateNode>();
ExprAggregateNodeUtil.GetAggregatesBottomUp(ExpiryExpression, aggregateNodes);
if (aggregateNodes.IsNotEmpty())
{
try {
AggregationServiceFactoryDesc = AggregationServiceFactoryFactory.GetService(
Collections.GetEmptyList <ExprAggregateNode>(),
Collections.GetEmptyMap <ExprNode, String>(),
Collections.GetEmptyList <ExprDeclaredNode>(),
null, aggregateNodes,
Collections.GetEmptyList <ExprAggregateNode>(),
Collections.GetEmptyList <ExprAggregateNodeGroupKey>(), false,
statementContext.Annotations,
statementContext.VariableService, false, false, null, null,
AggregationServiceFactoryServiceImpl.DEFAULT_FACTORY,
streamTypeService.EventTypes,
statementContext.MethodResolutionService, null,
statementContext.ContextName,
null, null);
}
catch (ExprValidationException ex) {
throw new ViewParameterException(ex.Message, ex);
}
}
}
public static ExprNode ValidateNodes(
StatementSpecRaw statementSpec,
StreamTypeService typeService,
ViewResourceDelegateExpr viewResourceDelegate,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices compileTimeServices)
{
var intoTableName = statementSpec.IntoTableSpec == null ? null : statementSpec.IntoTableSpec.Name;
ExprNode whereClauseValidated = null;
if (statementSpec.WhereClause != null) {
var whereClause = statementSpec.WhereClause;
// Validate where clause, initializing nodes to the stream ids used
try {
var validationContext = new ExprValidationContextBuilder(
typeService,
statementRawInfo,
compileTimeServices)
.WithViewResourceDelegate(viewResourceDelegate)
.WithAllowBindingConsumption(true)
.WithIntoTableName(intoTableName)
.Build();
whereClause = ExprNodeUtilityValidate.GetValidatedSubtree(
ExprNodeOrigin.FILTER,
whereClause,
validationContext);
if (whereClause.Forge.EvaluationType != typeof(bool) &&
whereClause.Forge.EvaluationType != typeof(bool?)) {
throw new ExprValidationException(
"The where-clause filter expression must return a boolean value");
}
whereClauseValidated = whereClause;
// Make sure there is no aggregation in the where clause
IList<ExprAggregateNode> aggregateNodes = new List<ExprAggregateNode>();
ExprAggregateNodeUtil.GetAggregatesBottomUp(whereClause, aggregateNodes);
if (!aggregateNodes.IsEmpty()) {
throw new ExprValidationException(
"An aggregate function may not appear in a WHERE clause (use the HAVING clause)");
}
}
catch (ExprValidationException ex) {
throw new ExprValidationException("Failed to validate expression: " + ex.Message, ex);
}
}
if ((statementSpec.OutputLimitSpec != null) &&
(statementSpec.OutputLimitSpec.WhenExpressionNode != null ||
statementSpec.OutputLimitSpec.AndAfterTerminateExpr != null ||
statementSpec.OutputLimitSpec.AndAfterTerminateThenExpressions != null)) {
// Validate where clause, initializing nodes to the stream ids used
EventType outputLimitType = OutputConditionExpressionTypeUtil.GetBuiltInEventType(
statementRawInfo.ModuleName,
compileTimeServices.BeanEventTypeFactoryPrivate);
StreamTypeService typeServiceOutputWhen = new StreamTypeServiceImpl(
new[] {outputLimitType},
new string[] {null},
new[] {true},
false,
false);
var validationContext = new ExprValidationContextBuilder(
typeServiceOutputWhen,
statementRawInfo,
compileTimeServices)
.WithIntoTableName(intoTableName)
.Build();
var outputLimitWhenNode = statementSpec.OutputLimitSpec.WhenExpressionNode;
if (outputLimitWhenNode != null) {
outputLimitWhenNode = ExprNodeUtilityValidate.GetValidatedSubtree(
ExprNodeOrigin.OUTPUTLIMIT,
outputLimitWhenNode,
validationContext);
statementSpec.OutputLimitSpec.WhenExpressionNode = outputLimitWhenNode;
if (outputLimitWhenNode.Forge.EvaluationType.GetBoxedType() != typeof(bool?)) {
throw new ExprValidationException(
"The when-trigger expression in the OUTPUT WHEN clause must return a boolean-type value");
}
ValidateNoAggregations(
outputLimitWhenNode,
"An aggregate function may not appear in a OUTPUT LIMIT clause");
}
// validate and-terminate expression if provided
if (statementSpec.OutputLimitSpec.AndAfterTerminateExpr != null) {
if (statementSpec.OutputLimitSpec.RateType != OutputLimitRateType.WHEN_EXPRESSION &&
statementSpec.OutputLimitSpec.RateType != OutputLimitRateType.TERM) {
throw new ExprValidationException(
"A terminated-and expression must be used with the OUTPUT WHEN clause");
}
var validated = ExprNodeUtilityValidate.GetValidatedSubtree(
ExprNodeOrigin.OUTPUTLIMIT,
statementSpec.OutputLimitSpec.AndAfterTerminateExpr,
validationContext);
statementSpec.OutputLimitSpec.AndAfterTerminateExpr = validated;
if (validated.Forge.EvaluationType.GetBoxedType() != typeof(bool?)) {
throw new ExprValidationException(
"The terminated-and expression must return a boolean-type value");
}
ValidateNoAggregations(
validated,
"An aggregate function may not appear in a terminated-and clause");
}
// validate then-expression
ValidateThenSetAssignments(statementSpec.OutputLimitSpec.ThenExpressions, validationContext, false);
// validate after-terminated then-expression
ValidateThenSetAssignments(statementSpec.OutputLimitSpec.AndAfterTerminateThenExpressions, validationContext, false);
}
for (var outerJoinCount = 0; outerJoinCount < statementSpec.OuterJoinDescList.Count; outerJoinCount++) {
var outerJoinDesc = statementSpec.OuterJoinDescList[outerJoinCount];
// validate on-expression nodes, if provided
if (outerJoinDesc.OptLeftNode != null) {
var streamIdPair = ValidateOuterJoinPropertyPair(
outerJoinDesc.OptLeftNode,
outerJoinDesc.OptRightNode,
outerJoinCount,
typeService,
viewResourceDelegate,
statementRawInfo,
compileTimeServices
);
if (outerJoinDesc.AdditionalLeftNodes != null) {
ISet<int> streamSet = new HashSet<int>();
streamSet.Add(streamIdPair.First);
streamSet.Add(streamIdPair.Second);
for (var i = 0; i < outerJoinDesc.AdditionalLeftNodes.Length; i++) {
var streamIdPairAdd = ValidateOuterJoinPropertyPair(
outerJoinDesc.AdditionalLeftNodes[i],
outerJoinDesc.AdditionalRightNodes[i],
outerJoinCount,
typeService,
viewResourceDelegate,
statementRawInfo,
compileTimeServices
);
// make sure all additional properties point to the same two streams
if (!streamSet.Contains(streamIdPairAdd.First) ||
!streamSet.Contains(streamIdPairAdd.Second)) {
var message =
"Outer join ON-clause columns must refer to properties of the same joined streams" +
" when using multiple columns in the on-clause";
throw new ExprValidationException("Failed to validate outer-join expression: " + message);
}
}
}
}
}
return whereClauseValidated;
}
public static OrderByProcessorFactoryForge GetProcessor(
IList<SelectClauseExprCompiledSpec> selectionList,
IList<OrderByItem> orderByList,
RowLimitSpec rowLimitSpec,
VariableCompileTimeResolver variableCompileTimeResolver,
bool isSortUsingCollator,
string optionalContextName,
OrderByElementForge[][] orderByRollup)
{
// Get the order by expression nodes
IList<ExprNode> orderByNodes = new List<ExprNode>();
foreach (var element in orderByList) {
orderByNodes.Add(element.ExprNode);
}
// No order-by clause
if (orderByList.IsEmpty()) {
Log.Debug(".getProcessor Using no OrderByProcessor");
if (rowLimitSpec != null) {
var rowLimitProcessorFactory = new RowLimitProcessorFactoryForge(
rowLimitSpec,
variableCompileTimeResolver,
optionalContextName);
return new OrderByProcessorRowLimitOnlyForge(rowLimitProcessorFactory);
}
return null;
}
// Determine aggregate functions used in select, if any
IList<ExprAggregateNode> selectAggNodes = new List<ExprAggregateNode>();
foreach (var element in selectionList) {
ExprAggregateNodeUtil.GetAggregatesBottomUp(element.SelectExpression, selectAggNodes);
}
// Get all the aggregate functions occuring in the order-by clause
IList<ExprAggregateNode> orderAggNodes = new List<ExprAggregateNode>();
foreach (var orderByNode in orderByNodes) {
ExprAggregateNodeUtil.GetAggregatesBottomUp(orderByNode, orderAggNodes);
}
ValidateOrderByAggregates(selectAggNodes, orderAggNodes);
// Tell the order-by processor whether to compute group-by
// keys if they are not present
var needsGroupByKeys = !selectionList.IsEmpty() && !orderAggNodes.IsEmpty();
Log.Debug(".getProcessor Using OrderByProcessorImpl");
var elements = ToElementArray(orderByList);
var comparator = GetComparator(elements, isSortUsingCollator);
var orderByProcessorForge = new OrderByProcessorForgeImpl(
elements,
needsGroupByKeys,
orderByRollup,
comparator);
if (rowLimitSpec == null) {
return orderByProcessorForge;
}
{
var rowLimitProcessorFactory = new RowLimitProcessorFactoryForge(
rowLimitSpec,
variableCompileTimeResolver,
optionalContextName);
return new OrderByProcessorOrderedLimitForge(orderByProcessorForge, rowLimitProcessorFactory);
}
}
