/// <summary> /// Compile a select clause allowing subselects. /// </summary> /// <param name="spec">to compile</param> /// <returns>select clause compiled</returns> /// <throws>ExprValidationException when validation fails</throws> private static SelectClauseSpecCompiled CompileSelectAllowSubselect(SelectClauseSpecRaw spec) { // Look for expressions with sub-selects in select expression list and filter expression // Recursively compile the statement within the statement. var visitor = new ExprNodeSubselectDeclaredDotVisitor(); IList<SelectClauseElementCompiled> selectElements = new List<SelectClauseElementCompiled>(); foreach (var raw in spec.SelectExprList) { if (raw is SelectClauseExprRawSpec) { var rawExpr = (SelectClauseExprRawSpec) raw; rawExpr.SelectExpression.Accept(visitor); selectElements.Add( new SelectClauseExprCompiledSpec( rawExpr.SelectExpression, rawExpr.OptionalAsName, rawExpr.OptionalAsName, rawExpr.IsEvents)); } else if (raw is SelectClauseStreamRawSpec) { var rawExpr = (SelectClauseStreamRawSpec) raw; selectElements.Add(new SelectClauseStreamCompiledSpec(rawExpr.StreamName, rawExpr.OptionalAsName)); } else if (raw is SelectClauseElementWildcard) { var wildcard = (SelectClauseElementWildcard) raw; selectElements.Add(wildcard); } else { throw new IllegalStateException("Unexpected select clause element class : " + raw.GetType().Name); } } return new SelectClauseSpecCompiled(selectElements.ToArray(), spec.IsDistinct); }
public static IList<ExprNode> ValidateAllowSubquery( ExprNodeOrigin exprNodeOrigin, IList<ExprNode> exprNodes, StreamTypeService streamTypeService, IDictionary<string, Pair<EventType, string>> taggedEventTypes, IDictionary<string, Pair<EventType, string>> arrayEventTypes, StatementRawInfo statementRawInfo, StatementCompileTimeServices services) { IList<ExprNode> validatedNodes = new List<ExprNode>(); ExprValidationContext validationContext = new ExprValidationContextBuilder(streamTypeService, statementRawInfo, services) .WithAllowBindingConsumption(true) .WithIsFilterExpression(true) .Build(); foreach (var node in exprNodes) { // Determine subselects var visitor = new ExprNodeSubselectDeclaredDotVisitor(); node.Accept(visitor); // Compile subselects if (!visitor.Subselects.IsEmpty()) { // The outer event type is the filtered-type itself foreach (var subselect in visitor.Subselects) { try { SubSelectHelperFilters.HandleSubselectSelectClauses( subselect, streamTypeService.EventTypes[0], streamTypeService.StreamNames[0], streamTypeService.StreamNames[0], taggedEventTypes, arrayEventTypes, statementRawInfo, services); } catch (ExprValidationException ex) { throw new ExprValidationException( "Failed to validate " + ExprNodeUtilityMake.GetSubqueryInfoText(subselect) + ": " + ex.Message, ex); } } } var validated = ExprNodeUtilityValidate.GetValidatedSubtree(exprNodeOrigin, node, validationContext); validatedNodes.Add(validated); if (validated.Forge.EvaluationType != typeof(bool?) && validated.Forge.EvaluationType != typeof(bool)) { throw new ExprValidationException( "Filter expression not returning a boolean value: '" + ExprNodeUtilityPrint.ToExpressionStringMinPrecedenceSafe(validated) + "'"); } } return validatedNodes; }
public static IDictionary<ExprDeclaredNode, IList<ExprDeclaredNode>> GetDeclaredExpressionCallHierarchy( ExprDeclaredNode[] declaredExpressions) { var visitor = new ExprNodeSubselectDeclaredDotVisitor(); IDictionary<ExprDeclaredNode, IList<ExprDeclaredNode>> calledToCallerMap = new Dictionary<ExprDeclaredNode, IList<ExprDeclaredNode>>(); foreach (var node in declaredExpressions) { visitor.Reset(); node.AcceptNoVisitParams(visitor); foreach (var called in visitor.DeclaredExpressions) { if (called == node) { continue; } var callers = calledToCallerMap.Get(called); if (callers == null) { callers = new List<ExprDeclaredNode>(2); calledToCallerMap.Put(called, callers); } callers.Add(node); } if (!calledToCallerMap.ContainsKey(node)) { calledToCallerMap.Put(node, Collections.GetEmptyList<ExprDeclaredNode>()); } } return calledToCallerMap; }
public static void ValidateNoSpecialsGroupByExpressions(ExprNode[] groupByNodes) { ExprNodeSubselectDeclaredDotVisitor visitorSubselects = new ExprNodeSubselectDeclaredDotVisitor(); ExprNodeGroupingVisitorWParent visitorGrouping = new ExprNodeGroupingVisitorWParent(); IList<ExprAggregateNode> aggNodesInGroupBy = new List<ExprAggregateNode>(1); foreach (ExprNode groupByNode in groupByNodes) { // no subselects groupByNode.Accept(visitorSubselects); if (visitorSubselects.Subselects.Count > 0) { throw new ExprValidationException("Subselects not allowed within group-by"); } // no special grouping-clauses groupByNode.Accept(visitorGrouping); if (!visitorGrouping.GroupingIdNodes.IsEmpty()) { throw ExprGroupingIdNode.MakeException("grouping_id"); } if (!visitorGrouping.GroupingNodes.IsEmpty()) { throw ExprGroupingIdNode.MakeException("grouping"); } // no aggregations allowed ExprAggregateNodeUtil.GetAggregatesBottomUp(groupByNode, aggNodesInGroupBy); if (!aggNodesInGroupBy.IsEmpty()) { throw new ExprValidationException("Group-by expressions cannot contain aggregate functions"); } } }
public static void WalkStreamSpecs(StatementSpecRaw spec, ExprNodeSubselectDeclaredDotVisitor visitor) { // Determine filter streams foreach (StreamSpecRaw rawSpec in spec.StreamSpecs) { if (rawSpec is FilterStreamSpecRaw) { FilterStreamSpecRaw raw = (FilterStreamSpecRaw)rawSpec; foreach (ExprNode filterExpr in raw.RawFilterSpec.FilterExpressions) { filterExpr.Accept(visitor); } } if (rawSpec is PatternStreamSpecRaw) { PatternStreamSpecRaw patternStreamSpecRaw = (PatternStreamSpecRaw)rawSpec; EvalNodeAnalysisResult analysisResult = EvalNodeUtil.RecursiveAnalyzeChildNodes(patternStreamSpecRaw.EvalFactoryNode); foreach (EvalFactoryNode evalNode in analysisResult.ActiveNodes) { if (evalNode is EvalFilterFactoryNode) { EvalFilterFactoryNode filterNode = (EvalFilterFactoryNode)evalNode; foreach (ExprNode filterExpr in filterNode.RawFilterSpec.FilterExpressions) { filterExpr.Accept(visitor); } } } } } }
// Special-case validation: When an on-merge query in the not-matched clause uses a subquery then // that subquery should not reference any of the stream's properties which are not-matched internal static void ValidateSubqueryExcludeOuterStream(ExprNode matchCondition) { var visitorSubselects = new ExprNodeSubselectDeclaredDotVisitor(); matchCondition.Accept(visitorSubselects); if (visitorSubselects.Subselects.IsEmpty()) { return; } var visitorProps = new ExprNodeIdentifierCollectVisitor(); foreach (var node in visitorSubselects.Subselects) { if (node.StatementSpecCompiled.FilterRootNode != null) { node.StatementSpecCompiled.FilterRootNode.Accept(visitorProps); } } foreach (var node in visitorProps.ExprProperties) { if (node.StreamId == 1) { throw new ExprValidationException("On-Merge not-matched filter expression may not use properties that are provided by the named window event"); } } }
public static void WalkSubselectAndDeclaredDotExpr( StatementSpecRaw spec, ExprNodeSubselectDeclaredDotVisitor visitor) { // Look for expressions with sub-selects in select expression list and filter expression // Recursively compile the statement within the statement. WalkSubselectSelectClause(spec.SelectClauseSpec.SelectExprList, visitor); spec.WhereClause?.Accept(visitor); spec.HavingClause?.Accept(visitor); if (spec.UpdateDesc != null) { spec.UpdateDesc.OptionalWhereClause?.Accept(visitor); foreach (OnTriggerSetAssignment assignment in spec.UpdateDesc.Assignments) { assignment.Expression.Accept(visitor); } } if (spec.OnTriggerDesc != null) { VisitSubselectOnTrigger(spec.OnTriggerDesc, visitor); } // walk streams WalkStreamSpecs(spec, visitor); // walk FAF WalkFAFSpec(spec.FireAndForgetSpec, visitor); }
private static ExprNode CopyVisitExpression(ExprNode expression, ExprNodeSubselectDeclaredDotVisitor visitor) { try { var node = (ExprNode)SerializableObjectCopier.Copy(expression); node.Accept(visitor); return(node); } catch (Exception e) { throw new Exception("Internal error providing expression tree: " + e.Message, e); } }
private static void WalkSubselectSelectClause( IList<SelectClauseElementRaw> selectClause, ExprNodeSubselectDeclaredDotVisitor visitor) { foreach (SelectClauseElementRaw element in selectClause) { if (element is SelectClauseExprRawSpec) { SelectClauseExprRawSpec selectExpr = (SelectClauseExprRawSpec) element; selectExpr.SelectExpression.Accept(visitor); } } }
public ExpressionCopier( StatementSpecRaw statementSpecRaw, ContextCompileTimeDescriptor contextCompileTimeDescriptor, StatementCompileTimeServices services, ExprNodeSubselectDeclaredDotVisitor visitor) { this.statementSpecRaw = statementSpecRaw; this.contextCompileTimeDescriptor = contextCompileTimeDescriptor; this.services = services; this.visitor = visitor; }
private static void VisitSubselectOnTrigger( OnTriggerDesc onTriggerDesc, ExprNodeSubselectDeclaredDotVisitor visitor) { if (onTriggerDesc is OnTriggerWindowUpdateDesc) { OnTriggerWindowUpdateDesc updates = (OnTriggerWindowUpdateDesc) onTriggerDesc; foreach (OnTriggerSetAssignment assignment in updates.Assignments) { assignment.Expression.Accept(visitor); } } else if (onTriggerDesc is OnTriggerSetDesc) { OnTriggerSetDesc sets = (OnTriggerSetDesc) onTriggerDesc; foreach (OnTriggerSetAssignment assignment in sets.Assignments) { assignment.Expression.Accept(visitor); } } else if (onTriggerDesc is OnTriggerSplitStreamDesc) { OnTriggerSplitStreamDesc splits = (OnTriggerSplitStreamDesc) onTriggerDesc; foreach (OnTriggerSplitStream split in splits.SplitStreams) { split.WhereClause?.Accept(visitor); if (split.SelectClause.SelectExprList != null) { WalkSubselectSelectClause(split.SelectClause.SelectExprList, visitor); } } } else if (onTriggerDesc is OnTriggerMergeDesc) { OnTriggerMergeDesc merge = (OnTriggerMergeDesc) onTriggerDesc; foreach (OnTriggerMergeMatched matched in merge.Items) { matched.OptionalMatchCond?.Accept(visitor); foreach (OnTriggerMergeAction action in matched.Actions) { action.OptionalWhereClause?.Accept(visitor); if (action is OnTriggerMergeActionUpdate) { OnTriggerMergeActionUpdate update = (OnTriggerMergeActionUpdate) action; foreach (OnTriggerSetAssignment assignment in update.Assignments) { assignment.Expression.Accept(visitor); } } if (action is OnTriggerMergeActionInsert) { OnTriggerMergeActionInsert insert = (OnTriggerMergeActionInsert) action; WalkSubselectSelectClause(insert.SelectClause, visitor); } } } if (merge.OptionalInsertNoMatch != null) { WalkSubselectSelectClause(merge.OptionalInsertNoMatch.SelectClause, visitor); } } }
private static void WalkFAFSpec( FireAndForgetSpec fireAndForgetSpec, ExprNodeSubselectDeclaredDotVisitor visitor) { if (fireAndForgetSpec == null) { return; } if (fireAndForgetSpec is FireAndForgetSpecUpdate update) { foreach (OnTriggerSetAssignment assignment in update.Assignments) { assignment.Expression.Accept(visitor); } } }
private static bool DetermineSubselectFilterStream(ExprNode exprNode) { ExprNodeSubselectDeclaredDotVisitor visitor = new ExprNodeSubselectDeclaredDotVisitor(); exprNode.Accept(visitor); if (visitor.Subselects.IsEmpty()) { return false; } foreach (ExprSubselectNode subselectNode in visitor.Subselects) { if (subselectNode.IsFilterStreamSubselect) { return true; } } return false; }
public static void WalkStreamSpecs( StatementSpecRaw spec, ExprNodeSubselectDeclaredDotVisitor visitor) { // determine pattern-filter subqueries foreach (StreamSpecRaw streamSpecRaw in spec.StreamSpecs) { if (streamSpecRaw is PatternStreamSpecRaw) { PatternStreamSpecRaw patternStreamSpecRaw = (PatternStreamSpecRaw) streamSpecRaw; EvalNodeAnalysisResult analysisResult = EvalNodeUtil.RecursiveAnalyzeChildNodes(patternStreamSpecRaw.EvalForgeNode); foreach (EvalForgeNode evalNode in analysisResult.ActiveNodes) { if (evalNode is EvalFilterForgeNode) { EvalFilterForgeNode filterNode = (EvalFilterForgeNode) evalNode; foreach (ExprNode filterExpr in filterNode.RawFilterSpec.FilterExpressions) { filterExpr.Accept(visitor); } } else if (evalNode is EvalObserverForgeNode) { int beforeCount = visitor.Subselects.Count; EvalObserverForgeNode observerNode = (EvalObserverForgeNode) evalNode; foreach (ExprNode param in observerNode.PatternObserverSpec.ObjectParameters) { param.Accept(visitor); } if (visitor.Subselects.Count != beforeCount) { throw new ExprValidationException( "Subselects are not allowed within pattern observer parameters, please consider using a variable instead"); } } } } } // determine filter streams foreach (StreamSpecRaw rawSpec in spec.StreamSpecs) { if (rawSpec is FilterStreamSpecRaw) { FilterStreamSpecRaw raw = (FilterStreamSpecRaw) rawSpec; foreach (ExprNode filterExpr in raw.RawFilterSpec.FilterExpressions) { filterExpr.Accept(visitor); } } } }
/// <summary> /// Check if the expression is minimal: does not have a subselect, aggregation and does not need view resources /// </summary> /// <param name="expression">to inspect</param> /// <returns>null if minimal, otherwise name of offending sub-expression</returns> public static string IsMinimalExpression(ExprNode expression) { ExprNodeSubselectDeclaredDotVisitor subselectVisitor = new ExprNodeSubselectDeclaredDotVisitor(); expression.Accept(subselectVisitor); if (subselectVisitor.Subselects.Count > 0) { return "a subselect"; } ExprNodeViewResourceVisitor viewResourceVisitor = new ExprNodeViewResourceVisitor(); expression.Accept(viewResourceVisitor); if (viewResourceVisitor.ExprNodes.Count > 0) { return "a function that requires view resources (prior, prev)"; } IList<ExprAggregateNode> aggregateNodes = new List<ExprAggregateNode>(); ExprAggregateNodeUtil.GetAggregatesBottomUp(expression, aggregateNodes); if (!aggregateNodes.IsEmpty()) { return "an aggregation function"; } return null; }
private static IList<ExprTableAccessNode> DetermineTableAccessNodes( ISet<ExprTableAccessNode> statementDirectTableAccess, ExprNodeSubselectDeclaredDotVisitor visitor) { ISet<ExprTableAccessNode> tableAccessNodes = new HashSet<ExprTableAccessNode>(); if (statementDirectTableAccess != null) { tableAccessNodes.AddAll(statementDirectTableAccess); } // include all declared expression usages var tableAccessVisitor = new ExprNodeTableAccessVisitor(tableAccessNodes); foreach (var declared in visitor.DeclaredExpressions) { declared.Body.Accept(tableAccessVisitor); } // include all subqueries (and their declared expressions) // This is nested as declared expressions can have more subqueries, however all subqueries are in this list. foreach (var subselectNode in visitor.Subselects) { tableAccessNodes.AddAll(subselectNode.StatementSpecRaw.TableExpressions); } return new List<ExprTableAccessNode>(tableAccessNodes); }
public DataFlowOpForgeInitializeResult InitializeForge(DataFlowOpForgeInitializeContext context) { if (context.InputPorts.IsEmpty()) { throw new ArgumentException("Select operator requires at least one input stream"); } if (context.OutputPorts.Count != 1) { throw new ArgumentException( "Select operator requires one output stream but produces " + context.OutputPorts.Count + " streams"); } var portZero = context.OutputPorts[0]; if (portZero.OptionalDeclaredType != null && !portZero.OptionalDeclaredType.IsUnderlying) { submitEventBean = true; } // determine adapter factories for each type var numStreams = context.InputPorts.Count; eventTypes = new EventType[numStreams]; for (var i = 0; i < numStreams; i++) { eventTypes[i] = context.InputPorts.Get(i).TypeDesc.EventType; } // validate if (select.InsertIntoDesc != null) { throw new ExprValidationException("Insert-into clause is not supported"); } if (select.SelectStreamSelectorEnum != SelectClauseStreamSelectorEnum.ISTREAM_ONLY) { throw new ExprValidationException("Selecting remove-stream is not supported"); } ExprNodeSubselectDeclaredDotVisitor visitor = StatementSpecRawWalkerSubselectAndDeclaredDot.WalkSubselectAndDeclaredDotExpr(select); GroupByClauseExpressions groupByExpressions = GroupByExpressionHelper.GetGroupByRollupExpressions( select.GroupByExpressions, select.SelectClauseSpec, select.WhereClause, select.OrderByList, null); if (!visitor.Subselects.IsEmpty()) { throw new ExprValidationException("Subselects are not supported"); } IDictionary<int, FilterStreamSpecRaw> streams = new Dictionary<int, FilterStreamSpecRaw>(); for (var streamNum = 0; streamNum < select.StreamSpecs.Count; streamNum++) { StreamSpecRaw rawStreamSpec = select.StreamSpecs[streamNum]; if (!(rawStreamSpec is FilterStreamSpecRaw)) { throw new ExprValidationException( "From-clause must contain only streams and cannot contain patterns or other constructs"); } streams.Put(streamNum, (FilterStreamSpecRaw) rawStreamSpec); } // compile offered streams IList<StreamSpecCompiled> streamSpecCompileds = new List<StreamSpecCompiled>(); originatingStreamToViewableStream = new int[select.StreamSpecs.Count]; for (var streamNum = 0; streamNum < select.StreamSpecs.Count; streamNum++) { var filter = streams.Get(streamNum); var inputPort = FindInputPort(filter.RawFilterSpec.EventTypeName, context.InputPorts); if (inputPort == null) { throw new ExprValidationException( "Failed to find stream '" + filter.RawFilterSpec.EventTypeName + "' among input ports, input ports are " + CompatExtensions.RenderAny(GetInputPortNames(context.InputPorts))); } var inputPortValue = inputPort.Value; var eventType = inputPortValue.Value.TypeDesc.EventType; originatingStreamToViewableStream[inputPortValue.Key] = streamNum; var streamAlias = filter.OptionalStreamName; var filterSpecCompiled = new FilterSpecCompiled( eventType, streamAlias, new IList<FilterSpecParamForge>[] { new EmptyList<FilterSpecParamForge>() }, null); ViewSpec[] viewSpecs = select.StreamSpecs[streamNum].ViewSpecs; var filterStreamSpecCompiled = new FilterStreamSpecCompiled( filterSpecCompiled, viewSpecs, streamAlias, StreamSpecOptions.DEFAULT); streamSpecCompileds.Add(filterStreamSpecCompiled); } // create compiled statement spec var selectClauseCompiled = StatementLifecycleSvcUtil.CompileSelectClause(select.SelectClauseSpec); Attribute[] mergedAnnotations = AnnotationUtil.MergeAnnotations( context.StatementRawInfo.Annotations, context.OperatorAnnotations); mergedAnnotations = AddObjectArrayRepresentation(mergedAnnotations); var streamSpecArray = streamSpecCompileds.ToArray(); // determine if snapshot output is needed var outputLimitSpec = select.OutputLimitSpec; if (iterate) { if (outputLimitSpec != null) { throw new ExprValidationException("Output rate limiting is not supported with 'iterate'"); } outputLimitSpec = new OutputLimitSpec(OutputLimitLimitType.SNAPSHOT, OutputLimitRateType.TERM); select.OutputLimitSpec = outputLimitSpec; } // override the statement spec var compiled = new StatementSpecCompiled( select, streamSpecArray, selectClauseCompiled, mergedAnnotations, groupByExpressions, new EmptyList<ExprSubselectNode>(), new EmptyList<ExprDeclaredNode>(), new EmptyList<ExprTableAccessNode>()); var dataflowClassPostfix = context.CodegenEnv.ClassPostfix + "__dfo" + context.OperatorNumber; var containerStatement = context.Base.StatementSpec; context.Base.StatementSpec = compiled; // make forgable var forablesResult = StmtForgeMethodSelectUtil.Make( context.Container, true, context.CodegenEnv.Namespace, dataflowClassPostfix, context.Base, context.Services); // return the statement spec context.Base.StatementSpec = containerStatement; EventType outputEventType = forablesResult.EventType; var initializeResult = new DataFlowOpForgeInitializeResult(); initializeResult.TypeDescriptors = new[] {new GraphTypeDesc(false, true, outputEventType)}; initializeResult.AdditionalForgables = forablesResult.ForgeResult; foreach (StmtClassForgable forgable in forablesResult.ForgeResult.Forgables) { if (forgable.ForgableType == StmtClassForgableType.AIFACTORYPROVIDER) { classNameAIFactoryProvider = forgable.ClassName; } else if (forgable.ForgableType == StmtClassForgableType.FIELDS) { classNameFieldsFactoryProvider = forgable.ClassName; } } return initializeResult; }
public static GroupByClauseExpressions GetGroupByRollupExpressions( IList <GroupByClauseElement> groupByElements, SelectClauseSpecRaw selectClauseSpec, ExprNode optionalHavingNode, IList <OrderByItem> orderByList, ExprNodeSubselectDeclaredDotVisitor visitor) { if (groupByElements == null || groupByElements.Count == 0) { return(null); } // walk group-by-elements, determine group-by expressions and rollup nodes var groupByExpressionInfo = GroupByToRollupNodes(groupByElements); // obtain expression nodes, collect unique nodes and assign index var distinctGroupByExpressions = new List <ExprNode>(); var expressionToIndex = new Dictionary <ExprNode, int>(); foreach (ExprNode exprNode in groupByExpressionInfo.Expressions) { var found = false; for (var i = 0; i < distinctGroupByExpressions.Count; i++) { ExprNode other = distinctGroupByExpressions[i]; // find same expression if (ExprNodeUtility.DeepEquals(exprNode, other)) { expressionToIndex.Put(exprNode, i); found = true; break; } } // not seen before if (!found) { expressionToIndex.Put(exprNode, distinctGroupByExpressions.Count); distinctGroupByExpressions.Add(exprNode); } } // determine rollup, validate it is either (not both) var hasGroupingSet = false; var hasRollup = false; foreach (var element in groupByElements) { if (element is GroupByClauseElementGroupingSet) { hasGroupingSet = true; } if (element is GroupByClauseElementRollupOrCube) { hasRollup = true; } } // no-rollup or grouping-sets means simply validate var groupByExpressions = distinctGroupByExpressions.ToArray(); if (!hasRollup && !hasGroupingSet) { return(new GroupByClauseExpressions(groupByExpressions)); } // evaluate rollup node roots IList <GroupByRollupNodeBase> nodes = groupByExpressionInfo.Nodes; var perNodeCombinations = new Object[nodes.Count][]; var context = new GroupByRollupEvalContext(expressionToIndex); try { for (var i = 0; i < nodes.Count; i++) { var node = nodes[i]; var combinations = node.Evaluate(context); perNodeCombinations[i] = new Object[combinations.Count]; for (var j = 0; j < combinations.Count; j++) { perNodeCombinations[i][j] = combinations[j]; } } } catch (GroupByRollupDuplicateException ex) { if (ex.Indexes.Length == 0) { throw new ExprValidationException("Failed to validate the group-by clause, found duplicate specification of the overall grouping '()'"); } else { var writer = new StringWriter(); var delimiter = ""; for (var i = 0; i < ex.Indexes.Length; i++) { writer.Write(delimiter); writer.Write(groupByExpressions[ex.Indexes[i]].ToExpressionStringMinPrecedenceSafe()); delimiter = ", "; } throw new ExprValidationException("Failed to validate the group-by clause, found duplicate specification of expressions (" + writer.ToString() + ")"); } } // enumerate combinations building an index list var combinationEnumeration = new CombinationEnumeration(perNodeCombinations); ICollection <int> combination = new SortedSet <int>(); ICollection <MultiKeyInt> indexList = new LinkedHashSet <MultiKeyInt>(); while (combinationEnumeration.MoveNext()) { combination.Clear(); Object[] combinationOA = combinationEnumeration.Current; foreach (var indexes in combinationOA) { var indexarr = (int[])indexes; foreach (var anIndex in indexarr) { combination.Add(anIndex); } } var indexArr = CollectionUtil.IntArray(combination); indexList.Add(new MultiKeyInt(indexArr)); } // obtain rollup levels var rollupLevels = new int[indexList.Count][]; var count = 0; foreach (var mk in indexList) { rollupLevels[count++] = mk.Keys; } var numberOfLevels = rollupLevels.Length; if (numberOfLevels == 1 && rollupLevels[0].Length == 0) { throw new ExprValidationException("Failed to validate the group-by clause, the overall grouping '()' cannot be the only grouping"); } // obtain select-expression copies for rewrite var expressions = selectClauseSpec.SelectExprList; var selects = new ExprNode[numberOfLevels][]; for (var i = 0; i < numberOfLevels; i++) { selects[i] = new ExprNode[expressions.Count]; for (var j = 0; j < expressions.Count; j++) { SelectClauseElementRaw selectRaw = expressions[j]; if (!(selectRaw is SelectClauseExprRawSpec)) { throw new ExprValidationException("Group-by with rollup requires that the select-clause does not use wildcard"); } var compiled = (SelectClauseExprRawSpec)selectRaw; selects[i][j] = CopyVisitExpression(compiled.SelectExpression, visitor); } } // obtain having-expression copies for rewrite ExprNode[] optHavingNodeCopy = null; if (optionalHavingNode != null) { optHavingNodeCopy = new ExprNode[numberOfLevels]; for (var i = 0; i < numberOfLevels; i++) { optHavingNodeCopy[i] = CopyVisitExpression(optionalHavingNode, visitor); } } // obtain orderby-expression copies for rewrite ExprNode[][] optOrderByCopy = null; if (orderByList != null && orderByList.Count > 0) { optOrderByCopy = new ExprNode[numberOfLevels][]; for (var i = 0; i < numberOfLevels; i++) { optOrderByCopy[i] = new ExprNode[orderByList.Count]; for (var j = 0; j < orderByList.Count; j++) { OrderByItem element = orderByList[j]; optOrderByCopy[i][j] = CopyVisitExpression(element.ExprNode, visitor); } } } return(new GroupByClauseExpressions(groupByExpressions, rollupLevels, selects, optHavingNodeCopy, optOrderByCopy)); }
public static StatementSpecCompiled Compile( StatementSpecRaw spec, Compilable compilable, bool isSubquery, bool isOnDemandQuery, Attribute[] annotations, IList<ExprSubselectNode> subselectNodes, IList<ExprTableAccessNode> tableAccessNodes, StatementRawInfo statementRawInfo, StatementCompileTimeServices compileTimeServices) { IList<StreamSpecCompiled> compiledStreams; ISet<string> eventTypeReferences = new HashSet<string>(); if (!isOnDemandQuery && spec.FireAndForgetSpec != null) { throw new StatementSpecCompileException( "Provided EPL expression is an on-demand query expression (not a continuous query)", compilable.ToEPL()); } // If not using a join and not specifying a data window, make the where-clause, if present, the filter of the stream // if selecting using filter spec, and not subquery in where clause if (spec.StreamSpecs.Count == 1 && spec.StreamSpecs[0] is FilterStreamSpecRaw && spec.StreamSpecs[0].ViewSpecs.Length == 0 && spec.WhereClause != null && spec.OnTriggerDesc == null && !isSubquery && !isOnDemandQuery && (tableAccessNodes == null || tableAccessNodes.IsEmpty())) { bool disqualified; ExprNode whereClause = spec.WhereClause; var visitorX = new ExprNodeSubselectDeclaredDotVisitor(); whereClause.Accept(visitorX); disqualified = visitorX.Subselects.Count > 0 || HintEnum.DISABLE_WHEREEXPR_MOVETO_FILTER.GetHint(annotations) != null; if (!disqualified) { var viewResourceVisitor = new ExprNodeViewResourceVisitor(); whereClause.Accept(viewResourceVisitor); disqualified = viewResourceVisitor.ExprNodes.Count > 0; } if (!disqualified) { spec.WhereClause = null; var streamSpec = (FilterStreamSpecRaw) spec.StreamSpecs[0]; streamSpec.RawFilterSpec.FilterExpressions.Add(whereClause); } } // compile select-clause var selectClauseCompiled = CompileSelectClause(spec.SelectClauseSpec); // Determine subselects in filter streams, these may need special handling for locking var visitor = new ExprNodeSubselectDeclaredDotVisitor(); try { StatementSpecRawWalkerSubselectAndDeclaredDot.WalkStreamSpecs(spec, visitor); } catch (ExprValidationException ex) { throw new StatementSpecCompileException(ex.Message, ex, compilable.ToEPL()); } foreach (var subselectNode in visitor.Subselects) { subselectNode.IsFilterStreamSubselect = true; } // Determine subselects for compilation, and lambda-expression shortcut syntax for named windows visitor.Reset(); GroupByClauseExpressions groupByRollupExpressions; try { StatementSpecRawWalkerSubselectAndDeclaredDot.WalkSubselectAndDeclaredDotExpr(spec, visitor); var expressionCopier = new ExpressionCopier( spec, statementRawInfo.OptionalContextDescriptor, compileTimeServices, visitor); groupByRollupExpressions = GroupByExpressionHelper.GetGroupByRollupExpressions( spec.GroupByExpressions, spec.SelectClauseSpec, spec.HavingClause, spec.OrderByList, expressionCopier); } catch (ExprValidationException ex) { throw new StatementSpecCompileException(ex.Message, ex, compilable.ToEPL()); } if (isSubquery && !visitor.Subselects.IsEmpty()) { throw new StatementSpecCompileException( "Invalid nested subquery, subquery-within-subquery is not supported", compilable.ToEPL()); } if (isOnDemandQuery && !visitor.Subselects.IsEmpty()) { throw new StatementSpecCompileException( "Subqueries are not a supported feature of on-demand queries", compilable.ToEPL()); } foreach (var subselectNode in visitor.Subselects) { if (!subselectNodes.Contains(subselectNode)) { subselectNodes.Add(subselectNode); } } // Compile subselects found var subselectNumber = 0; foreach (var subselect in subselectNodes) { StatementSpecRaw raw = subselect.StatementSpecRaw; var compiled = Compile( raw, compilable, true, isOnDemandQuery, annotations, Collections.GetEmptyList<ExprSubselectNode>(), Collections.GetEmptyList<ExprTableAccessNode>(), statementRawInfo, compileTimeServices); subselect.SetStatementSpecCompiled(compiled, subselectNumber); subselectNumber++; } // Set table-access number var tableAccessNumber = 0; foreach (var tableAccess in tableAccessNodes) { tableAccess.TableAccessNumber = tableAccessNumber; tableAccessNumber++; } // compile each stream used try { compiledStreams = new List<StreamSpecCompiled>(spec.StreamSpecs.Count); var streamNum = 0; foreach (var rawSpec in spec.StreamSpecs) { streamNum++; var compiled = StreamSpecCompiler.Compile( rawSpec, eventTypeReferences, spec.InsertIntoDesc != null, spec.StreamSpecs.Count > 1, false, spec.OnTriggerDesc != null, rawSpec.OptionalStreamName, streamNum, statementRawInfo, compileTimeServices); compiledStreams.Add(compiled); } } catch (ExprValidationException ex) { if (ex.Message == null) { throw new StatementSpecCompileException( "Unexpected exception compiling statement, please consult the log file and report the exception", ex, compilable.ToEPL()); } throw new StatementSpecCompileException(ex.Message, ex, compilable.ToEPL()); } catch (EPException ex) { throw new StatementSpecCompileException(ex.Message, ex, compilable.ToEPL()); } catch (Exception ex) { var text = "Unexpected error compiling statement"; Log.Error(text, ex); throw new StatementSpecCompileException( text + ": " + ex.GetType().Name + ":" + ex.Message, ex, compilable.ToEPL()); } return new StatementSpecCompiled( spec, compiledStreams.ToArray(), selectClauseCompiled, annotations, groupByRollupExpressions, subselectNodes, visitor.DeclaredExpressions, tableAccessNodes); }
public DataFlowOpInitializeResult Initialize(DataFlowOpInitializateContext context) { if (context.InputPorts.IsEmpty()) { throw new ArgumentException("Select operator requires at least one input stream"); } if (context.OutputPorts.Count != 1) { throw new ArgumentException("Select operator requires one output stream but produces " + context.OutputPorts.Count + " streams"); } DataFlowOpOutputPort portZero = context.OutputPorts[0]; if (portZero.OptionalDeclaredType != null && !portZero.OptionalDeclaredType.IsUnderlying) { _submitEventBean = true; } // determine adapter factories for each type int numStreams = context.InputPorts.Count; _adapterFactories = new EventBeanAdapterFactory[numStreams]; for (int i = 0; i < numStreams; i++) { EventType eventType = context.InputPorts.Get(i).TypeDesc.EventType; _adapterFactories[i] = context.StatementContext.EventAdapterService.GetAdapterFactoryForType(eventType); } // Compile and prepare execution // StatementContext statementContext = context.StatementContext; EPServicesContext servicesContext = context.ServicesContext; AgentInstanceContext agentInstanceContext = context.AgentInstanceContext; // validate if (select.InsertIntoDesc != null) { throw new ExprValidationException("Insert-into clause is not supported"); } if (select.SelectStreamSelectorEnum != SelectClauseStreamSelectorEnum.ISTREAM_ONLY) { throw new ExprValidationException("Selecting remove-stream is not supported"); } ExprNodeSubselectDeclaredDotVisitor visitor = StatementSpecRawAnalyzer.WalkSubselectAndDeclaredDotExpr(select); GroupByClauseExpressions groupByExpressions = GroupByExpressionHelper.GetGroupByRollupExpressions( servicesContext.Container, select.GroupByExpressions, select.SelectClauseSpec, select.HavingExprRootNode, select.OrderByList, visitor); if (!visitor.Subselects.IsEmpty()) { throw new ExprValidationException("Subselects are not supported"); } IDictionary <int, FilterStreamSpecRaw> streams = new Dictionary <int, FilterStreamSpecRaw>(); for (int streamNum = 0; streamNum < select.StreamSpecs.Count; streamNum++) { var rawStreamSpec = select.StreamSpecs[streamNum]; if (!(rawStreamSpec is FilterStreamSpecRaw)) { throw new ExprValidationException("From-clause must contain only streams and cannot contain patterns or other constructs"); } streams.Put(streamNum, (FilterStreamSpecRaw)rawStreamSpec); } // compile offered streams IList <StreamSpecCompiled> streamSpecCompileds = new List <StreamSpecCompiled>(); for (int streamNum = 0; streamNum < select.StreamSpecs.Count; streamNum++) { var filter = streams.Get(streamNum); var inputPort = FindInputPort(filter.RawFilterSpec.EventTypeName, context.InputPorts); if (inputPort == null) { throw new ExprValidationException( string.Format("Failed to find stream '{0}' among input ports, input ports are {1}", filter.RawFilterSpec.EventTypeName, GetInputPortNames(context.InputPorts).Render(", ", "[]"))); } var eventType = inputPort.Value.Value.TypeDesc.EventType; var streamAlias = filter.OptionalStreamName; var filterSpecCompiled = new FilterSpecCompiled(eventType, streamAlias, new IList <FilterSpecParam>[] { Collections.GetEmptyList <FilterSpecParam>() }, null); var filterStreamSpecCompiled = new FilterStreamSpecCompiled(filterSpecCompiled, select.StreamSpecs[0].ViewSpecs, streamAlias, StreamSpecOptions.DEFAULT); streamSpecCompileds.Add(filterStreamSpecCompiled); } // create compiled statement spec SelectClauseSpecCompiled selectClauseCompiled = StatementLifecycleSvcUtil.CompileSelectClause(select.SelectClauseSpec); // determine if snapshot output is needed OutputLimitSpec outputLimitSpec = select.OutputLimitSpec; _isOutputLimited = outputLimitSpec != null; if (iterate) { if (outputLimitSpec != null) { throw new ExprValidationException("Output rate limiting is not supported with 'iterate'"); } outputLimitSpec = new OutputLimitSpec(OutputLimitLimitType.SNAPSHOT, OutputLimitRateType.TERM); } var mergedAnnotations = AnnotationUtil.MergeAnnotations(statementContext.Annotations, context.OperatorAnnotations); var orderByArray = OrderByItem.ToArray(select.OrderByList); var outerJoinArray = OuterJoinDesc.ToArray(select.OuterJoinDescList); var streamSpecArray = streamSpecCompileds.ToArray(); var compiled = new StatementSpecCompiled(null, null, null, null, null, null, null, SelectClauseStreamSelectorEnum.ISTREAM_ONLY, selectClauseCompiled, streamSpecArray, outerJoinArray, select.FilterExprRootNode, select.HavingExprRootNode, outputLimitSpec, orderByArray, ExprSubselectNode.EMPTY_SUBSELECT_ARRAY, ExprNodeUtility.EMPTY_DECLARED_ARR, ExprNodeUtility.EMPTY_SCRIPTS, select.ReferencedVariables, select.RowLimitSpec, CollectionUtil.EMPTY_STRING_ARRAY, mergedAnnotations, null, null, null, null, null, null, null, null, null, groupByExpressions, null, null); // create viewable per port var viewables = new EPLSelectViewable[context.InputPorts.Count]; _viewablesPerPort = viewables; foreach (var entry in context.InputPorts) { EPLSelectViewable viewable = new EPLSelectViewable(entry.Value.TypeDesc.EventType); viewables[entry.Key] = viewable; } var activatorFactory = new ProxyViewableActivatorFactory { ProcCreateActivatorSimple = filterStreamSpec => { EPLSelectViewable found = null; foreach (EPLSelectViewable sviewable in viewables) { if (sviewable.EventType == filterStreamSpec.FilterSpec.FilterForEventType) { found = sviewable; } } if (found == null) { throw new IllegalStateException("Failed to find viewable for filter"); } EPLSelectViewable viewable = found; return(new ProxyViewableActivator( (agentInstanceContext2, isSubselect, isRecoveringResilient) => new ViewableActivationResult( viewable, new ProxyStopCallback(() => { }), null, null, null, false, false, null))); } }; // for per-row deliver, register select expression result callback OutputProcessViewCallback optionalOutputProcessViewCallback = null; if (!iterate && !_isOutputLimited) { _deliveryCallback = new EPLSelectDeliveryCallback(); optionalOutputProcessViewCallback = this; } // prepare EPStatementStartMethodSelectDesc selectDesc = EPStatementStartMethodSelectUtil.Prepare(compiled, servicesContext, statementContext, false, agentInstanceContext, false, activatorFactory, optionalOutputProcessViewCallback, _deliveryCallback); // start _selectResult = (StatementAgentInstanceFactorySelectResult)selectDesc.StatementAgentInstanceFactorySelect.NewContext(agentInstanceContext, false); // for output-rate-limited, register a dispatch view if (_isOutputLimited) { _selectResult.FinalView.AddView(new EPLSelectUpdateDispatchView(this)); } // assign strategies to expression nodes EPStatementStartMethodHelperAssignExpr.AssignExpressionStrategies( selectDesc, _selectResult.OptionalAggegationService, _selectResult.SubselectStrategies, _selectResult.PriorNodeStrategies, _selectResult.PreviousNodeStrategies, null, null, _selectResult.TableAccessEvalStrategies); EventType outputEventType = selectDesc.ResultSetProcessorPrototypeDesc.ResultSetProcessorFactory.ResultEventType; _agentInstanceContext = agentInstanceContext; return(new DataFlowOpInitializeResult(new GraphTypeDesc[] { new GraphTypeDesc(false, true, outputEventType) })); }
public static ExprNodeSubselectDeclaredDotVisitor WalkSubselectAndDeclaredDotExpr(StatementSpecRaw spec) { // Look for expressions with sub-selects in select expression list and filter expression // Recursively compile the statement within the statement. var visitor = new ExprNodeSubselectDeclaredDotVisitor(); foreach (var raw in spec.SelectClauseSpec.SelectExprList) { if (raw is SelectClauseExprRawSpec) { var rawExpr = (SelectClauseExprRawSpec)raw; rawExpr.SelectExpression.Accept(visitor); } } if (spec.FilterRootNode != null) { spec.FilterRootNode.Accept(visitor); } if (spec.HavingExprRootNode != null) { spec.HavingExprRootNode.Accept(visitor); } if (spec.UpdateDesc != null) { if (spec.UpdateDesc.OptionalWhereClause != null) { spec.UpdateDesc.OptionalWhereClause.Accept(visitor); } foreach (var assignment in spec.UpdateDesc.Assignments) { assignment.Expression.Accept(visitor); } } if (spec.OnTriggerDesc != null) { VisitSubselectOnTrigger(spec.OnTriggerDesc, visitor); } // Determine pattern-filter subqueries foreach (var streamSpecRaw in spec.StreamSpecs) { if (streamSpecRaw is PatternStreamSpecRaw) { var patternStreamSpecRaw = (PatternStreamSpecRaw)streamSpecRaw; var analysisResult = EvalNodeUtil.RecursiveAnalyzeChildNodes(patternStreamSpecRaw.EvalFactoryNode); foreach (var evalNode in analysisResult.ActiveNodes) { if (evalNode is EvalFilterFactoryNode) { var filterNode = (EvalFilterFactoryNode)evalNode; foreach (var filterExpr in filterNode.RawFilterSpec.FilterExpressions) { filterExpr.Accept(visitor); } } else if (evalNode is EvalObserverFactoryNode) { var beforeCount = visitor.Subselects.Count; var observerNode = (EvalObserverFactoryNode)evalNode; foreach (var param in observerNode.PatternObserverSpec.ObjectParameters) { param.Accept(visitor); } if (visitor.Subselects.Count != beforeCount) { throw new ExprValidationException("Subselects are not allowed within pattern observer parameters, please consider using a variable instead"); } } } } } // Determine filter streams foreach (var rawSpec in spec.StreamSpecs) { if (rawSpec is FilterStreamSpecRaw) { var raw = (FilterStreamSpecRaw)rawSpec; foreach (var filterExpr in raw.RawFilterSpec.FilterExpressions) { filterExpr.Accept(visitor); } } } return(visitor); }
private static void VisitSubselectOnTrigger(OnTriggerDesc onTriggerDesc, ExprNodeSubselectDeclaredDotVisitor visitor) { if (onTriggerDesc is OnTriggerWindowUpdateDesc) { var updates = (OnTriggerWindowUpdateDesc)onTriggerDesc; foreach (var assignment in updates.Assignments) { assignment.Expression.Accept(visitor); } } else if (onTriggerDesc is OnTriggerSetDesc) { var sets = (OnTriggerSetDesc)onTriggerDesc; foreach (var assignment in sets.Assignments) { assignment.Expression.Accept(visitor); } } else if (onTriggerDesc is OnTriggerSplitStreamDesc) { var splits = (OnTriggerSplitStreamDesc)onTriggerDesc; foreach (var split in splits.SplitStreams) { if (split.WhereClause != null) { split.WhereClause.Accept(visitor); } if (split.SelectClause.SelectExprList != null) { foreach (var element in split.SelectClause.SelectExprList) { if (element is SelectClauseExprRawSpec) { var selectExpr = (SelectClauseExprRawSpec)element; selectExpr.SelectExpression.Accept(visitor); } } } } } else if (onTriggerDesc is OnTriggerMergeDesc) { var merge = (OnTriggerMergeDesc)onTriggerDesc; foreach (var matched in merge.Items) { if (matched.OptionalMatchCond != null) { matched.OptionalMatchCond.Accept(visitor); } foreach (var action in matched.Actions) { if (action.OptionalWhereClause != null) { action.OptionalWhereClause.Accept(visitor); } if (action is OnTriggerMergeActionUpdate) { var update = (OnTriggerMergeActionUpdate)action; foreach (var assignment in update.Assignments) { assignment.Expression.Accept(visitor); } } if (action is OnTriggerMergeActionInsert) { var insert = (OnTriggerMergeActionInsert)action; foreach (var element in insert.SelectClause) { if (element is SelectClauseExprRawSpec) { var selectExpr = (SelectClauseExprRawSpec)element; selectExpr.SelectExpression.Accept(visitor); } } } } } } }
/// <summary> /// Validates expression nodes and returns a list of validated nodes. /// </summary> /// <param name="exprNodeOrigin">The expr node origin.</param> /// <param name="exprNodes">is the nodes to validate</param> /// <param name="streamTypeService">is provding type information for each stream</param> /// <param name="statementContext">context</param> /// <param name="taggedEventTypes">pattern tagged types</param> /// <param name="arrayEventTypes">@return list of validated expression nodes</param> /// <returns> /// expr nodes /// </returns> /// <exception cref="ExprValidationException"> /// Failed to validate + EPStatementStartMethodHelperSubselect.GetSubqueryInfoText(count, subselect) + : + ex.Message /// or /// Filter expression not returning a boolean value: ' + ExprNodeUtility.ToExpressionStringMinPrecedenceSafe(validated) + ' /// </exception> /// <throws>ExprValidationException for validation errors</throws> public static IList <ExprNode> ValidateAllowSubquery( ExprNodeOrigin exprNodeOrigin, IList <ExprNode> exprNodes, StreamTypeService streamTypeService, StatementContext statementContext, IDictionary <string, Pair <EventType, string> > taggedEventTypes, IDictionary <string, Pair <EventType, string> > arrayEventTypes) { IList <ExprNode> validatedNodes = new List <ExprNode>(); var evaluatorContextStmt = new ExprEvaluatorContextStatement(statementContext, false); var validationContext = new ExprValidationContext( streamTypeService, statementContext.MethodResolutionService, null, statementContext.TimeProvider, statementContext.VariableService, statementContext.TableService, evaluatorContextStmt, statementContext.EventAdapterService, statementContext.StatementName, statementContext.StatementId, statementContext.Annotations, statementContext.ContextDescriptor, statementContext.ScriptingService, false, false, true, false, null, true); foreach (var node in exprNodes) { // Determine subselects var visitor = new ExprNodeSubselectDeclaredDotVisitor(); node.Accept(visitor); // Compile subselects if (!visitor.Subselects.IsEmpty()) { // The outer event type is the filtered-type itself var subselectStreamNumber = 2048; var count = -1; foreach (var subselect in visitor.Subselects) { count++; subselectStreamNumber++; try { HandleSubselectSelectClauses(subselectStreamNumber, statementContext, subselect, streamTypeService.EventTypes[0], streamTypeService.StreamNames[0], streamTypeService.StreamNames[0], taggedEventTypes, arrayEventTypes); } catch (ExprValidationException ex) { throw new ExprValidationException("Failed to validate " + EPStatementStartMethodHelperSubselect.GetSubqueryInfoText(count, subselect) + ": " + ex.Message, ex); } } } var validated = ExprNodeUtility.GetValidatedSubtree(exprNodeOrigin, node, validationContext); validatedNodes.Add(validated); if ((validated.ExprEvaluator.ReturnType != typeof(bool?)) && ((validated.ExprEvaluator.ReturnType != typeof(bool)))) { throw new ExprValidationException("Filter expression not returning a boolean value: '" + ExprNodeUtility.ToExpressionStringMinPrecedenceSafe(validated) + "'"); } } return(validatedNodes); }
public static ExprNodeSubselectDeclaredDotVisitor WalkSubselectAndDeclaredDotExpr(StatementSpecRaw spec) { ExprNodeSubselectDeclaredDotVisitor visitor = new ExprNodeSubselectDeclaredDotVisitor(); WalkSubselectAndDeclaredDotExpr(spec, visitor); return visitor; }