private void TryProcessCandidate( KeyValuePair<Node, Node> candidate, GroupAggregateVarInfo groupAggregateVarInfo) { IList<Node> functionAncestors; IList<Node> groupByAncestors; var definingGroupNode = groupAggregateVarInfo.DefiningGroupNode; FindPathsToLeastCommonAncestor(candidate.Key, definingGroupNode, out functionAncestors, out groupByAncestors); //Check whether all ancestors of the GroupByInto node are of type that we support propagating through if (!AreAllNodesSupportedForPropagation(groupByAncestors)) { return; } //Add the function to the group by node var definingGroupOp = (GroupByIntoOp)definingGroupNode.Op; PlanCompiler.Assert(definingGroupOp.Inputs.Count == 1, "There should be one input var to GroupByInto at this stage"); var inputVar = definingGroupOp.Inputs.First; var functionOp = (FunctionOp)candidate.Key.Op; // // Remap the template from referencing the groupAggregate var to reference the input to // the group by into // var argumentNode = OpCopier.Copy(m_command, candidate.Value); var dictionary = new Dictionary<Var, Var>(1); dictionary.Add(groupAggregateVarInfo.GroupAggregateVar, inputVar); var remapper = new VarRemapper(m_command, dictionary); remapper.RemapSubtree(argumentNode); var newFunctionDefiningNode = m_command.CreateNode( m_command.CreateAggregateOp(functionOp.Function, false), argumentNode); Var newFunctionVar; var varDefNode = m_command.CreateVarDefNode(newFunctionDefiningNode, out newFunctionVar); // Add the new aggregate to the list of aggregates definingGroupNode.Child2.Children.Add(varDefNode); var groupByOp = (GroupByIntoOp)definingGroupNode.Op; groupByOp.Outputs.Set(newFunctionVar); //Propagate the new var throught the ancestors of the GroupByInto for (var i = 0; i < groupByAncestors.Count; i++) { var groupByAncestor = groupByAncestors[i]; if (groupByAncestor.Op.OpType == OpType.Project) { var ancestorProjectOp = (ProjectOp)groupByAncestor.Op; ancestorProjectOp.Outputs.Set(newFunctionVar); } } //Update the functionNode candidate.Key.Op = m_command.CreateVarRefOp(newFunctionVar); candidate.Key.Children.Clear(); }
/// <summary> /// (1) If the given var or the given property of the given var are defined over a group aggregate var, /// (2) and if that group aggregate var matches the var represented by represented by _targetGroupAggregateVarInfo /// if any /// /// it returns the corresponding translation over the group aggregate var. Also, if _targetGroupAggregateVarInfo /// is not set, it sets it to the group aggregate var representing the referenced var. /// </summary> /// <param name="var"></param> /// <param name="property"></param> /// <returns></returns> private Node TranslateOverGroupAggregateVar(Var var, EdmMember property) { GroupAggregateVarRefInfo groupAggregateVarRefInfo; EdmMember localProperty; if (_groupAggregateVarInfoManager.TryGetReferencedGroupAggregateVarInfo(var, out groupAggregateVarRefInfo)) { localProperty = property; } else if (_groupAggregateVarInfoManager.TryGetReferencedGroupAggregateVarInfo(var, property, out groupAggregateVarRefInfo)) { localProperty = null; } else { return(null); } if (_targetGroupAggregateVarInfo == null) { _targetGroupAggregateVarInfo = groupAggregateVarRefInfo.GroupAggregateVarInfo; _isUnnested = groupAggregateVarRefInfo.IsUnnested; } else if (_targetGroupAggregateVarInfo != groupAggregateVarRefInfo.GroupAggregateVarInfo || _isUnnested != groupAggregateVarRefInfo.IsUnnested) { return(null); } Node computationTemplate = groupAggregateVarRefInfo.Computation; if (localProperty != null) { computationTemplate = this._command.CreateNode(this._command.CreatePropertyOp(localProperty), computationTemplate); } return(computationTemplate); }
/// <summary> /// Add an entry that the given property of the given var is a computation represented /// by the computationTemplate over the var represented by the given groupAggregateVarInfo /// </summary> /// <param name="var"></param> /// <param name="groupAggregateVarInfo"></param> /// <param name="computationTemplate"></param> /// <param name="isUnnested"></param> /// <param name="property"></param> internal void Add(Var var, GroupAggregateVarInfo groupAggregateVarInfo, Node computationTemplate, bool isUnnested, EdmMember property) { if (property == null) { Add(var, groupAggregateVarInfo, computationTemplate, isUnnested); return; } if (this._groupAggregateVarRelatedVarPropertyToInfo == null) { this._groupAggregateVarRelatedVarPropertyToInfo = new Dictionary <Var, Dictionary <System.Data.Metadata.Edm.EdmMember, GroupAggregateVarRefInfo> >(); } Dictionary <EdmMember, GroupAggregateVarRefInfo> varPropertyDictionary; if (!_groupAggregateVarRelatedVarPropertyToInfo.TryGetValue(var, out varPropertyDictionary)) { varPropertyDictionary = new Dictionary <System.Data.Metadata.Edm.EdmMember, GroupAggregateVarRefInfo>(); _groupAggregateVarRelatedVarPropertyToInfo.Add(var, varPropertyDictionary); } varPropertyDictionary.Add(property, new GroupAggregateVarRefInfo(groupAggregateVarInfo, computationTemplate, isUnnested)); // Note: The following line is not necessary with the current usage pattern, this method is // never called with a new groupAggregateVarInfo thus it is a no-op. _groupAggregateVarInfos.Add(groupAggregateVarInfo); }
/// <summary> /// Add an entry that the given property of the given var is a computation represented /// by the computationTemplate over the var represented by the given groupAggregateVarInfo /// </summary> /// <param name="var"></param> /// <param name="groupAggregateVarInfo"></param> /// <param name="computationTemplate"></param> /// <param name="isUnnested"></param> /// <param name="property"></param> internal void Add( Var var, GroupAggregateVarInfo groupAggregateVarInfo, Node computationTemplate, bool isUnnested, EdmMember property) { if (property == null) { Add(var, groupAggregateVarInfo, computationTemplate, isUnnested); return; } if (_groupAggregateVarRelatedVarPropertyToInfo == null) { _groupAggregateVarRelatedVarPropertyToInfo = new Dictionary<Var, Dictionary<EdmMember, GroupAggregateVarRefInfo>>(); } Dictionary<EdmMember, GroupAggregateVarRefInfo> varPropertyDictionary; if (!_groupAggregateVarRelatedVarPropertyToInfo.TryGetValue(var, out varPropertyDictionary)) { varPropertyDictionary = new Dictionary<EdmMember, GroupAggregateVarRefInfo>(); _groupAggregateVarRelatedVarPropertyToInfo.Add(var, varPropertyDictionary); } varPropertyDictionary.Add(property, new GroupAggregateVarRefInfo(groupAggregateVarInfo, computationTemplate, isUnnested)); // Note: The following line is not necessary with the current usage pattern, this method is // never called with a new groupAggregateVarInfo thus it is a no-op. _groupAggregateVarInfos.Add(groupAggregateVarInfo); }
/// <summary> /// Public constructor /// </summary> /// <param name="groupAggregateVarInfo"></param> /// <param name="computation"></param> internal GroupAggregateVarRefInfo(GroupAggregateVarInfo groupAggregateVarInfo, Node computation, bool isUnnested) { _groupAggregateVarInfo = groupAggregateVarInfo; _computation = computation; _isUnnested = isUnnested; }
/// <summary> /// Try to produce an equivalent tree to the input subtree, over a single group aggregate variable. /// Such translation can only be produced if all external references of the input subtree are to a /// single group aggregate var, or to vars that are can be translated over that single group /// aggregate var /// </summary> /// <param name="subtree">The input subtree</param> /// <param name="isVarDefinition"></param> /// <param name="command"></param> /// <param name="groupAggregateVarInfoManager"></param> /// <param name="groupAggregateVarInfo">The groupAggregateVarInfo over which the input subtree can be translated </param> /// <param name="templateNode">A tree that is equvalent to the input tree, but over the group aggregate variable /// represented by the groupAggregetVarInfo</param> /// <param name="isUnnested"></param> /// <returns>True, if the translation can be done, false otherwise</returns> public static bool TryTranslateOverGroupAggregateVar( Node subtree, bool isVarDefinition, Command command, GroupAggregateVarInfoManager groupAggregateVarInfoManager, out GroupAggregateVarInfo groupAggregateVarInfo, out Node templateNode, out bool isUnnested) { var handler = new GroupAggregateVarComputationTranslator(command, groupAggregateVarInfoManager); var inputNode = subtree; SoftCastOp softCastOp = null; bool isCollect; if (inputNode.Op.OpType == OpType.SoftCast) { softCastOp = (SoftCastOp)inputNode.Op; inputNode = inputNode.Child0; } if (inputNode.Op.OpType == OpType.Collect) { templateNode = handler.VisitCollect(inputNode); isCollect = true; } else { templateNode = handler.VisitNode(inputNode); isCollect = false; } groupAggregateVarInfo = handler._targetGroupAggregateVarInfo; isUnnested = handler._isUnnested; if (handler._targetGroupAggregateVarInfo == null || templateNode == null) { return false; } if (softCastOp != null) { SoftCastOp newSoftCastOp; // // The type needs to be fixed only if the unnesting happened during this translation. // That can be recognized by these two cases: // 1) if the input node was a collect, or // 2) if the input did not represent a var definition, but a function aggregate argument and // the template is VarRef of a group aggregate var. // if (isCollect || !isVarDefinition && AggregatePushdownUtil.IsVarRefOverGivenVar(templateNode, handler._targetGroupAggregateVarInfo.GroupAggregateVar)) { newSoftCastOp = command.CreateSoftCastOp(TypeHelpers.GetEdmType<CollectionType>(softCastOp.Type).TypeUsage); } else { newSoftCastOp = softCastOp; } templateNode = command.CreateNode(newSoftCastOp, templateNode); } return true; }
/// <summary> /// (1) If the given var or the given property of the given var are defined over a group aggregate var, /// (2) and if that group aggregate var matches the var represented by represented by _targetGroupAggregateVarInfo /// if any /// /// it returns the corresponding translation over the group aggregate var. Also, if _targetGroupAggregateVarInfo /// is not set, it sets it to the group aggregate var representing the referenced var. /// </summary> /// <param name="var"></param> /// <param name="property"></param> /// <returns></returns> private Node TranslateOverGroupAggregateVar(Var var, EdmMember property) { GroupAggregateVarRefInfo groupAggregateVarRefInfo; EdmMember localProperty; if (_groupAggregateVarInfoManager.TryGetReferencedGroupAggregateVarInfo(var, out groupAggregateVarRefInfo)) { localProperty = property; } else if (_groupAggregateVarInfoManager.TryGetReferencedGroupAggregateVarInfo(var, property, out groupAggregateVarRefInfo)) { localProperty = null; } else { return null; } if (_targetGroupAggregateVarInfo == null) { _targetGroupAggregateVarInfo = groupAggregateVarRefInfo.GroupAggregateVarInfo; _isUnnested = groupAggregateVarRefInfo.IsUnnested; } else if (_targetGroupAggregateVarInfo != groupAggregateVarRefInfo.GroupAggregateVarInfo || _isUnnested != groupAggregateVarRefInfo.IsUnnested) { return null; } var computationTemplate = groupAggregateVarRefInfo.Computation; if (localProperty != null) { computationTemplate = _command.CreateNode(_command.CreatePropertyOp(localProperty), computationTemplate); } return computationTemplate; }
private Node VisitCollect(Node n) { //Make sure the only children are projects over unnest var currentNode = n.Child0; var constantDefinitions = new Dictionary<Var, Node>(); while (currentNode.Child0.Op.OpType == OpType.Project) { currentNode = currentNode.Child0; //Visit the VarDefListOp child if (VisitDefault(currentNode.Child1) == null) { return null; } foreach (var definitionNode in currentNode.Child1.Children) { if (IsConstant(definitionNode.Child0)) { constantDefinitions.Add(((VarDefOp)definitionNode.Op).Var, definitionNode.Child0); } } } if (currentNode.Child0.Op.OpType != OpType.Unnest) { return null; } // Handle the unnest var unnestOp = (UnnestOp)currentNode.Child0.Op; GroupAggregateVarRefInfo groupAggregateVarRefInfo; if (_groupAggregateVarInfoManager.TryGetReferencedGroupAggregateVarInfo(unnestOp.Var, out groupAggregateVarRefInfo)) { if (_targetGroupAggregateVarInfo == null) { _targetGroupAggregateVarInfo = groupAggregateVarRefInfo.GroupAggregateVarInfo; } else if (_targetGroupAggregateVarInfo != groupAggregateVarRefInfo.GroupAggregateVarInfo) { return null; } if (!_isUnnested) { return null; } } else { return null; } var physicalProjectOp = (PhysicalProjectOp)n.Child0.Op; PlanCompiler.Assert(physicalProjectOp.Outputs.Count == 1, "Physical project should only have one output at this stage"); var outputVar = physicalProjectOp.Outputs[0]; var computationTemplate = TranslateOverGroupAggregateVar(outputVar, null); if (computationTemplate != null) { _isUnnested = true; return computationTemplate; } Node constantDefinitionNode; if (constantDefinitions.TryGetValue(outputVar, out constantDefinitionNode)) { _isUnnested = true; return constantDefinitionNode; } return null; }
/// <summary> /// Try to push the given function aggregate candidate to the corresponding group into node. /// The candidate can be pushed if all ancestors of the group into node up to the least common /// ancestor between the group into node and the function aggregate have one of the following node op types: /// Project /// Filter /// ConstraintSortOp /// </summary> /// <param name="command"></param> /// <param name="candidate"></param> /// <param name="groupAggregateVarInfo"></param> /// <param name="m_childToParent"></param> private void TryProcessCandidate( KeyValuePair <Node, Node> candidate, GroupAggregateVarInfo groupAggregateVarInfo) { IList <Node> functionAncestors; IList <Node> groupByAncestors; Node definingGroupNode = groupAggregateVarInfo.DefiningGroupNode; FindPathsToLeastCommonAncestor(candidate.Key, definingGroupNode, out functionAncestors, out groupByAncestors); //Check whether all ancestors of the GroupByInto node are of type that we support propagating through if (!AreAllNodesSupportedForPropagation(groupByAncestors)) { return; } //Add the function to the group by node GroupByIntoOp definingGroupOp = (GroupByIntoOp)definingGroupNode.Op; PlanCompiler.Assert(definingGroupOp.Inputs.Count == 1, "There should be one input var to GroupByInto at this stage"); Var inputVar = definingGroupOp.Inputs.First; FunctionOp functionOp = (FunctionOp)candidate.Key.Op; // // Remap the template from referencing the groupAggregate var to reference the input to // the group by into // Node argumentNode = OpCopier.Copy(m_command, candidate.Value); Dictionary <Var, Var> dictionary = new Dictionary <Var, Var>(1); dictionary.Add(groupAggregateVarInfo.GroupAggregateVar, inputVar); VarRemapper remapper = new VarRemapper(m_command, dictionary); remapper.RemapSubtree(argumentNode); Node newFunctionDefiningNode = m_command.CreateNode( m_command.CreateAggregateOp(functionOp.Function, false), argumentNode); Var newFunctionVar; Node varDefNode = m_command.CreateVarDefNode(newFunctionDefiningNode, out newFunctionVar); // Add the new aggregate to the list of aggregates definingGroupNode.Child2.Children.Add(varDefNode); GroupByIntoOp groupByOp = (GroupByIntoOp)definingGroupNode.Op; groupByOp.Outputs.Set(newFunctionVar); //Propagate the new var throught the ancestors of the GroupByInto for (int i = 0; i < groupByAncestors.Count; i++) { Node groupByAncestor = groupByAncestors[i]; if (groupByAncestor.Op.OpType == OpType.Project) { ProjectOp ancestorProjectOp = (ProjectOp)groupByAncestor.Op; ancestorProjectOp.Outputs.Set(newFunctionVar); } } //Update the functionNode candidate.Key.Op = m_command.CreateVarRefOp(newFunctionVar); candidate.Key.Children.Clear(); }
/// <summary> /// If the Subtree rooted at the collect is of the following structure: /// /// PhysicalProject(outputVar) /// | /// Project(s) /// | /// Unnest /// /// where the unnest is over the group aggregate var and the output var /// is either a reference to the group aggregate var or to a constant, it returns the /// translation of the ouput var. /// </summary> /// <param name="n"></param> /// <returns></returns> private Node VisitCollect(Node n) { //Make sure the only children are projects over unnest Node currentNode = n.Child0; Dictionary <Var, Node> constantDefinitions = new Dictionary <Var, Node>(); while (currentNode.Child0.Op.OpType == OpType.Project) { currentNode = currentNode.Child0; //Visit the VarDefListOp child if (VisitDefault(currentNode.Child1) == null) { return(null); } foreach (Node definitionNode in currentNode.Child1.Children) { if (IsConstant(definitionNode.Child0)) { constantDefinitions.Add(((VarDefOp)definitionNode.Op).Var, definitionNode.Child0); } } } if (currentNode.Child0.Op.OpType != OpType.Unnest) { return(null); } // Handle the unnest UnnestOp unnestOp = (UnnestOp)currentNode.Child0.Op; GroupAggregateVarRefInfo groupAggregateVarRefInfo; if (_groupAggregateVarInfoManager.TryGetReferencedGroupAggregateVarInfo(unnestOp.Var, out groupAggregateVarRefInfo)) { if (_targetGroupAggregateVarInfo == null) { _targetGroupAggregateVarInfo = groupAggregateVarRefInfo.GroupAggregateVarInfo; } else if (_targetGroupAggregateVarInfo != groupAggregateVarRefInfo.GroupAggregateVarInfo) { return(null); } if (!_isUnnested) { return(null); } } else { return(null); } PhysicalProjectOp physicalProjectOp = (PhysicalProjectOp)n.Child0.Op; PlanCompiler.Assert(physicalProjectOp.Outputs.Count == 1, "PhysicalProject should only have one output at this stage"); Var outputVar = physicalProjectOp.Outputs[0]; Node computationTemplate = TranslateOverGroupAggregateVar(outputVar, null); if (computationTemplate != null) { _isUnnested = true; return(computationTemplate); } Node constantDefinitionNode; if (constantDefinitions.TryGetValue(outputVar, out constantDefinitionNode)) { _isUnnested = true; return(constantDefinitionNode); } return(null); }
/// <summary> /// Try to produce an equivalent tree to the input subtree, over a single group aggregate variable. /// Such translation can only be produced if all external references of the input subtree are to a /// single group aggregate var, or to vars that are can be translated over that single group /// aggregate var /// </summary> /// <param name="subtree">The input subtree</param> /// <param name="isVarDefinition"></param> /// <param name="command"></param> /// <param name="groupAggregateVarInfoManager"></param> /// <param name="groupAggregateVarInfo">The groupAggregateVarInfo over which the input subtree can be translated </param> /// <param name="templateNode">A tree that is equvalent to the input tree, but over the group aggregate variable /// represented by the groupAggregetVarInfo</param> /// <param name="isUnnested"></param> /// <returns>True, if the translation can be done, false otherwise</returns> public static bool TryTranslateOverGroupAggregateVar( Node subtree, bool isVarDefinition, Command command, GroupAggregateVarInfoManager groupAggregateVarInfoManager, out GroupAggregateVarInfo groupAggregateVarInfo, out Node templateNode, out bool isUnnested) { GroupAggregateVarComputationTranslator handler = new GroupAggregateVarComputationTranslator(command, groupAggregateVarInfoManager); Node inputNode = subtree; SoftCastOp softCastOp = null; bool isCollect; if (inputNode.Op.OpType == OpType.SoftCast) { softCastOp = (SoftCastOp)inputNode.Op; inputNode = inputNode.Child0; } if (inputNode.Op.OpType == OpType.Collect) { templateNode = handler.VisitCollect(inputNode); isCollect = true; } else { templateNode = handler.VisitNode(inputNode); isCollect = false; } groupAggregateVarInfo = handler._targetGroupAggregateVarInfo; isUnnested = handler._isUnnested; if (handler._targetGroupAggregateVarInfo == null || templateNode == null) { return(false); } if (softCastOp != null) { SoftCastOp newSoftCastOp; // // The type needs to be fixed only if the unnesting happened during this translation. // That can be recognized by these two cases: // 1) if the input node was a collect, or // 2) if the input did not represent a var definition, but a function aggregate argument and // the template is VarRef of a group aggregate var. // if (isCollect || !isVarDefinition && AggregatePushdownUtil.IsVarRefOverGivenVar(templateNode, handler._targetGroupAggregateVarInfo.GroupAggregateVar)) { newSoftCastOp = command.CreateSoftCastOp(TypeHelpers.GetEdmType <CollectionType>(softCastOp.Type).TypeUsage); } else { newSoftCastOp = softCastOp; } templateNode = command.CreateNode(newSoftCastOp, templateNode); } return(true); }
/// <summary> /// Add an entry that var is a computation represented by the computationTemplate /// over the var represented by the given groupAggregateVarInfo /// </summary> /// <param name="var"></param> /// <param name="groupAggregateVarInfo"></param> /// <param name="computationTemplate"></param> /// <param name="isUnnested"></param> internal void Add(Var var, GroupAggregateVarInfo groupAggregateVarInfo, Node computationTemplate, bool isUnnested) { this._groupAggregateVarRelatedVarToInfo.Add(var, new GroupAggregateVarRefInfo(groupAggregateVarInfo, computationTemplate, isUnnested)); _groupAggregateVarInfos.Add(groupAggregateVarInfo); }
/// <summary> /// Public constructor /// </summary> /// <param name="groupAggregateVarInfo"></param> /// <param name="computation"></param> internal GroupAggregateVarRefInfo(GroupAggregateVarInfo groupAggregateVarInfo, Node computation, bool isUnnested) { this._groupAggregateVarInfo = groupAggregateVarInfo; this._computation = computation; this._isUnnested = isUnnested; }
/// <summary> /// Add an entry that var is a computation represented by the computationTemplate /// over the var represented by the given groupAggregateVarInfo /// </summary> /// <param name="var"></param> /// <param name="groupAggregateVarInfo"></param> /// <param name="computationTemplate"></param> /// <param name="isUnnested"></param> internal void Add(Var var, GroupAggregateVarInfo groupAggregateVarInfo, Node computationTemplate, bool isUnnested) { this._groupAggregateVarRelatedVarToInfo.Add(var, new GroupAggregateVarRefInfo(groupAggregateVarInfo, computationTemplate, isUnnested)); _groupAggregateVarInfos.Add(groupAggregateVarInfo); }