예제 #1
0
 internal virtual string Visit(DryadDynamicNode node,
                               CodeMemberMethod vertexMethod,
                               string[] readerNames,
                               string[] writerNames)
 {
     return node.AddVertexCode(vertexMethod, readerNames, writerNames);
 }
예제 #2
0
 internal void AddAggregateNode(DryadQueryNode node)
 {
     switch (this.m_dynamicManager.ManagerType)
     {
         case DynamicManagerType.None:
         {
             DryadDynamicNode dnode = new DryadDynamicNode(DynamicManagerType.FullAggregator, this);
             this.m_dynamicManager = new DynamicManager(DynamicManagerType.FullAggregator, dnode);
             break;
         }
         case DynamicManagerType.HashDistributor:
         {
             DryadQueryNode firstVertex = this.m_dynamicManager.GetVertexNode(0);
             DryadDynamicNode dnode = firstVertex as DryadDynamicNode;
             if (dnode == null || dnode.DynamicType != DynamicManagerType.FullAggregator)
             {
                 dnode = new DryadDynamicNode(DynamicManagerType.FullAggregator, this);
                 this.m_dynamicManager.InsertVertexNode(0, dnode);
             }
             break;
         }
         case DynamicManagerType.FullAggregator:
         {
             break;
         }
         default:
         {
             //@@TODO: this should not be reachable. could change to Assert/InvalidOpEx
             throw new DryadLinqException(HpcLinqErrorCode.Internal,
                                        String.Format(SR.DynamicManagerType,
                                                      this.m_dynamicManager.ManagerType));
         }
     }
     ((DryadDynamicNode)this.m_dynamicManager.GetVertexNode(0)).AddNode(node);
 }
예제 #3
0
        private void Initialize(LambdaExpression keySelectExpr,
                                Expression comparerExpr,
                                bool isDescending,
                                bool isTemp,
                                DryadQueryNode child,
                                Expression queryExpr)
        {
            this.m_opName = (keySelectExpr == null) ? "Merge" : "MergeSort";
            this.m_keySelectExpression = keySelectExpr;
            this.m_comparerExpression = comparerExpr;
            this.m_isDescending = isDescending;
            this.m_isTemp = isTemp;

            if (keySelectExpr != null)
            {
                Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
                if (comparerExpr == null && !TypeSystem.HasDefaultComparer(keyType))
                {
                    throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable,
                                                  String.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable, keyType),
                                                  queryExpr);
                }
            }

            this.m_comparer = null;
            this.m_comparerIdx = -1;
            if (comparerExpr != null)
            {
                ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
                this.m_comparer = evaluator.Eval(comparerExpr);
                this.m_comparerIdx = HpcLinqObjectStore.Put(this.m_comparer);
            }

            this.m_dynamicManager = DynamicManager.None;
            if (this.OpName == "MergeSort" && StaticConfig.UseSMBAggregation)
            {
                DryadDynamicNode dnode = new DryadDynamicNode(DynamicManagerType.FullAggregator, this);
                this.m_dynamicManager = new DynamicManager(DynamicManagerType.FullAggregator, dnode);
            }

            DryadQueryNode child1 = child;
            if ((child is DryadHashPartitionNode) &&
                ((DryadHashPartitionNode)child).IsDynamicDistributor)
            {
                child1 = child.Children[0];
                this.Children[0] = child1;
                bool found = child1.UpdateParent(child, this);
                this.m_dynamicManager = this.m_dynamicManager.CreateManager(DynamicManagerType.HashDistributor);
                this.m_dynamicManager.InsertVertexNode(-1, child);
            }

            DataSetInfo childInfo = child1.OutputDataSetInfo;
            PartitionInfo pinfo;
            if (child1.ConOpType == ConnectionOpType.CrossProduct)
            {
                this.m_partitionCount = childInfo.partitionInfo.Count;
                pinfo = childInfo.partitionInfo;
            }
            else
            {
                this.m_partitionCount = 1;
                pinfo = DataSetInfo.OnePartition;
            }

            DistinctInfo dinfo = childInfo.distinctInfo;
            OrderByInfo oinfo = DataSetInfo.NoOrderBy;
            if (this.OpName == "MergeSort")
            {
                Type[] typeArgs = this.m_keySelectExpression.Type.GetGenericArguments();
                oinfo = OrderByInfo.Create(this.m_keySelectExpression, this.m_comparer, this.m_isDescending, typeArgs[1]);
            }
            this.m_outputDataSetInfo = new DataSetInfo(pinfo, oinfo, dinfo);

            this.m_isSplitting = (((child is DryadHashPartitionNode) &&
                                   ((DryadHashPartitionNode)child).IsDynamicDistributor) ||
                                  ((child is DryadRangePartitionNode) &&
                                   ((DryadRangePartitionNode)child).IsDynamicDistributor));
        }
예제 #4
0
        internal DryadAggregateNode(string opName,
                                    Expression seedExpr,
                                    LambdaExpression funcLambda,
                                    LambdaExpression combinerLambda,
                                    LambdaExpression resultLambda,
                                    int stage,
                                    bool isQuery,
                                    Expression queryExpr,
                                    DryadQueryNode child,
                                    bool functionIsExpensive)
            : base(QueryNodeType.Aggregate, child.QueryGen, queryExpr, child)
        {
            this.m_seedExpression = seedExpr;
            this.m_funcLambda = funcLambda;
            this.m_combinerLambda = combinerLambda;
            this.m_resultLambda = resultLambda;
            this.m_stage = stage;
            this.m_isQuery = isQuery;
            this.m_opName = opName;

            this.m_seedValue = null;
            this.m_seedIdx = -1;
            if (seedExpr != null)
            {
                this.m_seedValue = ExpressionSimplifier.Evaluate(seedExpr);
                if (!seedExpr.Type.IsPrimitive)
                {
                    this.m_seedIdx = HpcLinqObjectStore.Put(m_seedValue);
                }
            }

            if (stage != 3)
            {
                this.m_partitionCount = child.OutputDataSetInfo.partitionInfo.Count;
                this.m_outputDataSetInfo = new DataSetInfo();
                this.m_outputDataSetInfo.partitionInfo = new RandomPartition(this.m_partitionCount);
                this.m_dynamicManager = this.InferDynamicManager();
            }
            else
            {
                this.m_partitionCount = 1;
                this.m_outputDataSetInfo = new DataSetInfo();
                if (functionIsExpensive)
                {
                    DryadDynamicNode dnode = new DryadDynamicNode(DynamicManagerType.FullAggregator, child);
                    this.m_dynamicManager = new DynamicManager(DynamicManagerType.FullAggregator, dnode);
                    this.m_dynamicManager.AggregationLevels = 2;
                }
                else
                {
                    this.m_dynamicManager = DynamicManager.None;
                }
            }
        }