コード例 #1
0
 internal virtual string Visit(DLinqTeeNode node,
                               CodeMemberMethod vertexMethod,
                               string[] readerNames,
                               string[] writerNames)
 {
     return node.AddVertexCode(vertexMethod, readerNames, writerNames);
 }
コード例 #2
0
        private DLinqQueryNode VisitMultiApply(QueryNodeInfo source,
                                               LambdaExpression procLambda,
                                               bool perPartition,
                                               bool isFirstOnly,
                                               MethodCallExpression queryExpr)
        {
            DLinqQueryNode[] childs = new DLinqQueryNode[source.Children.Count];
            for (int i = 0; i < source.Children.Count; ++i)
            {
                childs[i] = this.Visit(source.Children[i].Child);
            }

            bool isDynamic = childs.Any(x => x.IsDynamic);
            if (perPartition && !isDynamic)
            {
                // Homomorphic case.
                if (isFirstOnly)
                {
                    for (int i = 1; i < childs.Length; ++i)
                    {
                        childs[i] = new DLinqTeeNode(childs[i].OutputTypes[0], true, queryExpr, childs[i]);
                        childs[i].ConOpType = ConnectionOpType.CrossProduct;
                        childs[i] = new DLinqMergeNode(childs[0].OutputPartition.Count, queryExpr, childs[i]);
                    }
                }
                else
                {
                    int count = childs[0].OutputPartition.Count;
                    for (int i = 1; i < childs.Length; ++i)
                    {
                        if (childs[i].OutputPartition.Count != count)
                        {
                            throw DryadLinqException.Create(DryadLinqErrorCode.HomomorphicApplyNeedsSamePartitionCount,
                                                            SR.HomomorphicApplyNeedsSamePartitionCount,
                                                            queryExpr);
                        }
                    }
                }
            }
            else
            {
                // Non-homomorphic case.
                for (int i = 0; i < childs.Length; ++i)
                {
                    if (childs[i].IsDynamic || childs[i].OutputPartition.Count > 1)
                    {
                        childs[i] = new DLinqMergeNode(true, queryExpr, childs[i]);
                    }
                }
            }
            DLinqQueryNode applyNode = new DLinqApplyNode(procLambda, true, queryExpr, childs);
            return applyNode;
        }
コード例 #3
0
ファイル: DryadLinqQueryNode.cs プロジェクト: pszmyd/Dryad
 internal DLinqQueryNode InsertTee(bool isForked)
 {
     if (this.OutputArity != 1)
     {
         throw new DryadLinqException(DryadLinqErrorCode.Internal, SR.CannotAddTeeToNode);
     }
     List<DLinqQueryNode> pnodes = new List<DLinqQueryNode>(this.Parents);
     this.Parents.Clear();
     DLinqTeeNode teeNode = new DLinqTeeNode(this.OutputTypes[0], isForked, this.QueryExpression, this);
     teeNode.m_uniqueId = this.m_uniqueId;
     teeNode.Parents.AddRange(pnodes);
     DLinqQueryNode oldNode = this.OutputNode;
     foreach (DLinqQueryNode pn in pnodes)
     {
         pn.UpdateChildren(oldNode, teeNode);
     }
     return teeNode;
 }
コード例 #4
0
        private DLinqQueryNode VisitApply(QueryNodeInfo source1,
                                          QueryNodeInfo source2,
                                          LambdaExpression procLambda,
                                          bool perPartition,
                                          bool isFirstOnly,
                                          Expression queryExpr)
        {
            DLinqQueryNode child1 = this.Visit(source1);

            DLinqQueryNode applyNode;
            if (source2 == null)
            {
                // Unary-apply case:
                if (perPartition)
                {
                    // homomorphic
                    applyNode = this.PromoteConcat(source1, child1, x => new DLinqApplyNode(procLambda, queryExpr, x));
                }
                else
                {
                    // non-homomorphic
                    if (child1.IsDynamic || child1.OutputPartition.Count > 1)
                    {
                        child1 = new DLinqMergeNode(true, queryExpr, child1);
                    }
                    applyNode = new DLinqApplyNode(procLambda, queryExpr, child1);
                }
            }
            else
            {
                // Binary-apply case:
                DLinqQueryNode child2 = this.Visit(source2);
                if (perPartition && isFirstOnly)
                {
                    // The function is left homomorphic:
                    if (child1.IsDynamic || child1.OutputPartition.Count > 1)
                    {
                        // The normal cases..
                        if (IsMergeNodeNeeded(child2))
                        {
                            if (child1.IsDynamic)
                            {
                                child2 = new DLinqMergeNode(true, queryExpr, child2);
                                child2.IsForked = true;
                            }
                            else
                            {
                                // Rather than do full merge and broadcast, which has lots of data-movement  
                                //   1. Tee output2 with output cross-product
                                //   2. Do a merge-stage which will have input1.nPartition nodes each performing a merge.
                                // This acheives a distribution of the entire input2 to the Apply nodes with least data-movement.
                                child2 = new DLinqTeeNode(child2.OutputTypes[0], true, queryExpr, child2);
                                child2.ConOpType = ConnectionOpType.CrossProduct;
                                child2 = new DLinqMergeNode(child1.OutputPartition.Count, queryExpr, child2);
                            }
                        }
                        else
                        {
                            // the right-data is alread a single partition, so just tee it.
                            // this will provide a copy to each of the apply nodes.
                            child2 = new DLinqTeeNode(child2.OutputTypes[0], true, queryExpr, child2);
                        }
                    }
                    else
                    {
                        // a full merge of the right-data may be necessary.
                        if (child2.IsDynamic || child2.OutputPartition.Count > 1)
                        {
                            child2 = new DLinqMergeNode(true, queryExpr, child2);
                            if (child1.IsDynamic || child1.OutputPartition.Count > 1)
                            {
                                child2.IsForked = true;
                            }
                        }
                    }
                    applyNode = new DLinqApplyNode(procLambda, queryExpr, child1, child2);
                }
                else if (perPartition && !isFirstOnly && !child1.IsDynamic && !child2.IsDynamic)
                {
                    // Full homomorphic
                    // No merging occurs.
                    // NOTE: We generally expect that both the left and right datasets have matching partitionCount.
                    //       however, we don't test for it yet as users might know what they are doing, and it makes
                    //       LocalDebug inconsistent as LocalDebug doesn't throw in that situation.
                    applyNode = new DLinqApplyNode(procLambda, queryExpr, child1, child2);
                }
                else
                {
                    // Non-homomorphic
                    // Full merges of both data sets is necessary.
                    if (child1.IsDynamic || child1.OutputPartition.Count > 1)
                    {
                        child1 = new DLinqMergeNode(true, queryExpr, child1);
                    }
                    if (child2.IsDynamic || child2.OutputPartition.Count > 1)
                    {
                        child2 = new DLinqMergeNode(true, queryExpr, child2);
                    }
                    applyNode = new DLinqApplyNode(procLambda, queryExpr, child1, child2);
                }
            }
            return applyNode;
        }
コード例 #5
0
ファイル: DryadLinqQueryNode.cs プロジェクト: pszmyd/Dryad
     internal DLinqForkNode(LambdaExpression fork,
                            Expression keysExpr,
                            Expression queryExpr,
                            DLinqQueryNode child)
         : base(QueryNodeType.Fork, child.QueryGen, queryExpr, child)
     {
         this.m_forkLambda = fork;
         this.m_keysExpression = keysExpr;
         this.m_opName = "Fork";
 
         ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
         this.m_keys = null;
         this.m_keysIdx = -1;
         if (keysExpr != null)
         {
             this.m_keys = evaluator.Eval(keysExpr);
             this.m_keysIdx = DryadLinqObjectStore.Put(m_keys);
         }
         
         this.m_partitionCount = child.OutputPartition.Count;
         PartitionInfo pinfo = new RandomPartition(child.OutputDataSetInfo.partitionInfo.Count);
         this.m_outputDataSetInfo = new DataSetInfo(pinfo, DataSetInfo.NoOrderBy, DataSetInfo.NoDistinct);
 
         this.m_dynamicManager = this.InferDynamicManager();
 
         // Finally, create all the children of this:
         if (keysExpr == null)
         {
             Type forkTupleType = fork.Type.GetGenericArguments()[1];
             if (forkTupleType.GetGenericTypeDefinition() == typeof(IEnumerable<>))
             {
                 forkTupleType = forkTupleType.GetGenericArguments()[0];
             }
             Type[] queryTypeArgs = forkTupleType.GetGenericArguments();
             this.m_outputTypes = new Type[queryTypeArgs.Length];
             for (int i = 0; i < queryTypeArgs.Length; i++)
             {
                 this.m_outputTypes[i] = queryTypeArgs[i];
                 DLinqQueryNode parentNode = new DLinqTeeNode(queryTypeArgs[i], true, queryExpr, this);
             }
         }
         else
         {
             int forkCnt = ((Array)m_keys).Length;
             Type forkType = fork.Type.GetGenericArguments()[0];
             this.m_outputTypes = new Type[forkCnt];
             for (int i = 0; i < forkCnt; i++)
             {
                 this.m_outputTypes[i] = forkType;
                 DLinqQueryNode parentNode = new DLinqTeeNode(forkType, true, queryExpr, this);
             }
         }
     }