Пример #1
0
        private DryadQueryNode VisitMultiApply(QueryNodeInfo source,
                                               LambdaExpression procLambda,
                                               bool perPartition,
                                               bool isFirstOnly,
                                               MethodCallExpression queryExpr)
        {
            DryadQueryNode[] childs = new DryadQueryNode[source.children.Count];
            for (int i = 0; i < source.children.Count; ++i)
            {
                childs[i] = this.Visit(source.children[i].child);
            }

            bool isDynamic = childs.Any(x => x.IsDynamic);
            if (perPartition && !isDynamic)
            {
                // Homomorphic case.
                if (isFirstOnly)
                {
                    for (int i = 1; i < childs.Length; ++i)
                    {
                        childs[i] = new DryadTeeNode(childs[i].OutputTypes[0], true, queryExpr, childs[i]);
                        childs[i].ConOpType = ConnectionOpType.CrossProduct;
                        childs[i] = new DryadMergeNode(childs[0].OutputPartition.Count, queryExpr, childs[i]);
                    }
                }
                else
                {
                    int count = childs[0].OutputPartition.Count;
                    for (int i = 1; i < childs.Length; ++i)
                    {
                        if (childs[i].OutputPartition.Count != count)
                        {
                            throw DryadLinqException.Create(HpcLinqErrorCode.HomomorphicApplyNeedsSamePartitionCount,
                                                          SR.HomomorphicApplyNeedsSamePartitionCount,
                                                          queryExpr);
                        }
                    }
                }
            }
            else
            {
                // Non-homomorphic case.
                for (int i = 0; i < childs.Length; ++i)
                {
                    if (childs[i].IsDynamic || childs[i].OutputPartition.Count > 1)
                    {
                        childs[i] = new DryadMergeNode(true, false, queryExpr, childs[i]);
                    }
                }
            }
            DryadQueryNode applyNode = new DryadApplyNode(procLambda, true, queryExpr, childs);
            return applyNode;
        }
Пример #2
0
        private DryadQueryNode VisitApply(QueryNodeInfo source1,
                                          QueryNodeInfo source2,
                                          LambdaExpression procLambda,
                                          bool perPartition,
                                          bool isFirstOnly,
                                          Expression queryExpr)
        {
            DryadQueryNode child1 = this.Visit(source1);

            DryadQueryNode applyNode;
            if (source2 == null)
            {
                // Unary-apply case:
                if (perPartition)
                {
                    //homomorphic
                    applyNode = this.PromoteConcat(source1, child1, x => new DryadApplyNode(procLambda, queryExpr, x));
                }
                else
                {
                    //non-homomorphic
                    if (child1.IsDynamic || child1.OutputPartition.Count > 1)
                    {
                        child1 = new DryadMergeNode(true, false, queryExpr, child1);
                    }
                    applyNode = new DryadApplyNode(procLambda, queryExpr, child1);
                }
            }
            else
            {
                // Binary-apply case:
                DryadQueryNode child2 = this.Visit(source2);

                if (perPartition && isFirstOnly)
                {
                    // The function is left homomorphic:
                    if (!child2.IsForked && (child1.IsDynamic || child1.OutputPartition.Count > 1))
                    {
                        // The normal cases..
                        if (IsMergeNodeNeeded(child2))
                        {
                            if (child1.IsDynamic)
                            {
                                child2 = new DryadMergeNode(true, false, queryExpr, child2);
                                child2.IsForked = true;
                            }
                            else
                            {
                                // Rather than do full merge and broadcast, which has lots of data-movement
                                //   1. Tee output2 with output cross-product
                                //   2. Do a merge-stage which will have input1.nPartition nodes each performing a merge.
                                //  This acheives a distribution of the entire input2 to the Apply nodes with least data-movement.
                                child2 = new DryadTeeNode(child2.OutputTypes[0], true, queryExpr, child2);
                                child2.ConOpType = ConnectionOpType.CrossProduct;
                                child2 = new DryadMergeNode(child1.OutputPartition.Count, queryExpr, child2);
                            }
                        }
                        else
                        {
                            // the right-data is alread a single partition, so just tee it.
                            // this will provide a copy to each of the apply nodes.
                            child2 = new DryadTeeNode(child2.OutputTypes[0], true, queryExpr, child2);
                        }
                    }
                    else
                    {
                        // Less common cases..
                        // a full merge of the right-data may be necessary.
                        if (child2.IsDynamic || child2.OutputPartition.Count > 1)
                        {
                            child2 = new DryadMergeNode(true, false, queryExpr, child2);
                            if (child1.IsDynamic || child1.OutputPartition.Count > 1)
                            {
                                child2.IsForked = true;
                            }
                        }
                    }
                    applyNode = new DryadApplyNode(procLambda, queryExpr, child1, child2);
                }
                else if (perPartition && !isFirstOnly && !child1.IsDynamic && !child2.IsDynamic)
                {
                    // Full homomorphic
                    // No merging occurs.
                    // NOTE: We generally expect that both the left and right datasets have matching partitionCount.
                    //       however, we don't test for it yet as users might know what they are doing, and it makes
                    //       LocalDebug inconsistent as LocalDebug doesn't throw in that situation.
                    applyNode = new DryadApplyNode(procLambda, queryExpr, child1, child2);
                }
                else
                {
                    // Non-homomorphic
                    // Full merges of both data sets is necessary.
                    if (child1.IsDynamic || child1.OutputPartition.Count > 1)
                    {
                        child1 = new DryadMergeNode(true, false, queryExpr, child1);
                    }
                    if (child2.IsDynamic || child2.OutputPartition.Count > 1)
                    {
                        child2 = new DryadMergeNode(true, false, queryExpr, child2);
                    }
                    applyNode = new DryadApplyNode(procLambda, queryExpr, child1, child2);
                }
            }
            return applyNode;
        }
Пример #3
0
 internal DryadQueryNode InsertTee(bool isForked)
 {
     if (this.OutputArity != 1)
     {
         //@@TODO: this should not be reachable. could change to Assert/InvalidOpEx
         throw new DryadLinqException(HpcLinqErrorCode.Internal, SR.CannotAddTeeToNode);
     }
     List<DryadQueryNode> pnodes = new List<DryadQueryNode>(this.Parents);
     this.Parents.Clear();
     DryadTeeNode teeNode = new DryadTeeNode(this.OutputTypes[0], isForked, this.QueryExpression, this);
     teeNode.m_uniqueId = this.m_uniqueId;
     teeNode.Parents.AddRange(pnodes);
     DryadQueryNode oldNode = this.OutputNode;
     foreach (DryadQueryNode pn in pnodes)
     {
         pn.UpdateChildren(oldNode, teeNode);
     }
     return teeNode;
 }
Пример #4
0
 internal virtual string Visit(DryadTeeNode node,
                               CodeMemberMethod vertexMethod,
                               string[] readerNames,
                               string[] writerNames)
 {
     return node.AddVertexCode(vertexMethod, readerNames, writerNames);
 }
Пример #5
0
        internal DryadForkNode(LambdaExpression fork,
                               Expression keysExpr,
                               Expression queryExpr,
                               DryadQueryNode child)
            : base(QueryNodeType.Fork, child.QueryGen, queryExpr, child)
        {
            this.m_forkLambda = fork;
            this.m_keysExpression = keysExpr;
            this.m_opName = "Fork";

            ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
            this.m_keys = null;
            this.m_keysIdx = -1;
            if (keysExpr != null)
            {
                this.m_keys = evaluator.Eval(keysExpr);
                this.m_keysIdx = HpcLinqObjectStore.Put(m_keys);
            }

            this.m_partitionCount = child.OutputPartition.Count;
            PartitionInfo pinfo = new RandomPartition(child.OutputDataSetInfo.partitionInfo.Count);
            this.m_outputDataSetInfo = new DataSetInfo(pinfo, DataSetInfo.NoOrderBy, DataSetInfo.NoDistinct);

            this.m_dynamicManager = this.InferDynamicManager();

            // Finally, create all the children of this:
            if (keysExpr == null)
            {
                Type forkTupleType = fork.Type.GetGenericArguments()[1];
                if (forkTupleType.GetGenericTypeDefinition() == typeof(IEnumerable<>))
                {
                    forkTupleType = forkTupleType.GetGenericArguments()[0];
                }
                Type[] queryTypeArgs = forkTupleType.GetGenericArguments();
                this.m_outputTypes = new Type[queryTypeArgs.Length];
                for (int i = 0; i < queryTypeArgs.Length; i++)
                {
                    this.m_outputTypes[i] = queryTypeArgs[i];
                    DryadQueryNode parentNode = new DryadTeeNode(queryTypeArgs[i], true, queryExpr, this);
                }
            }
            else
            {
                int forkCnt = ((Array)m_keys).Length;
                Type forkType = fork.Type.GetGenericArguments()[0];
                this.m_outputTypes = new Type[forkCnt];
                for (int i = 0; i < forkCnt; i++)
                {
                    this.m_outputTypes[i] = forkType;
                    DryadQueryNode parentNode = new DryadTeeNode(forkType, true, queryExpr, this);
                }
            }
        }