Beispiel #1
0
 private DryadQueryNode PromoteConcat(QueryNodeInfo source,
                                      DryadQueryNode sourceNode,
                                      Func<DryadQueryNode, DryadQueryNode> func)
 {
     DryadQueryNode resNode = sourceNode;
     if ((resNode is DryadConcatNode) && !source.IsForked)
     {
         DryadQueryNode[] children = resNode.Children;
         DryadQueryNode[] newChildren = new DryadQueryNode[children.Length];
         for (int i = 0; i < children.Length; i++)
         {
             children[i].Parents.Remove(resNode);
             newChildren[i] = func(children[i]);
         }
         resNode = new DryadConcatNode(source.queryExpression, newChildren);
     }
     else
     {
         resNode = func(resNode);
     }
     return resNode;
 }
Beispiel #2
0
        private DryadQueryNode VisitConcat(QueryNodeInfo source, MethodCallExpression queryExpr)
        {
            DryadQueryNode[] childs = new DryadQueryNode[source.children.Count];
            for (int i = 0; i < source.children.Count; ++i)
            {
                childs[i] = this.Visit(source.children[i].child);
            }
            DryadQueryNode resNode = new DryadConcatNode(queryExpr, childs);

            int parCount = resNode.OutputPartition.Count;
            if (!resNode.IsDynamic && parCount > StaticConfig.MaxPartitionCount)
            {
                // Too many partitions, need to repartition
                int newParCount = parCount / 2;
                DryadQueryNode countNode = new DryadBasicAggregateNode(null, AggregateOpType.LongCount,
                                                                       true, false, queryExpr, resNode);
                DryadQueryNode mergeCountNode = new DryadMergeNode(true, false, queryExpr, countNode);

                // Apply node for s => IndexedCount(s)
                Type paramType = typeof(IEnumerable<>).MakeGenericType(typeof(long));
                ParameterExpression param = Expression.Parameter(paramType, "s");
                MethodInfo minfo = typeof(HpcLinqHelper).GetMethod("IndexedCount");
                minfo = minfo.MakeGenericMethod(typeof(long));
                Expression body = Expression.Call(minfo, param);
                Type funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
                LambdaExpression indexedCountFunc = Expression.Lambda(funcType, body, param);
                DryadQueryNode indexedCountNode = new DryadApplyNode(indexedCountFunc, queryExpr, mergeCountNode);

                // HashPartition(x => x.index, parCount)
                param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
                Expression keySelectBody = Expression.Property(param, "Index");
                funcType = typeof(Func<,>).MakeGenericType(param.Type, keySelectBody.Type);
                LambdaExpression keySelectExpr = Expression.Lambda(funcType, keySelectBody, param);
                DryadQueryNode distCountNode = new DryadHashPartitionNode(keySelectExpr,
                                                                          null,
                                                                          parCount,
                                                                          queryExpr,
                                                                          indexedCountNode);

                // Apply node for (x, y) => AddPartitionIndex(x, y, newParCount)
                ParameterExpression param1 = Expression.Parameter(body.Type, "x");
                Type paramType2 = typeof(IEnumerable<>).MakeGenericType(resNode.OutputTypes[0]);
                ParameterExpression param2 = Expression.Parameter(paramType2, "y");
                minfo = typeof(HpcLinqHelper).GetMethod("AddPartitionIndex");
                minfo = minfo.MakeGenericMethod(resNode.OutputTypes[0]);
                body = Expression.Call(minfo, param1, param2, Expression.Constant(newParCount));
                funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
                LambdaExpression addIndexFunc = Expression.Lambda(funcType, body, param1, param2);
                DryadQueryNode addIndexNode = new DryadApplyNode(addIndexFunc, queryExpr, distCountNode, resNode);

                // HashPartition(x => x.index, x => x.value, newParCount)
                param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
                body = Expression.Property(param, "Index");
                funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
                keySelectExpr = Expression.Lambda(funcType, body, param);
                body = Expression.Property(param, "Value");
                funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
                LambdaExpression resultSelectExpr = Expression.Lambda(funcType, body, param);
                resNode = new DryadHashPartitionNode(keySelectExpr,
                                                     resultSelectExpr,
                                                     null,
                                                     newParCount,
                                                     false,
                                                     queryExpr,
                                                     addIndexNode);
                resNode = new DryadMergeNode(true, true, queryExpr, resNode);
            }
            return resNode;
        }
Beispiel #3
0
 internal virtual string Visit(DryadConcatNode node,
                               CodeMemberMethod vertexMethod,
                               string[] readerNames,
                               string[] writerNames)
 {
     return node.AddVertexCode(vertexMethod, readerNames, writerNames);
 }