private DLinqQueryNode VisitConcat(QueryNodeInfo source, MethodCallExpression queryExpr) { DLinqQueryNode[] childs = new DLinqQueryNode[source.Children.Count]; for (int i = 0; i < source.Children.Count; ++i) { childs[i] = this.Visit(source.Children[i].Child); } DLinqQueryNode resNode = new DLinqConcatNode(queryExpr, childs); int parCount = resNode.OutputPartition.Count; if (!resNode.IsDynamic && parCount > StaticConfig.MaxPartitionCount) { // Too many partitions, need to repartition int newParCount = parCount / 2; DLinqQueryNode countNode = new DLinqBasicAggregateNode(null, AggregateOpType.LongCount, true, false, queryExpr, resNode); DLinqQueryNode mergeCountNode = new DLinqMergeNode(true, queryExpr, countNode); // Apply node for s => IndexedCount(s) Type paramType = typeof(IEnumerable<>).MakeGenericType(typeof(long)); ParameterExpression param = Expression.Parameter(paramType, "s"); MethodInfo minfo = typeof(DryadLinqHelper).GetMethod("IndexedCount"); minfo = minfo.MakeGenericMethod(typeof(long)); Expression body = Expression.Call(minfo, param); Type funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type); LambdaExpression indexedCountFunc = Expression.Lambda(funcType, body, param); DLinqQueryNode indexedCountNode = new DLinqApplyNode(indexedCountFunc, queryExpr, mergeCountNode); // HashPartition(x => x.index, parCount) param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x"); Expression keySelectBody = Expression.Property(param, "Index"); funcType = typeof(Func<,>).MakeGenericType(param.Type, keySelectBody.Type); LambdaExpression keySelectExpr = Expression.Lambda(funcType, keySelectBody, param); DLinqQueryNode distCountNode = new DLinqHashPartitionNode(keySelectExpr, null, parCount, queryExpr, indexedCountNode); // Apply node for (x, y) => AddPartitionIndex(x, y, newParCount) ParameterExpression param1 = Expression.Parameter(body.Type, "x"); Type paramType2 = typeof(IEnumerable<>).MakeGenericType(resNode.OutputTypes[0]); ParameterExpression param2 = Expression.Parameter(paramType2, "y"); minfo = typeof(DryadLinqHelper).GetMethod("AddPartitionIndex"); minfo = minfo.MakeGenericMethod(resNode.OutputTypes[0]); body = Expression.Call(minfo, param1, param2, Expression.Constant(newParCount)); funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type); LambdaExpression addIndexFunc = Expression.Lambda(funcType, body, param1, param2); DLinqQueryNode addIndexNode = new DLinqApplyNode(addIndexFunc, queryExpr, distCountNode, resNode); // HashPartition(x => x.index, x => x.value, newParCount) param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x"); body = Expression.Property(param, "Index"); funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type); keySelectExpr = Expression.Lambda(funcType, body, param); body = Expression.Property(param, "Value"); funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type); LambdaExpression resultSelectExpr = Expression.Lambda(funcType, body, param); resNode = new DLinqHashPartitionNode(keySelectExpr, resultSelectExpr, null, newParCount, false, queryExpr, addIndexNode); resNode = new DLinqMergeNode(true, queryExpr, resNode); } return resNode; }
internal virtual string Visit(DLinqConcatNode node, CodeMemberMethod vertexMethod, string[] readerNames, string[] writerNames) { return node.AddVertexCode(vertexMethod, readerNames, writerNames); }
private DLinqQueryNode PromoteConcat(QueryNodeInfo source, DLinqQueryNode sourceNode, Func<DLinqQueryNode, DLinqQueryNode> func) { DLinqQueryNode resNode = sourceNode; if ((resNode is DLinqConcatNode) && !source.IsForked) { DLinqQueryNode[] children = resNode.Children; DLinqQueryNode[] newChildren = new DLinqQueryNode[children.Length]; for (int i = 0; i < children.Length; i++) { children[i].Parents.Remove(resNode); newChildren[i] = func(children[i]); } resNode = new DLinqConcatNode(source.QueryExpression, newChildren); } else { resNode = func(resNode); } return resNode; }