private DLinqQueryNode VisitZip(QueryNodeInfo first,
QueryNodeInfo second,
LambdaExpression resultSelector,
MethodCallExpression queryExpr)
{
DLinqQueryNode child1 = this.Visit(first);
DLinqQueryNode child2 = this.Visit(second);
if (child1.IsDynamic || child2.IsDynamic)
{
// Well, let us for now do it on a single machine
child1 = new DLinqMergeNode(true, queryExpr, child1);
child2 = new DLinqMergeNode(true, queryExpr, child2);
// Apply node for (x, y) => Zip(x, y, resultSelector)
Type paramType1 = typeof(IEnumerable<>).MakeGenericType(child1.OutputTypes[0]);
ParameterExpression param1 = Expression.Parameter(paramType1, "s1");
Type paramType2 = typeof(IEnumerable<>).MakeGenericType(child2.OutputTypes[0]);
ParameterExpression param2 = Expression.Parameter(paramType2, "s2");
MethodInfo minfo = typeof(DryadLinqHelper).GetMethod("Zip");
minfo = minfo.MakeGenericMethod(child1.OutputTypes[0]);
Expression body = Expression.Call(minfo, param1, param2, resultSelector);
Type funcType = typeof(Func<,>).MakeGenericType(param1.Type, param2.Type, body.Type);
LambdaExpression procFunc = Expression.Lambda(funcType, body, param1, param2);
return new DLinqApplyNode(procFunc, queryExpr, child1, child2);
}
else
{
int parCount1 = child1.OutputPartition.Count;
int parCount2 = child2.OutputPartition.Count;
// Count nodes
DLinqQueryNode countNode1 = new DLinqBasicAggregateNode(null, AggregateOpType.LongCount,
true, false, queryExpr, child1);
DLinqQueryNode countNode2 = new DLinqBasicAggregateNode(null, AggregateOpType.LongCount,
true, false, queryExpr, child2);
countNode1 = new DLinqMergeNode(true, queryExpr, countNode1);
countNode2 = new DLinqMergeNode(true, queryExpr, countNode2);
// Apply node for (x, y) => ZipCount(x, y)
Type paramType1 = typeof(IEnumerable<>).MakeGenericType(typeof(long));
ParameterExpression param1 = Expression.Parameter(paramType1, "x");
ParameterExpression param2 = Expression.Parameter(paramType1, "y");
MethodInfo minfo = typeof(DryadLinqHelper).GetMethod("ZipCount");
Expression body = Expression.Call(minfo, param1, param2);
Type funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
LambdaExpression zipCount = Expression.Lambda(funcType, body, param1, param2);
DLinqQueryNode indexedCountNode = new DLinqApplyNode(zipCount, queryExpr, countNode1, countNode2);
// HashPartition(x => x.index, parCount2)
ParameterExpression param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
Expression keySelectBody = Expression.Property(param, "Index");
funcType = typeof(Func<,>).MakeGenericType(param.Type, keySelectBody.Type);
LambdaExpression keySelectExpr = Expression.Lambda(funcType, keySelectBody, param);
DLinqQueryNode distCountNode = new DLinqHashPartitionNode(keySelectExpr,
null,
parCount2,
queryExpr,
indexedCountNode);
// Apply node for (x, y) => AssignPartitionIndex(x, y)
param1 = Expression.Parameter(body.Type, "x");
Type paramType2 = typeof(IEnumerable<>).MakeGenericType(child2.OutputTypes[0]);
param2 = Expression.Parameter(paramType2, "y");
minfo = typeof(DryadLinqHelper).GetMethod("AssignPartitionIndex");
minfo = minfo.MakeGenericMethod(child2.OutputTypes[0]);
body = Expression.Call(minfo, param1, param2);
funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
LambdaExpression assignIndex = Expression.Lambda(funcType, body, param1, param2);
DLinqQueryNode addIndexNode = new DLinqApplyNode(assignIndex, queryExpr, distCountNode, child2);
// HashPartition(x => x.index, x => x.value, parCount1)
param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
body = Expression.Property(param, "Index");
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
keySelectExpr = Expression.Lambda(funcType, body, param);
body = Expression.Property(param, "Value");
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
LambdaExpression resultSelectExpr = Expression.Lambda(funcType, body, param);
DLinqQueryNode newChild2 = new DLinqHashPartitionNode(keySelectExpr,
resultSelectExpr,
null,
parCount1,
false,
queryExpr,
addIndexNode);
newChild2 = new DLinqMergeNode(true, queryExpr, newChild2);
// Finally the zip node
return new DLinqZipNode(resultSelector, queryExpr, child1, newChild2);
}
}
internal virtual string Visit(DLinqBasicAggregateNode node,
CodeMemberMethod vertexMethod,
string[] readerNames,
string[] writerNames)
{
return node.AddVertexCode(vertexMethod, readerNames, writerNames);
}
private DLinqQueryNode VisitBasicAggregate(QueryNodeInfo source,
LambdaExpression lambda,
AggregateOpType aggType,
bool isQuery,
Expression queryExpr)
{
DLinqQueryNode child = this.Visit(source);
if (aggType == AggregateOpType.Min || aggType == AggregateOpType.Max)
{
Type elemType = child.OutputTypes[0];
if (lambda != null)
{
elemType = lambda.Body.Type;
}
if (!TypeSystem.HasDefaultComparer(elemType))
{
throw DryadLinqException.Create(DryadLinqErrorCode.AggregationOperatorRequiresIComparable,
String.Format(SR.AggregationOperatorRequiresIComparable, aggType ),
queryExpr);
}
}
DLinqQueryNode resNode = this.PromoteConcat(
source, child,
x => new DLinqBasicAggregateNode(lambda, aggType, true, isQuery, queryExpr, x));
switch (aggType)
{
case AggregateOpType.Count:
case AggregateOpType.LongCount:
{
resNode = new DLinqBasicAggregateNode(null, AggregateOpType.Sum, false,
isQuery, queryExpr, resNode);
break;
}
case AggregateOpType.Sum:
case AggregateOpType.Min:
case AggregateOpType.Max:
case AggregateOpType.Average:
{
resNode = new DLinqBasicAggregateNode(null, aggType, false,
isQuery, queryExpr, resNode);
break;
}
default:
{
throw DryadLinqException.Create(DryadLinqErrorCode.OperatorNotSupported,
String.Format(SR.OperatorNotSupported, aggType),
queryExpr);
}
}
return resNode;
}
private DLinqQueryNode VisitQuantifier(QueryNodeInfo source,
LambdaExpression lambda,
AggregateOpType aggType,
bool isQuery,
Expression queryExpr)
{
DLinqQueryNode child = this.Visit(source);
DLinqQueryNode resNode = this.PromoteConcat(
source, child,
x => new DLinqBasicAggregateNode(
lambda, aggType, true, isQuery, queryExpr, x));
resNode = new DLinqBasicAggregateNode(null, aggType, false, isQuery, queryExpr, resNode);
return resNode;
}
private DLinqQueryNode VisitContains(QueryNodeInfo source,
Expression valueExpr,
Expression comparerExpr,
bool isQuery,
Expression queryExpr)
{
DLinqQueryNode child = this.Visit(source);
Type keyType = child.OutputTypes[0];
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
{
throw DryadLinqException.Create(DryadLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
queryExpr);
}
DLinqQueryNode resNode = this.PromoteConcat(
source, child,
x => new DLinqContainsNode(valueExpr, comparerExpr, queryExpr, x));
resNode = new DLinqBasicAggregateNode(null, AggregateOpType.Any, false, isQuery, queryExpr, resNode);
return resNode;
}
private DLinqQueryNode VisitConcat(QueryNodeInfo source, MethodCallExpression queryExpr)
{
DLinqQueryNode[] childs = new DLinqQueryNode[source.Children.Count];
for (int i = 0; i < source.Children.Count; ++i)
{
childs[i] = this.Visit(source.Children[i].Child);
}
DLinqQueryNode resNode = new DLinqConcatNode(queryExpr, childs);
int parCount = resNode.OutputPartition.Count;
if (!resNode.IsDynamic && parCount > StaticConfig.MaxPartitionCount)
{
// Too many partitions, need to repartition
int newParCount = parCount / 2;
DLinqQueryNode countNode = new DLinqBasicAggregateNode(null, AggregateOpType.LongCount,
true, false, queryExpr, resNode);
DLinqQueryNode mergeCountNode = new DLinqMergeNode(true, queryExpr, countNode);
// Apply node for s => IndexedCount(s)
Type paramType = typeof(IEnumerable<>).MakeGenericType(typeof(long));
ParameterExpression param = Expression.Parameter(paramType, "s");
MethodInfo minfo = typeof(DryadLinqHelper).GetMethod("IndexedCount");
minfo = minfo.MakeGenericMethod(typeof(long));
Expression body = Expression.Call(minfo, param);
Type funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
LambdaExpression indexedCountFunc = Expression.Lambda(funcType, body, param);
DLinqQueryNode indexedCountNode = new DLinqApplyNode(indexedCountFunc, queryExpr, mergeCountNode);
// HashPartition(x => x.index, parCount)
param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
Expression keySelectBody = Expression.Property(param, "Index");
funcType = typeof(Func<,>).MakeGenericType(param.Type, keySelectBody.Type);
LambdaExpression keySelectExpr = Expression.Lambda(funcType, keySelectBody, param);
DLinqQueryNode distCountNode = new DLinqHashPartitionNode(keySelectExpr,
null,
parCount,
queryExpr,
indexedCountNode);
// Apply node for (x, y) => AddPartitionIndex(x, y, newParCount)
ParameterExpression param1 = Expression.Parameter(body.Type, "x");
Type paramType2 = typeof(IEnumerable<>).MakeGenericType(resNode.OutputTypes[0]);
ParameterExpression param2 = Expression.Parameter(paramType2, "y");
minfo = typeof(DryadLinqHelper).GetMethod("AddPartitionIndex");
minfo = minfo.MakeGenericMethod(resNode.OutputTypes[0]);
body = Expression.Call(minfo, param1, param2, Expression.Constant(newParCount));
funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
LambdaExpression addIndexFunc = Expression.Lambda(funcType, body, param1, param2);
DLinqQueryNode addIndexNode = new DLinqApplyNode(addIndexFunc, queryExpr, distCountNode, resNode);
// HashPartition(x => x.index, x => x.value, newParCount)
param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
body = Expression.Property(param, "Index");
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
keySelectExpr = Expression.Lambda(funcType, body, param);
body = Expression.Property(param, "Value");
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
LambdaExpression resultSelectExpr = Expression.Lambda(funcType, body, param);
resNode = new DLinqHashPartitionNode(keySelectExpr,
resultSelectExpr,
null,
newParCount,
false,
queryExpr,
addIndexNode);
resNode = new DLinqMergeNode(true, queryExpr, resNode);
}
return resNode;
}
private DLinqQueryNode CreateOffset(bool isLong, Expression queryExpr, DLinqQueryNode child)
{
// Count node
DLinqQueryNode countNode = new DLinqBasicAggregateNode(null, AggregateOpType.LongCount,
true, false, queryExpr, child);
// Apply node for x => Offsets(x)
Type paramType = typeof(IEnumerable<>).MakeGenericType(typeof(long));
ParameterExpression param = Expression.Parameter(paramType, "x");
MethodInfo minfo = typeof(DryadLinqEnumerable).GetMethod("Offsets");
Expression body = Expression.Call(minfo, param, Expression.Constant(isLong, typeof(bool)));
Type type = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
LambdaExpression procFunc = Expression.Lambda(type, body, param);
DLinqQueryNode mergeCountNode = new DLinqMergeNode(true, queryExpr, countNode);
DLinqQueryNode offsetsNode = new DLinqApplyNode(procFunc, queryExpr, mergeCountNode);
// HashPartition
LambdaExpression keySelectExpr = IdentityFunction.Instance(typeof(IndexedValue<long>));
int pcount = child.OutputPartition.Count;
DLinqQueryNode hdistNode = new DLinqHashPartitionNode(keySelectExpr, null, null, pcount,
false, queryExpr, offsetsNode);
DLinqQueryNode resNode = new DLinqMergeNode(false, queryExpr, hdistNode);
return resNode;
}