private DryadQueryNode VisitZip(QueryNodeInfo first,
QueryNodeInfo second,
LambdaExpression resultSelector,
MethodCallExpression queryExpr)
{
DryadQueryNode child1 = this.Visit(first);
DryadQueryNode child2 = this.Visit(second);
if (child1.IsDynamic || child2.IsDynamic)
{
// Well, let us for now do it on a single machine
child1 = new DryadMergeNode(true, false, queryExpr, child1);
child2 = new DryadMergeNode(true, false, queryExpr, child2);
// Apply node for (x, y) => Zip(x, y, resultSelector)
Type paramType1 = typeof(IEnumerable<>).MakeGenericType(child1.OutputTypes[0]);
ParameterExpression param1 = Expression.Parameter(paramType1, "s1");
Type paramType2 = typeof(IEnumerable<>).MakeGenericType(child2.OutputTypes[0]);
ParameterExpression param2 = Expression.Parameter(paramType2, "s2");
MethodInfo minfo = typeof(HpcLinqHelper).GetMethod("Zip");
minfo = minfo.MakeGenericMethod(child1.OutputTypes[0]);
Expression body = Expression.Call(minfo, param1, param2, resultSelector);
Type funcType = typeof(Func<,>).MakeGenericType(param1.Type, param2.Type, body.Type);
LambdaExpression procFunc = Expression.Lambda(funcType, body, param1, param2);
return new DryadApplyNode(procFunc, queryExpr, child1, child2);
}
else
{
int parCount1 = child1.OutputPartition.Count;
int parCount2 = child2.OutputPartition.Count;
// Count nodes
DryadQueryNode countNode1 = new DryadBasicAggregateNode(null, AggregateOpType.LongCount,
true, false, queryExpr, child1);
DryadQueryNode countNode2 = new DryadBasicAggregateNode(null, AggregateOpType.LongCount,
true, false, queryExpr, child2);
countNode1 = new DryadMergeNode(true, false, queryExpr, countNode1);
countNode2 = new DryadMergeNode(true, false, queryExpr, countNode2);
// Apply node for (x, y) => ZipCount(x, y)
Type paramType1 = typeof(IEnumerable<>).MakeGenericType(typeof(long));
ParameterExpression param1 = Expression.Parameter(paramType1, "x");
ParameterExpression param2 = Expression.Parameter(paramType1, "y");
MethodInfo minfo = typeof(HpcLinqHelper).GetMethod("ZipCount");
Expression body = Expression.Call(minfo, param1, param2);
Type funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
LambdaExpression zipCount = Expression.Lambda(funcType, body, param1, param2);
DryadQueryNode indexedCountNode = new DryadApplyNode(zipCount, queryExpr, countNode1, countNode2);
// HashPartition(x => x.index, parCount2)
ParameterExpression param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
Expression keySelectBody = Expression.Property(param, "Index");
funcType = typeof(Func<,>).MakeGenericType(param.Type, keySelectBody.Type);
LambdaExpression keySelectExpr = Expression.Lambda(funcType, keySelectBody, param);
DryadQueryNode distCountNode = new DryadHashPartitionNode(keySelectExpr,
null,
parCount2,
queryExpr,
indexedCountNode);
// Apply node for (x, y) => AssignPartitionIndex(x, y)
param1 = Expression.Parameter(body.Type, "x");
Type paramType2 = typeof(IEnumerable<>).MakeGenericType(child2.OutputTypes[0]);
param2 = Expression.Parameter(paramType2, "y");
minfo = typeof(HpcLinqHelper).GetMethod("AssignPartitionIndex");
minfo = minfo.MakeGenericMethod(child2.OutputTypes[0]);
body = Expression.Call(minfo, param1, param2);
funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
LambdaExpression assignIndex = Expression.Lambda(funcType, body, param1, param2);
DryadQueryNode addIndexNode = new DryadApplyNode(assignIndex, queryExpr, distCountNode, child2);
// HashPartition(x => x.index, x => x.value, parCount1)
param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
body = Expression.Property(param, "Index");
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
keySelectExpr = Expression.Lambda(funcType, body, param);
body = Expression.Property(param, "Value");
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
LambdaExpression resultSelectExpr = Expression.Lambda(funcType, body, param);
DryadQueryNode newChild2 = new DryadHashPartitionNode(keySelectExpr,
resultSelectExpr,
null,
parCount1,
false,
queryExpr,
addIndexNode);
newChild2 = new DryadMergeNode(true, true, queryExpr, newChild2);
// Finally the zip node
return new DryadZipNode(resultSelector, queryExpr, child1, newChild2);
}
}
private DryadQueryNode CreateOffset(bool isLong, Expression queryExpr, DryadQueryNode child)
{
// Count node
DryadQueryNode countNode = new DryadBasicAggregateNode(null, AggregateOpType.LongCount,
true, false, queryExpr, child);
// Apply node for x => Offsets(x)
Type paramType = typeof(IEnumerable<>).MakeGenericType(typeof(long));
ParameterExpression param = Expression.Parameter(paramType, "x");
MethodInfo minfo = typeof(HpcLinqEnumerable).GetMethod("Offsets");
Expression body = Expression.Call(minfo, param, Expression.Constant(isLong, typeof(bool)));
Type type = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
LambdaExpression procFunc = Expression.Lambda(type, body, param);
DryadQueryNode mergeCountNode = new DryadMergeNode(true, true, queryExpr, countNode);
DryadQueryNode offsetsNode = new DryadApplyNode(procFunc, queryExpr, mergeCountNode);
// HashPartition
LambdaExpression keySelectExpr = IdentityFunction.Instance(typeof(IndexedValue<long>));
int pcount = child.OutputPartition.Count;
DryadQueryNode hdistNode = new DryadHashPartitionNode(keySelectExpr, null, null, pcount,
false, queryExpr, offsetsNode);
DryadQueryNode resNode = new DryadMergeNode(false, true, queryExpr, hdistNode);
return resNode;
}
private DryadQueryNode VisitQuantifier(QueryNodeInfo source,
LambdaExpression lambda,
AggregateOpType aggType,
bool isQuery,
Expression queryExpr)
{
DryadQueryNode child = this.Visit(source);
DryadQueryNode resNode = this.PromoteConcat(
source, child,
x => new DryadBasicAggregateNode(
lambda, aggType, true, isQuery, queryExpr, x));
resNode = new DryadBasicAggregateNode(null, aggType, false, isQuery, queryExpr, resNode);
return resNode;
}
private DryadQueryNode VisitContains(QueryNodeInfo source,
Expression valueExpr,
Expression comparerExpr,
bool isQuery,
Expression queryExpr)
{
DryadQueryNode child = this.Visit(source);
Type keyType = child.OutputTypes[0];
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
queryExpr);
}
DryadQueryNode resNode = this.PromoteConcat(
source, child,
x => new DryadContainsNode(valueExpr, comparerExpr, queryExpr, x));
resNode = new DryadBasicAggregateNode(null, AggregateOpType.Any, false, isQuery, queryExpr, resNode);
return resNode;
}
private DryadQueryNode VisitConcat(QueryNodeInfo source, MethodCallExpression queryExpr)
{
DryadQueryNode[] childs = new DryadQueryNode[source.children.Count];
for (int i = 0; i < source.children.Count; ++i)
{
childs[i] = this.Visit(source.children[i].child);
}
DryadQueryNode resNode = new DryadConcatNode(queryExpr, childs);
int parCount = resNode.OutputPartition.Count;
if (!resNode.IsDynamic && parCount > StaticConfig.MaxPartitionCount)
{
// Too many partitions, need to repartition
int newParCount = parCount / 2;
DryadQueryNode countNode = new DryadBasicAggregateNode(null, AggregateOpType.LongCount,
true, false, queryExpr, resNode);
DryadQueryNode mergeCountNode = new DryadMergeNode(true, false, queryExpr, countNode);
// Apply node for s => IndexedCount(s)
Type paramType = typeof(IEnumerable<>).MakeGenericType(typeof(long));
ParameterExpression param = Expression.Parameter(paramType, "s");
MethodInfo minfo = typeof(HpcLinqHelper).GetMethod("IndexedCount");
minfo = minfo.MakeGenericMethod(typeof(long));
Expression body = Expression.Call(minfo, param);
Type funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
LambdaExpression indexedCountFunc = Expression.Lambda(funcType, body, param);
DryadQueryNode indexedCountNode = new DryadApplyNode(indexedCountFunc, queryExpr, mergeCountNode);
// HashPartition(x => x.index, parCount)
param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
Expression keySelectBody = Expression.Property(param, "Index");
funcType = typeof(Func<,>).MakeGenericType(param.Type, keySelectBody.Type);
LambdaExpression keySelectExpr = Expression.Lambda(funcType, keySelectBody, param);
DryadQueryNode distCountNode = new DryadHashPartitionNode(keySelectExpr,
null,
parCount,
queryExpr,
indexedCountNode);
// Apply node for (x, y) => AddPartitionIndex(x, y, newParCount)
ParameterExpression param1 = Expression.Parameter(body.Type, "x");
Type paramType2 = typeof(IEnumerable<>).MakeGenericType(resNode.OutputTypes[0]);
ParameterExpression param2 = Expression.Parameter(paramType2, "y");
minfo = typeof(HpcLinqHelper).GetMethod("AddPartitionIndex");
minfo = minfo.MakeGenericMethod(resNode.OutputTypes[0]);
body = Expression.Call(minfo, param1, param2, Expression.Constant(newParCount));
funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
LambdaExpression addIndexFunc = Expression.Lambda(funcType, body, param1, param2);
DryadQueryNode addIndexNode = new DryadApplyNode(addIndexFunc, queryExpr, distCountNode, resNode);
// HashPartition(x => x.index, x => x.value, newParCount)
param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
body = Expression.Property(param, "Index");
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
keySelectExpr = Expression.Lambda(funcType, body, param);
body = Expression.Property(param, "Value");
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
LambdaExpression resultSelectExpr = Expression.Lambda(funcType, body, param);
resNode = new DryadHashPartitionNode(keySelectExpr,
resultSelectExpr,
null,
newParCount,
false,
queryExpr,
addIndexNode);
resNode = new DryadMergeNode(true, true, queryExpr, resNode);
}
return resNode;
}
private DryadQueryNode VisitBasicAggregate(QueryNodeInfo source,
LambdaExpression lambda,
AggregateOpType aggType,
bool isQuery,
Expression queryExpr)
{
DryadQueryNode child = this.Visit(source);
if (aggType == AggregateOpType.Min || aggType == AggregateOpType.Max)
{
Type elemType = child.OutputTypes[0];
if (lambda != null)
{
elemType = lambda.Body.Type;
}
if (!TypeSystem.HasDefaultComparer(elemType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.AggregationOperatorRequiresIComparable,
String.Format(SR.AggregationOperatorRequiresIComparable, aggType ),
queryExpr);
}
}
DryadQueryNode resNode = this.PromoteConcat(
source, child,
x => new DryadBasicAggregateNode(lambda, aggType, true, isQuery, queryExpr, x));
switch (aggType)
{
case AggregateOpType.Count:
case AggregateOpType.LongCount:
{
resNode = new DryadBasicAggregateNode(null, AggregateOpType.Sum, false,
isQuery, queryExpr, resNode);
break;
}
case AggregateOpType.Sum:
case AggregateOpType.Min:
case AggregateOpType.Max:
case AggregateOpType.Average:
{
resNode = new DryadBasicAggregateNode(null, aggType, false,
isQuery, queryExpr, resNode);
break;
}
default:
{
throw DryadLinqException.Create(HpcLinqErrorCode.OperatorNotSupported,
String.Format(SR.OperatorNotSupported, aggType),
queryExpr);
}
}
return resNode;
}
internal virtual string Visit(DryadBasicAggregateNode node,
CodeMemberMethod vertexMethod,
string[] readerNames,
string[] writerNames)
{
return node.AddVertexCode(vertexMethod, readerNames, writerNames);
}
