// Delete this NodeInfo.
// Precondition: this.children.Count < 2
public void Delete()
{
Debug.Assert(this.Children.Count < 2);
if (this.Children.Count == 0)
{
foreach (NodeInfoEdge edge in this.Parents)
{
edge.Parent.Children.Remove(edge);
}
}
else
{
QueryNodeInfo child = this.Children[0].Child;
child.Parents.Remove(this.Children[0]);
foreach (NodeInfoEdge edge in this.Parents)
{
NodeInfoEdge newEdge = new NodeInfoEdge(edge.Parent, child);
NodeInfoEdge.UpdateEdge(edge.Parent.Children, edge, newEdge);
child.Parents.Add(newEdge);
}
}
this.Parents.Clear();
this.Children.Clear();
}
public void Swap(QueryNodeInfo other)
{
Debug.Assert(this.IsQueryOperator && other.IsQueryOperator);
Debug.Assert(this.QueryNode == null && other.QueryNode == null);
Expression queryExpr = this.QueryExpression;
this.QueryExpression = other.QueryExpression;
other.QueryExpression = queryExpr;
}
// Insert a node info on this edge.
public void Insert(QueryNodeInfo nextInfo)
{
Debug.Assert(nextInfo.children.Count == 0 && nextInfo.parents.Count == 0);
NodeInfoEdge edge1 = new NodeInfoEdge(this.parent, nextInfo);
NodeInfoEdge edge2 = new NodeInfoEdge(nextInfo, this.child);
UpdateEdge(this.parent.children, this, edge1);
nextInfo.parents.Add(edge1);
UpdateEdge(this.child.parents, this, edge2);
nextInfo.children.Add(edge2);
}
// Insert a node info on this edge.
public void Insert(QueryNodeInfo nextInfo)
{
Debug.Assert(nextInfo.Children.Count == 0 && nextInfo.Parents.Count == 0);
NodeInfoEdge edge1 = new NodeInfoEdge(this.Parent, nextInfo);
NodeInfoEdge edge2 = new NodeInfoEdge(nextInfo, this.Child);
UpdateEdge(this.Parent.Children, this, edge1);
nextInfo.Parents.Add(edge1);
UpdateEdge(this.Child.Parents, this, edge2);
nextInfo.Children.Add(edge2);
}
public DoWhileQueryNodeInfo(Expression queryExpression,
QueryNodeInfo body,
QueryNodeInfo cond,
QueryNodeInfo bodySource,
QueryNodeInfo condSource1,
QueryNodeInfo condSource2,
params QueryNodeInfo[] children)
: base(queryExpression, false, children)
{
this.Body = body;
this.Cond = cond;
this.BodySource = bodySource;
this.CondSource1 = condSource1;
this.CondSource2 = condSource2;
}
private DryadQueryNode VisitSlidingWindow(QueryNodeInfo source,
LambdaExpression procLambda,
Expression windowSizeExpr,
Expression queryExpr)
{
// var windows = source.Apply(s => HpcLinqHelper.Last(s, windowSize));
// var slided = windows.Apply(s => HpcLinqHelper.Slide(s)).HashPartition(x => x.Index);
// slided.Apply(source, (x, y) => HpcLinqHelper.ProcessWindows(x, y, procFunc, windowSize));
DryadQueryNode child = this.Visit(source);
if (child.IsDynamic)
{
throw new DryadLinqException("SlidingWindow is only supported for static partition count");
}
ExpressionSimplifier<int> evaluator = new ExpressionSimplifier<int>();
Expression windowSize = Expression.Constant(evaluator.Eval(windowSizeExpr), typeof(int));
child.IsForked = true;
// Apply node for s => Last(s, windowSize)
Type paramType = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
ParameterExpression param = Expression.Parameter(paramType, HpcLinqCodeGen.MakeUniqueName("s"));
MethodInfo minfo = typeof(HpcLinqHelper).GetMethod("Last");
minfo = minfo.MakeGenericMethod(child.OutputTypes[0]);
Expression body = Expression.Call(minfo, param, windowSize);
Type funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
LambdaExpression procFunc = Expression.Lambda(funcType, body, param);
DryadQueryNode lastNode = new DryadApplyNode(procFunc, queryExpr, child);
lastNode = new DryadMergeNode(true, true, queryExpr, lastNode);
// Apply node for s => Slide(s)
param = Expression.Parameter(body.Type, HpcLinqCodeGen.MakeUniqueName("s"));
minfo = typeof(HpcLinqHelper).GetMethod("Slide");
minfo = minfo.MakeGenericMethod(child.OutputTypes[0]);
body = Expression.Call(minfo, param);
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
procFunc = Expression.Lambda(funcType, body, param);
DryadQueryNode slideNode = new DryadApplyNode(procFunc, queryExpr, lastNode);
// Hash partition to distribute from partition i to i+1
int pcount = child.OutputPartition.Count;
param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
Expression keySelectBody = Expression.Property(param, "Index");
funcType = typeof(Func<,>).MakeGenericType(param.Type, keySelectBody.Type);
LambdaExpression keySelectExpr = Expression.Lambda(funcType, keySelectBody, param);
DryadQueryNode hdistNode = new DryadHashPartitionNode(keySelectExpr,
null,
pcount,
queryExpr,
slideNode);
// Apply node for (x, y) => ProcessWindows(x, y, proclambda, windowSize)
Type paramType1 = typeof(IEnumerable<>).MakeGenericType(body.Type);
ParameterExpression param1 = Expression.Parameter(paramType1, HpcLinqCodeGen.MakeUniqueName("x"));
Type paramType2 = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
ParameterExpression param2 = Expression.Parameter(paramType2, HpcLinqCodeGen.MakeUniqueName("y"));
minfo = typeof(HpcLinqHelper).GetMethod("ProcessWindows");
minfo = minfo.MakeGenericMethod(child.OutputTypes[0], procLambda.Body.Type);
body = Expression.Call(minfo, param1, param2, procLambda, windowSize);
funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
procFunc = Expression.Lambda(funcType, body, param1, param2);
return new DryadApplyNode(procFunc, queryExpr, hdistNode, child);
}
private DryadQueryNode VisitSequenceEqual(QueryNodeInfo source1,
QueryNodeInfo source2,
Expression comparerExpr,
Expression queryExpr)
{
DryadQueryNode child1 = this.Visit(source1);
DryadQueryNode child2 = this.Visit(source2);
Type elemType = child1.OutputTypes[0];
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(elemType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerExpressionMustBeSpecifiedOrElementTypeMustBeIEquatable,
String.Format(SR.ComparerExpressionMustBeSpecifiedOrElementTypeMustBeIEquatable, elemType),
queryExpr);
}
// Well, let us do it on a single machine for now
child1 = new DryadMergeNode(true, false, queryExpr, child1);
child2 = new DryadMergeNode(true, false, queryExpr, child2);
// Apply node for (x, y) => SequenceEqual(x, y, c)
Type paramType = typeof(IEnumerable<>).MakeGenericType(elemType);
ParameterExpression param1 = Expression.Parameter(paramType, "s1");
ParameterExpression param2 = Expression.Parameter(paramType, "s2");
MethodInfo minfo = typeof(HpcLinqHelper).GetMethod("SequenceEqual");
minfo = minfo.MakeGenericMethod(elemType);
if (comparerExpr == null)
{
comparerExpr = Expression.Constant(null, typeof(IEqualityComparer<>).MakeGenericType(elemType));
}
Expression body = Expression.Call(minfo, param1, param2, comparerExpr);
Type funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
LambdaExpression procFunc = Expression.Lambda(funcType, body, param1, param2);
return new DryadApplyNode(procFunc, queryExpr, child1, child2);
}
// Basic plan: (reverse all partitions) then (reverse data in each partition)
// The main complication is to perform the first step.
// Approach:
// - tee the input.
// - have a dummy apply node that produces the singleton {0} at each partition
// - merge to get a seq {0,0,..} whose length = nPartition.
// - convert that seq to { (0,n), (1,n), ...}
// - hash-partition to send one item to each of the n workers.
// - use binary-apply to attach targetIndex to each source item
// Apply( seq1 = indexCountPair, seq2 = original data) => ({tgt, item0}, {tgt, item1}, .. )
// - hash-partition to move items to target partition.
// - use local LINQ reverse to do the local data reversal.
private DryadQueryNode VisitReverse(QueryNodeInfo source, Expression queryExpr)
{
DryadQueryNode child = this.Visit(source);
if (child.IsDynamic)
{
throw new DryadLinqException("Reverse is only supported for static partition count");
}
child.IsForked = true;
// Apply node for s => ValueZero(s)
Type paramType = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
ParameterExpression param = Expression.Parameter(paramType, "s");
MethodInfo minfo = typeof(HpcLinqHelper).GetMethod("ValueZero");
minfo = minfo.MakeGenericMethod(child.OutputTypes[0]);
Expression body = Expression.Call(minfo, param);
Type funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
LambdaExpression procFunc = Expression.Lambda(funcType, body, param);
DryadQueryNode valueZeroNode = new DryadApplyNode(procFunc, queryExpr, child);
// Apply node for s => ReverseIndex(s)
paramType = typeof(IEnumerable<>).MakeGenericType(typeof(int));
param = Expression.Parameter(paramType, "s");
minfo = typeof(HpcLinqHelper).GetMethod("MakeIndexCountPairs");
body = Expression.Call(minfo, param);
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
procFunc = Expression.Lambda(funcType, body, param);
DryadQueryNode mergeZeroNode = new DryadMergeNode(true, true, queryExpr, valueZeroNode);
DryadQueryNode indexCountNode = new DryadApplyNode(procFunc, queryExpr, mergeZeroNode);
// HashPartition to distribute the indexCounts -- one to each partition.
// each partition will receive (myPartitionID, pcount).
int pcount = child.OutputPartition.Count;
param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
Expression keySelectBody = Expression.Property(param, "Index");
funcType = typeof(Func<,>).MakeGenericType(param.Type, keySelectBody.Type);
LambdaExpression keySelectExpr = Expression.Lambda(funcType, keySelectBody, param);
DryadQueryNode hdistNode = new DryadHashPartitionNode(keySelectExpr,
null,
pcount,
queryExpr,
indexCountNode);
// Apply node for (x, y) => AddIndexForReverse(x, y)
ParameterExpression param1 = Expression.Parameter(body.Type, "x");
Type paramType2 = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
ParameterExpression param2 = Expression.Parameter(paramType2, "y");
minfo = typeof(HpcLinqHelper).GetMethod("AddIndexForReverse");
minfo = minfo.MakeGenericMethod(child.OutputTypes[0]);
body = Expression.Call(minfo, param1, param2);
funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
LambdaExpression addIndexFunc = Expression.Lambda(funcType, body, param1, param2);
DryadQueryNode addIndexNode = new DryadApplyNode(addIndexFunc, queryExpr, hdistNode, child);
// HashPartition(x => x.index, x => x.value, pcount)
// Moves all data to correct target partition. (each worker will direct all its items to one target partition)
param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
body = Expression.Property(param, "Index");
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
keySelectExpr = Expression.Lambda(funcType, body, param);
body = Expression.Property(param, "Value");
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
LambdaExpression resultSelectExpr = Expression.Lambda(funcType, body, param);
DryadQueryNode reversePartitionNode = new DryadHashPartitionNode(
keySelectExpr, resultSelectExpr, null,
pcount, false, queryExpr, addIndexNode);
// Reverse node
paramType = typeof(IEnumerable<>).MakeGenericType(reversePartitionNode.OutputTypes[0]);
param = Expression.Parameter(paramType, "x");
minfo = typeof(HpcLinqVertex).GetMethod("Reverse");
minfo = minfo.MakeGenericMethod(child.OutputTypes[0]);
body = Expression.Call(minfo, param);
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
procFunc = Expression.Lambda(funcType, body, param);
DryadQueryNode resNode = new DryadMergeNode(true, true, queryExpr, reversePartitionNode);
resNode = new DryadApplyNode(procFunc, queryExpr, resNode);
return resNode;
}
private DryadQueryNode VisitQueryOperatorCall(QueryNodeInfo nodeInfo)
{
DryadQueryNode resNode = nodeInfo.queryNode;
if (resNode != null) return resNode;
MethodCallExpression expression = (MethodCallExpression)nodeInfo.queryExpression;
string methodName = expression.Method.Name;
#region LINQMETHODS
switch (methodName)
{
case "Aggregate":
case "AggregateAsQuery":
{
bool isQuery = (methodName == "AggregateAsQuery");
if (expression.Arguments.Count == 2)
{
LambdaExpression funcLambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (funcLambda != null && funcLambda.Parameters.Count == 2)
{
resNode = this.VisitAggregate(nodeInfo.children[0].child,
null,
funcLambda,
null,
isQuery,
expression);
}
}
else if (expression.Arguments.Count == 3)
{
LambdaExpression funcLambda = HpcLinqExpression.GetLambda(expression.Arguments[2]);
if (funcLambda != null && funcLambda.Parameters.Count == 2)
{
resNode = this.VisitAggregate(nodeInfo.children[0].child,
expression.Arguments[1],
funcLambda,
null,
isQuery,
expression);
}
}
else if (expression.Arguments.Count == 4)
{
LambdaExpression funcLambda = HpcLinqExpression.GetLambda(expression.Arguments[2]);
LambdaExpression resultLambda = HpcLinqExpression.GetLambda(expression.Arguments[3]);
if (funcLambda != null && funcLambda.Parameters.Count == 2 &&
resultLambda != null && resultLambda.Parameters.Count == 1)
{
resNode = this.VisitAggregate(nodeInfo.children[0].child,
expression.Arguments[1],
funcLambda,
resultLambda,
isQuery,
expression);
}
}
break;
}
case "Select":
case "LongSelect":
{
if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && (lambda.Parameters.Count == 1 || lambda.Parameters.Count == 2))
{
resNode = this.VisitSelect(nodeInfo.children[0].child,
QueryNodeType.Select,
lambda,
null,
expression);
}
}
break;
}
case "SelectMany":
case "LongSelectMany":
{
if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count <= 2)
{
resNode = this.VisitSelect(nodeInfo.children[0].child,
QueryNodeType.SelectMany,
lambda,
null,
expression);
}
}
else if (expression.Arguments.Count == 3)
{
LambdaExpression lambda1 = HpcLinqExpression.GetLambda(expression.Arguments[1]);
LambdaExpression lambda2 = HpcLinqExpression.GetLambda(expression.Arguments[2]);
if (lambda1 != null && (lambda1.Parameters.Count == 1 || lambda1.Parameters.Count == 2) &&
lambda2 != null && lambda2.Parameters.Count == 2)
{
resNode = this.VisitSelect(nodeInfo.children[0].child,
QueryNodeType.SelectMany,
lambda1,
lambda2,
expression);
}
}
break;
}
case "Join":
case "GroupJoin":
{
QueryNodeType nodeType = (methodName == "Join") ? QueryNodeType.Join : QueryNodeType.GroupJoin;
if (expression.Arguments.Count == 5)
{
LambdaExpression lambda2 = HpcLinqExpression.GetLambda(expression.Arguments[2]);
LambdaExpression lambda3 = HpcLinqExpression.GetLambda(expression.Arguments[3]);
LambdaExpression lambda4 = HpcLinqExpression.GetLambda(expression.Arguments[4]);
if (lambda2 != null && lambda2.Parameters.Count == 1 &&
lambda3 != null && lambda3.Parameters.Count == 1 &&
lambda4 != null && lambda4.Parameters.Count == 2)
{
resNode = this.VisitJoin(nodeInfo.children[0].child,
nodeInfo.children[1].child,
nodeType,
lambda2,
lambda3,
lambda4,
null,
expression);
}
}
else if (expression.Arguments.Count == 6)
{
LambdaExpression lambda2 = HpcLinqExpression.GetLambda(expression.Arguments[2]);
LambdaExpression lambda3 = HpcLinqExpression.GetLambda(expression.Arguments[3]);
LambdaExpression lambda4 = HpcLinqExpression.GetLambda(expression.Arguments[4]);
if (lambda2 != null && lambda2.Parameters.Count == 1 &&
lambda3 != null && lambda3.Parameters.Count == 1 &&
lambda4 != null && lambda4.Parameters.Count == 2)
{
resNode = this.VisitJoin(nodeInfo.children[0].child,
nodeInfo.children[1].child,
nodeType,
lambda2,
lambda3,
lambda4,
expression.Arguments[5],
expression);
}
}
break;
}
case "OfType":
{
if (expression.Arguments.Count == 1)
{
Type ofType = expression.Method.GetGenericArguments()[0];
resNode = this.VisitOfType(nodeInfo.children[0].child, ofType, expression);
}
break;
}
case "Where":
case "LongWhere":
{
if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && (lambda.Parameters.Count == 1 || lambda.Parameters.Count == 2))
{
resNode = this.VisitWhere(nodeInfo.children[0].child,
lambda,
expression);
}
}
break;
}
case "First":
case "FirstOrDefault":
case "FirstAsQuery":
{
AggregateOpType aggType = (methodName == "FirstOrDefault") ? AggregateOpType.FirstOrDefault : AggregateOpType.First;
bool isQuery = (methodName == "FirstAsQuery");
if (expression.Arguments.Count == 1)
{
resNode = this.VisitFirst(nodeInfo.children[0].child, null, aggType, isQuery, expression);
}
else if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitFirst(nodeInfo.children[0].child,
lambda,
aggType,
isQuery,
expression);
}
}
break;
}
case "Single":
case "SingleOrDefault":
case "SingleAsQuery":
{
AggregateOpType aggType = (methodName == "SingleOrDefault") ? AggregateOpType.SingleOrDefault : AggregateOpType.Single;
bool isQuery = (methodName == "SingleAsQuery");
if (expression.Arguments.Count == 1)
{
resNode = this.VisitFirst(nodeInfo.children[0].child, null, aggType, isQuery, expression);
}
else if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitFirst(nodeInfo.children[0].child,
lambda,
aggType,
isQuery,
expression);
}
}
break;
}
case "Last":
case "LastOrDefault":
case "LastAsQuery":
{
AggregateOpType aggType = (methodName == "LastOrDefault") ? AggregateOpType.LastOrDefault : AggregateOpType.Last;
bool isQuery = (methodName == "LastAsQuery");
if (expression.Arguments.Count == 1)
{
resNode = this.VisitFirst(nodeInfo.children[0].child, null, aggType, isQuery, expression);
}
else if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitFirst(nodeInfo.children[0].child,
lambda,
aggType,
isQuery,
expression);
}
}
break;
}
case "Distinct":
{
if (expression.Arguments.Count == 1)
{
resNode = this.VisitDistinct(nodeInfo.children[0].child, null, expression);
}
else if (expression.Arguments.Count == 2)
{
resNode = this.VisitDistinct(nodeInfo.children[0].child,
expression.Arguments[1],
expression);
}
break;
}
case "DefaultIfEmpty":
{
if (expression.Arguments.Count == 1)
{
resNode = this.VisitDefaultIfEmpty(nodeInfo.children[0].child,
null,
expression);
}
else if (expression.Arguments.Count == 2)
{
resNode = this.VisitDefaultIfEmpty(nodeInfo.children[0].child,
expression.Arguments[1],
expression);
}
break;
}
case "Concat":
{
if (expression.Arguments.Count == 2)
{
resNode = this.VisitConcat(nodeInfo, expression);
}
break;
}
case "Union":
{
if (expression.Arguments.Count == 2)
{
resNode = this.VisitSetOperation(nodeInfo.children[0].child,
nodeInfo.children[1].child,
QueryNodeType.Union,
null,
expression);
}
else if (expression.Arguments.Count == 3)
{
resNode = this.VisitSetOperation(nodeInfo.children[0].child,
nodeInfo.children[1].child,
QueryNodeType.Union,
expression.Arguments[2],
expression);
}
break;
}
case "Intersect":
{
if (expression.Arguments.Count == 2)
{
resNode = this.VisitSetOperation(nodeInfo.children[0].child,
nodeInfo.children[1].child,
QueryNodeType.Intersect,
null,
expression);
}
else if (expression.Arguments.Count == 3)
{
resNode = this.VisitSetOperation(nodeInfo.children[0].child,
nodeInfo.children[1].child,
QueryNodeType.Intersect,
expression.Arguments[2],
expression);
}
break;
}
case "Except":
{
if (expression.Arguments.Count == 2)
{
resNode = this.VisitSetOperation(nodeInfo.children[0].child,
nodeInfo.children[1].child,
QueryNodeType.Except,
null,
expression);
}
else if (expression.Arguments.Count == 3)
{
resNode = this.VisitSetOperation(nodeInfo.children[0].child,
nodeInfo.children[1].child,
QueryNodeType.Except,
expression.Arguments[2],
expression);
}
break;
}
case "Any":
case "AnyAsQuery":
{
bool isQuery = (methodName == "AnyAsQuery");
if (expression.Arguments.Count == 1)
{
Type type = expression.Method.GetGenericArguments()[0];
ParameterExpression param = Expression.Parameter(type, "x");
Type delegateType = typeof(Func<,>).MakeGenericType(type, typeof(bool));
Expression body = Expression.Constant(true);
LambdaExpression lambda = Expression.Lambda(delegateType, body, param);
resNode = this.VisitQuantifier(nodeInfo.children[0].child,
lambda,
AggregateOpType.Any,
isQuery,
expression);
}
else if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitQuantifier(nodeInfo.children[0].child,
lambda,
AggregateOpType.Any,
isQuery,
expression);
}
}
break;
}
case "All":
case "AllAsQuery":
{
bool isQuery = (methodName == "AllAsQuery");
if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitQuantifier(nodeInfo.children[0].child,
lambda,
AggregateOpType.All,
isQuery,
expression);
}
}
break;
}
case "Count":
case "CountAsQuery":
{
bool isQuery = (methodName == "CountAsQuery");
if (expression.Arguments.Count == 1)
{
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
null,
AggregateOpType.Count,
isQuery,
expression);
}
else if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
lambda,
AggregateOpType.Count,
isQuery,
expression);
}
}
break;
}
case "LongCount":
case "LongCountAsQuery":
{
bool isQuery = (methodName == "LongCountAsQuery");
if (expression.Arguments.Count == 1)
{
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
null,
AggregateOpType.LongCount,
isQuery,
expression);
}
else if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
lambda,
AggregateOpType.LongCount,
isQuery,
expression);
}
}
break;
}
case "Sum":
case "SumAsQuery":
{
bool isQuery = (methodName == "SumAsQuery");
if (expression.Arguments.Count == 1)
{
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
null,
AggregateOpType.Sum,
isQuery,
expression);
}
else if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
lambda,
AggregateOpType.Sum,
isQuery,
expression);
}
}
break;
}
case "Min":
case "MinAsQuery":
{
bool isQuery = (methodName == "MinAsQuery");
if (expression.Arguments.Count == 1)
{
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
null,
AggregateOpType.Min,
isQuery,
expression);
}
else if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
lambda,
AggregateOpType.Min,
isQuery,
expression);
}
}
break;
}
case "Max":
case "MaxAsQuery":
{
bool isQuery = (methodName == "MaxAsQuery");
if (expression.Arguments.Count == 1)
{
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
null,
AggregateOpType.Max,
isQuery,
expression);
}
else if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
lambda,
AggregateOpType.Max,
isQuery,
expression);
}
}
break;
}
case "Average":
case "AverageAsQuery":
{
bool isQuery = (methodName == "AverageAsQuery");
if (expression.Arguments.Count == 1)
{
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
null,
AggregateOpType.Average,
isQuery,
expression);
}
else if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
lambda,
AggregateOpType.Average,
isQuery,
expression);
}
}
break;
}
case "GroupBy":
{
// groupby can take 2, 3, 4, or 5 arguments.
if (expression.Arguments.Count == 2)
{
//Supplied arguments are as follows:(source, key selector)
LambdaExpression keySelExpr = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (keySelExpr != null && keySelExpr.Parameters.Count == 1)
{
resNode = this.VisitGroupBy(nodeInfo.children[0].child,
keySelExpr,
null,
null,
null,
expression);
}
}
else if (expression.Arguments.Count == 3)
{
//Supplied arguments are as follows:(source, key selector, element selector/result selector/comparer)
LambdaExpression keySelExpr = HpcLinqExpression.GetLambda(expression.Arguments[1]);
LambdaExpression lambda2 = HpcLinqExpression.GetLambda(expression.Arguments[2]);
if (keySelExpr != null && lambda2 == null)
{
resNode = this.VisitGroupBy(nodeInfo.children[0].child,
keySelExpr,
null,
null,
expression.Arguments[2],
expression);
}
else if (keySelExpr != null && keySelExpr.Parameters.Count == 1 && lambda2 != null)
{
LambdaExpression elemSelExpr = null;
LambdaExpression resSelExpr = null;
if (lambda2.Parameters.Count == 1)
{
elemSelExpr = lambda2;
}
else if (lambda2.Parameters.Count == 2)
{
resSelExpr = lambda2;
}
resNode = this.VisitGroupBy(nodeInfo.children[0].child,
keySelExpr,
elemSelExpr,
resSelExpr,
null,
expression);
}
}
else if (expression.Arguments.Count == 4)
{
//Argument-0 is source and Argument-1 is key selector expression
LambdaExpression keySelExpr = HpcLinqExpression.GetLambda(expression.Arguments[1]);
LambdaExpression lambda2 = HpcLinqExpression.GetLambda(expression.Arguments[2]);
LambdaExpression lambda3 = HpcLinqExpression.GetLambda(expression.Arguments[3]);
if (keySelExpr != null && keySelExpr.Parameters.Count == 1 && lambda3 == null)
{
//Argument-2 can be either result selector, element selector and argument-3 is comparer
LambdaExpression elemSelExpr = null;
LambdaExpression resSelExpr = null;
if (lambda2.Parameters.Count == 1)
{
elemSelExpr = lambda2;
}
else if (lambda2.Parameters.Count == 2)
{
resSelExpr = lambda2;
}
resNode = this.VisitGroupBy(nodeInfo.children[0].child,
keySelExpr,
elemSelExpr,
resSelExpr,
expression.Arguments[3],
expression);
}
else if (keySelExpr != null && keySelExpr.Parameters.Count == 1 &&
lambda2 != null && lambda2.Parameters.Count == 1 &&
lambda3 != null && lambda3.Parameters.Count == 2)
{
//Argument-2 is element selector and argument-3 is result selector
resNode = this.VisitGroupBy(nodeInfo.children[0].child,
keySelExpr,
lambda2,
lambda3,
null,
expression);
}
}
else if (expression.Arguments.Count == 5)
{
//Supplied arguments are as follows:(source, key selector, element selector, result selector, comparer)
LambdaExpression keySelExpr = HpcLinqExpression.GetLambda(expression.Arguments[1]);
LambdaExpression elemSelExpr = HpcLinqExpression.GetLambda(expression.Arguments[2]);
LambdaExpression resSelExpr = HpcLinqExpression.GetLambda(expression.Arguments[3]);
if (keySelExpr != null && keySelExpr.Parameters.Count == 1 &&
elemSelExpr != null && elemSelExpr.Parameters.Count == 1 &&
resSelExpr != null && resSelExpr.Parameters.Count == 2)
{
resNode = this.VisitGroupBy(nodeInfo.children[0].child,
keySelExpr,
elemSelExpr,
resSelExpr,
expression.Arguments[4],
expression);
}
}
break;
}
case "OrderBy":
case "OrderByDescending":
{
bool isDescending = (methodName == "OrderByDescending");
if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitOrderBy(nodeInfo.children[0].child,
lambda,
null,
isDescending,
expression);
}
}
else if (expression.Arguments.Count == 3)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitOrderBy(nodeInfo.children[0].child,
lambda,
expression.Arguments[2],
isDescending,
expression);
}
}
break;
}
case "ThenBy":
case "ThenByDescending":
{
bool isDescending = (methodName == "ThenByDescending");
if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitThenBy(nodeInfo.children[0].child,
lambda,
isDescending,
expression);
}
}
break;
}
case "ElementAt":
case "ElementAtOrDefault":
{
if (expression.Arguments.Count == 2)
{
resNode = this.VisitElementAt(methodName,
nodeInfo.children[0].child,
expression.Arguments[1],
expression);
}
break;
}
case "Take":
{
if (expression.Arguments.Count == 2)
{
resNode = this.VisitPartitionOp(methodName,
nodeInfo.children[0].child,
QueryNodeType.Take,
expression.Arguments[1],
expression);
}
break;
}
case "TakeWhile":
case "LongTakeWhile":
{
if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null &&
(lambda.Parameters.Count == 1 || lambda.Parameters.Count == 2))
{
resNode = this.VisitPartitionOp("TakeWhile",
nodeInfo.children[0].child,
QueryNodeType.TakeWhile,
lambda,
expression);
}
}
break;
}
case "Skip":
{
if (expression.Arguments.Count == 2)
{
resNode = this.VisitPartitionOp(methodName,
nodeInfo.children[0].child,
QueryNodeType.Skip,
expression.Arguments[1],
expression);
}
break;
}
case "SkipWhile":
case "LongSkipWhile":
{
if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null &&
(lambda.Parameters.Count == 1 || lambda.Parameters.Count == 2))
{
resNode = this.VisitPartitionOp(methodName,
nodeInfo.children[0].child,
QueryNodeType.SkipWhile,
lambda,
expression);
}
}
break;
}
case "Contains":
case "ContainsAsQuery":
{
bool isQuery = (methodName == "ContainsAsQuery");
if (expression.Arguments.Count == 2)
{
resNode = this.VisitContains(nodeInfo.children[0].child,
expression.Arguments[1],
null,
isQuery,
expression);
}
else if (expression.Arguments.Count == 3)
{
resNode = this.VisitContains(nodeInfo.children[0].child,
expression.Arguments[1],
expression.Arguments[2],
isQuery,
expression);
}
break;
}
case "Reverse":
{
resNode = this.VisitReverse(nodeInfo.children[0].child, expression);
break;
}
case "SequenceEqual":
case "SequenceEqualAsQuery":
{
if (expression.Arguments.Count == 2)
{
resNode = this.VisitSequenceEqual(nodeInfo.children[0].child,
nodeInfo.children[1].child,
null,
expression);
}
else if (expression.Arguments.Count == 3)
{
resNode = this.VisitSequenceEqual(nodeInfo.children[0].child,
nodeInfo.children[1].child,
expression.Arguments[2],
expression);
}
break;
}
case "Zip":
{
if (expression.Arguments.Count == 3)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[2]);
if (lambda != null && lambda.Parameters.Count == 2)
{
resNode = this.VisitZip(nodeInfo.children[0].child,
nodeInfo.children[1].child,
lambda,
expression);
}
}
break;
}
case "HashPartition":
{
if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitHashPartition(nodeInfo.children[0].child,
lambda,
null,
null,
expression);
}
}
else if (expression.Arguments.Count == 3)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
if (expression.Arguments[2].Type == typeof(int))
{
resNode = this.VisitHashPartition(nodeInfo.children[0].child,
lambda,
null,
expression.Arguments[2],
expression);
}
else
{
resNode = this.VisitHashPartition(nodeInfo.children[0].child,
lambda,
expression.Arguments[2],
null,
expression);
}
}
}
else if (expression.Arguments.Count == 4)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitHashPartition(nodeInfo.children[0].child,
lambda,
expression.Arguments[2],
expression.Arguments[3],
expression);
}
}
break;
}
case "RangePartition":
{
//overloads:
//
// 2-param:
// (source, keySelector)
//
// 3-param:
// (source, keySelector, pcount)
// (source, keySelector, isDescending)
// (source, keySelector, rangeSeparators)
//
// 4-param:
// (source, keySelector, isDescending, pcount)
// (source, keySelector, keyComparer, isDescending)
// (source, keySelector, rangeSeparators, keyComparer)
//
// 5-param:
// (source, keySelector, keyComparer, isDescending, pcount)
// (source, keySelector, rangeSeparators, keyComparer, isDescending)
if (expression.Arguments.Count == 2)
{
// Case: (source, keySelector)
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
resNode = this.VisitRangePartition(nodeInfo.children[0].child,
lambda,
null,
null,
null,
null,
expression);
}
else if (expression.Arguments.Count == 3)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
if (expression.Arguments[2].Type == typeof(int))
{
// Case: (source, keySelector, pcount)
resNode = this.VisitRangePartition(nodeInfo.children[0].child,
lambda,
null,
null,
null,
expression.Arguments[2],
expression);
}
if (expression.Arguments[2].Type == typeof(bool))
{
// Case: (source, keySelector, isDescending)
resNode = this.VisitRangePartition(nodeInfo.children[0].child,
lambda,
null,
null,
expression.Arguments[2],
null,
expression);
}
else if (expression.Arguments[2].Type.IsArray)
{
// Case: RangePartition(keySelector, TKey[] keys)
resNode = this.VisitRangePartition(nodeInfo.children[0].child,
lambda,
expression.Arguments[2],
null,
null,
null,
expression);
}
}
}
else if (expression.Arguments.Count == 4)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
if (expression.Arguments[2].Type == typeof(bool))
{
//case: (source, keySelector, isDescending, pcount)
resNode = this.VisitRangePartition(nodeInfo.children[0].child,
lambda,
null,
null,
expression.Arguments[2],
expression.Arguments[3],
expression);
}
else if (expression.Arguments[3].Type == typeof(bool))
{
//case: (source, keySelector, keyComparer, isDescending)
resNode = this.VisitRangePartition(nodeInfo.children[0].child,
lambda,
null,
expression.Arguments[2],
expression.Arguments[3],
null,
expression);
}
else if (expression.Arguments[2].Type.IsArray)
{
//case: (source, keySelector, rangeSeparators, keyComparer)
resNode = this.VisitRangePartition(nodeInfo.children[0].child,
lambda,
expression.Arguments[2],
expression.Arguments[3],
null,
null,
expression);
}
}
}
else if (expression.Arguments.Count == 5)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
if (expression.Arguments[3].Type == typeof(bool))
{
// case: (source, keySelector, keyComparer, isDescending, pcount)
resNode = this.VisitRangePartition(nodeInfo.children[0].child,
lambda,
null,
expression.Arguments[2],
expression.Arguments[3],
expression.Arguments[4],
expression);
}
else if (expression.Arguments[4].Type == typeof(bool))
{
// case: (source, keySelector, rangeSeparators, keyComparer, isDescending)
resNode = this.VisitRangePartition(nodeInfo.children[0].child,
lambda,
expression.Arguments[2],
expression.Arguments[3],
expression.Arguments[4],
null,
expression);
}
}
}
break;
}
case "Apply":
{
if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null &&
(lambda.Parameters.Count == 1 || lambda.Parameters.Count == 2))
{
resNode = this.VisitApply(nodeInfo.children[0].child,
null,
lambda,
false,
false,
expression);
}
}
else if (expression.Arguments.Count == 3)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[2]);
if (lambda != null)
{
if (lambda.Parameters.Count == 2)
{
resNode = this.VisitApply(nodeInfo.children[0].child,
nodeInfo.children[1].child,
lambda,
false,
false,
expression);
}
else
{
// Apply with multiple sources of the same type
resNode = this.VisitMultiApply(nodeInfo, lambda, false, false, expression);
}
}
}
break;
}
case "ApplyPerPartition":
{
if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null &&
(lambda.Parameters.Count == 1 || lambda.Parameters.Count == 2))
{
resNode = this.VisitApply(nodeInfo.children[0].child,
null,
lambda,
true,
false,
expression);
}
}
else if (expression.Arguments.Count == 4)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[2]);
ExpressionSimplifier<bool> evaluator = new ExpressionSimplifier<bool>();
bool isFirstOnly = evaluator.Eval(expression.Arguments[3]);
if (lambda != null)
{
if (lambda.Parameters.Count == 2)
{
resNode = this.VisitApply(nodeInfo.children[0].child,
nodeInfo.children[1].child,
lambda,
true,
isFirstOnly,
expression);
}
else
{
// Apply with multiple sources of the same type
resNode = this.VisitMultiApply(nodeInfo, lambda, true, isFirstOnly, expression);
}
}
}
break;
}
case "Fork":
{
if (expression.Arguments.Count == 2) // ForkSelect and ForkApply
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitFork(nodeInfo.children[0].child,
lambda,
null,
expression);
}
}
else if (expression.Arguments.Count == 3) // ForkByKey
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitFork(nodeInfo.children[0].child,
lambda,
expression.Arguments[2],
expression);
}
}
break;
}
case "ForkChoose":
{
if (expression.Arguments.Count == 2)
{
resNode = this.VisitForkChoose(nodeInfo.children[0].child,
expression.Arguments[1],
expression);
}
break;
}
case "AssumeHashPartition":
{
if (expression.Arguments.Count == 2)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitAssumeHashPartition(nodeInfo.children[0].child,
lambda,
null,
null,
expression);
}
}
else if (expression.Arguments.Count == 3)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitAssumeHashPartition(nodeInfo.children[0].child,
lambda,
null,
expression.Arguments[2],
expression);
}
}
break;
}
case "AssumeRangePartition":
{
if (expression.Arguments.Count == 3)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
if (expression.Arguments[2].Type.IsArray)
{
resNode = this.VisitAssumeRangePartition(nodeInfo.children[0].child,
lambda,
expression.Arguments[2],
null,
null,
expression);
}
else
{
resNode = this.VisitAssumeRangePartition(nodeInfo.children[0].child,
lambda,
null,
null,
expression.Arguments[2],
expression);
}
}
}
else if (expression.Arguments.Count == 4)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
if (expression.Arguments[2].Type.IsArray)
{
resNode = this.VisitAssumeRangePartition(nodeInfo.children[0].child,
lambda,
expression.Arguments[2],
expression.Arguments[3],
null,
expression);
}
else
{
resNode = this.VisitAssumeRangePartition(nodeInfo.children[0].child,
lambda,
null,
expression.Arguments[2],
expression.Arguments[3],
expression);
}
}
}
break;
}
case "AssumeOrderBy":
{
if (expression.Arguments.Count == 3)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitAssumeOrderBy(nodeInfo.children[0].child,
lambda,
null,
expression.Arguments[2],
expression);
}
}
else if (expression.Arguments.Count == 4)
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitAssumeOrderBy(nodeInfo.children[0].child,
lambda,
expression.Arguments[2],
expression.Arguments[3],
expression);
}
}
break;
}
case "SlidingWindow":
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitSlidingWindow(nodeInfo.children[0].child,
lambda,
expression.Arguments[2],
expression);
}
break;
}
case "SelectWithPartitionIndex":
case "ApplyWithPartitionIndex":
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
if (lambda != null && lambda.Parameters.Count == 1)
{
resNode = this.VisitApplyWithPartitionIndex(nodeInfo.children[0].child,
lambda,
expression);
}
break;
}
case ReflectedNames.DryadLinqIQueryable_ToDscWorker: // was case "ToPartitionedTableLazy":
{
//SHOULD NOT VISIT.. The DryadQueryGen ctors should be interrogating ToDsc nodes directly.
//Later if we do allow ToDsc in the middle of query chain, then we need to either
// 1. update the source node with an outputeTableUri
// OR 2. create an actual node and handle it later on (tee, etc)
throw DryadLinqException.Create(HpcLinqErrorCode.ToDscUsedIncorrectly,
String.Format(SR.ToDscUsedIncorrectly),
expression);
}
case ReflectedNames.DryadLinqIQueryable_ToHdfsWorker: // was case "ToPartitionedTableLazy":
{
//SHOULD NOT VISIT.. The DryadQueryGen ctors should be interrogating ToDsc nodes directly.
//Later if we do allow ToDsc in the middle of query chain, then we need to either
// 1. update the source node with an outputeTableUri
// OR 2. create an actual node and handle it later on (tee, etc)
throw DryadLinqException.Create(HpcLinqErrorCode.ToHdfsUsedIncorrectly, String.Format(SR.ToHdfsUsedIncorrectly), expression);
}
}
#endregion
if (resNode == null)
{
throw DryadLinqException.Create(HpcLinqErrorCode.OperatorNotSupported,
String.Format(SR.OperatorNotSupported, methodName),
expression);
}
resNode.IsForked = resNode.IsForked || nodeInfo.IsForked;
nodeInfo.queryNode = resNode;
return resNode;
}
private DryadQueryNode VisitPartitionOp(string opName,
QueryNodeInfo source,
QueryNodeType nodeType,
Expression controlExpr,
MethodCallExpression queryExpr)
{
DryadQueryNode resNode;
if (nodeType == QueryNodeType.TakeWhile &&
controlExpr.Type.GetGenericArguments().Length != 2)
{
// The "indexed" version.
resNode = this.Visit(source);
// The following block used to be skipped for resNode.OutputPartition.Count == 1,
// which causes compilation error (bug 13593)
// @@TODO[p3] : implement a working optimization for nPartition==1 that calls
// directly to Linq TakeWhile.
// Note: the test is: if (resNode.IsDynamic || resNode.OutputPartition.Count > 1)
{
resNode.IsForked = true;
bool isLong = (queryExpr.Method.Name == "LongTakeWhile");
DryadQueryNode offsetNode = this.CreateOffset(isLong, queryExpr, resNode);
// Create (x, y) => GroupIndexedTakeWhile(x, y, controlExpr)
Type ptype1 = typeof(IEnumerable<>).MakeGenericType(resNode.OutputTypes[0]);
Type ptype2 = typeof(IEnumerable<>).MakeGenericType(typeof(IndexedValue<long>));
ParameterExpression param1 = Expression.Parameter(ptype1, HpcLinqCodeGen.MakeUniqueName("x"));
ParameterExpression param2 = Expression.Parameter(ptype2, HpcLinqCodeGen.MakeUniqueName("y"));
string methodName = "GroupIndexed" + queryExpr.Method.Name;
MethodInfo minfo = typeof(HpcLinqEnumerable).GetMethod(methodName);
minfo = minfo.MakeGenericMethod(resNode.OutputTypes[0]);
Expression body = Expression.Call(minfo, param1, param2, controlExpr);
Type type = typeof(Func<,,>).MakeGenericType(ptype1, ptype2, body.Type);
LambdaExpression procFunc = Expression.Lambda(type, body, param1, param2);
resNode = new DryadApplyNode(procFunc, queryExpr, resNode, offsetNode);
}
}
else if (!source.IsForked &&
(nodeType == QueryNodeType.Take || nodeType == QueryNodeType.TakeWhile) &&
(source.OperatorName == "OrderBy" || source.OperatorName == "OrderByDescending"))
{
resNode = this.Visit(source.children[0].child);
bool isDescending = (source.OperatorName == "OrderByDescending");
MethodCallExpression sourceQueryExpr = (MethodCallExpression)source.queryExpression;
LambdaExpression keySelectExpr = HpcLinqExpression.GetLambda(sourceQueryExpr.Arguments[1]);
Expression comparerExpr = null;
if (sourceQueryExpr.Arguments.Count == 3)
{
comparerExpr = sourceQueryExpr.Arguments[2];
}
resNode = this.PromoteConcat(
source.children[0].child,
resNode,
delegate(DryadQueryNode x) {
DryadQueryNode y = new DryadOrderByNode(keySelectExpr, comparerExpr,
isDescending, sourceQueryExpr, x);
return FirstStagePartitionOp(opName, nodeType, controlExpr, queryExpr, y);
});
if (resNode.IsDynamic || resNode.OutputPartition.Count > 1)
{
// Need a mergesort
resNode = new DryadMergeNode(keySelectExpr, comparerExpr, isDescending, false, sourceQueryExpr, resNode);
}
}
else
{
resNode = this.Visit(source);
if (nodeType == QueryNodeType.Take || nodeType == QueryNodeType.TakeWhile)
{
resNode = this.PromoteConcat(
source, resNode,
x => FirstStagePartitionOp(opName, nodeType, controlExpr, queryExpr, x));
}
}
resNode = new DryadPartitionOpNode(opName, nodeType, controlExpr, false, queryExpr, resNode);
return resNode;
}
private DryadQueryNode VisitOfType(QueryNodeInfo source,
Type ofType,
Expression queryExpr)
{
// YY: Not very useful.
throw DryadLinqException.Create(HpcLinqErrorCode.OperatorNotSupported,
String.Format(SR.OperatorNotSupported, "OfType"),
queryExpr);
}
private DryadQueryNode VisitBasicAggregate(QueryNodeInfo source,
LambdaExpression lambda,
AggregateOpType aggType,
bool isQuery,
Expression queryExpr)
{
DryadQueryNode child = this.Visit(source);
if (aggType == AggregateOpType.Min || aggType == AggregateOpType.Max)
{
Type elemType = child.OutputTypes[0];
if (lambda != null)
{
elemType = lambda.Body.Type;
}
if (!TypeSystem.HasDefaultComparer(elemType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.AggregationOperatorRequiresIComparable,
String.Format(SR.AggregationOperatorRequiresIComparable, aggType ),
queryExpr);
}
}
DryadQueryNode resNode = this.PromoteConcat(
source, child,
x => new DryadBasicAggregateNode(lambda, aggType, true, isQuery, queryExpr, x));
switch (aggType)
{
case AggregateOpType.Count:
case AggregateOpType.LongCount:
{
resNode = new DryadBasicAggregateNode(null, AggregateOpType.Sum, false,
isQuery, queryExpr, resNode);
break;
}
case AggregateOpType.Sum:
case AggregateOpType.Min:
case AggregateOpType.Max:
case AggregateOpType.Average:
{
resNode = new DryadBasicAggregateNode(null, aggType, false,
isQuery, queryExpr, resNode);
break;
}
default:
{
throw DryadLinqException.Create(HpcLinqErrorCode.OperatorNotSupported,
String.Format(SR.OperatorNotSupported, aggType),
queryExpr);
}
}
return resNode;
}
private DryadQueryNode VisitAssumeRangePartition(QueryNodeInfo source,
LambdaExpression keySelectExpr,
Expression keysExpr,
Expression comparerExpr,
Expression isDescendingExpr,
Expression queryExpr)
{
DryadQueryNode child = this.Visit(source);
Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
if (comparerExpr == null && !TypeSystem.HasDefaultComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable,
String.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable, keyType),
queryExpr);
}
object comparer = null;
if (comparerExpr != null)
{
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
comparer = evaluator.Eval(comparerExpr);
}
object keys = null;
if (keysExpr != null)
{
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
keys = evaluator.Eval(keysExpr);
}
//count the number of keys provided.
if (keys != null)
{
int nSeparators = 0;
var ie = ((IEnumerable)keys).GetEnumerator();
while (ie.MoveNext())
{
nSeparators++;
}
if (!child.IsDynamic && nSeparators != child.PartitionCount - 1)
{
throw DryadLinqException.Create(
HpcLinqErrorCode.BadSeparatorCount,
String.Format(SR.BadSeparatorCount, nSeparators, child.PartitionCount - 1),
queryExpr);
}
}
bool? isDescending = null;
if (isDescendingExpr != null)
{
ExpressionSimplifier<bool> evaluator = new ExpressionSimplifier<bool>();
isDescending = evaluator.Eval(isDescendingExpr);
}
DataSetInfo outputInfo = new DataSetInfo(child.OutputDataSetInfo);
outputInfo.partitionInfo = PartitionInfo.CreateRange(keySelectExpr,
keys,
comparer,
isDescending,
child.OutputPartition.Count,
keyType);
child.OutputDataSetInfo = outputInfo;
return child;
}
public NodeInfoEdge(QueryNodeInfo parent, QueryNodeInfo child)
{
this.Parent = parent;
this.Child = child;
}
// Return true iff EPG is modified by this method.
public bool RewriteOne(int idx)
{
QueryNodeInfo curNode = this.m_nodeInfos[idx];
if (curNode.OperatorName == "Where" && !curNode.Children[0].Child.IsForked)
{
LambdaExpression lambda = DryadLinqExpression.GetLambda(((MethodCallExpression)curNode.QueryExpression).Arguments[1]);
if (lambda.Type.GetGenericArguments().Length == 2)
{
QueryNodeInfo child = curNode.Children[0].Child;
string[] names = new string[] { "OrderBy", "Distinct", "RangePartition", "HashPartition" };
if (names.Contains(child.OperatorName))
{
curNode.Swap(child);
return true;
}
if (child.OperatorName == "Concat")
{
curNode.Delete();
for (int i = 0; i < child.Children.Count; i++)
{
NodeInfoEdge edge = child.Children[i];
QueryNodeInfo node = curNode.Clone();
this.m_nodeInfos.Add(node);
edge.Insert(node);
}
return true;
}
}
}
else if ((curNode.OperatorName == "Select" || curNode.OperatorName == "SelectMany") &&
!curNode.Children[0].Child.IsForked)
{
LambdaExpression lambda = DryadLinqExpression.GetLambda(((MethodCallExpression)curNode.QueryExpression).Arguments[1]);
if (lambda.Type.GetGenericArguments().Length == 2)
{
QueryNodeInfo child = curNode.Children[0].Child;
if (child.OperatorName == "Concat")
{
curNode.Delete();
for (int i = 0; i < child.Children.Count; i++)
{
NodeInfoEdge edge = child.Children[i];
QueryNodeInfo node = curNode.Clone();
this.m_nodeInfos.Add(node);
edge.Insert(node);
}
return true;
}
}
}
else if (curNode.OperatorName == "Take" && !curNode.Children[0].Child.IsForked)
{
QueryNodeInfo child = curNode.Children[0].Child;
if (child.OperatorName == "Select")
{
QueryNodeInfo cchild = child.Children[0].Child;
if (cchild.OperatorName != "GroupBy")
{
curNode.Swap(child);
return true;
}
}
}
else if ((curNode.OperatorName == "Contains" ||
curNode.OperatorName == "ContainsAsQuery" ||
curNode.OperatorName == "All" ||
curNode.OperatorName == "AllAsQuery" ||
curNode.OperatorName == "Any" ||
curNode.OperatorName == "AnyAsQuery") &&
!curNode.Children[0].Child.IsForked)
{
QueryNodeInfo child = curNode.Children[0].Child;
string[] names = new string[] { "OrderBy", "Distinct", "RangePartition", "HashPartition" };
if (names.Contains(child.OperatorName))
{
child.Delete();
return true;
}
}
return false;
}
private DryadQueryNode VisitWhere(QueryNodeInfo source,
LambdaExpression predicate,
MethodCallExpression queryExpr)
{
DryadQueryNode child = this.Visit(source);
DryadQueryNode whereNode;
if (predicate.Type.GetGenericArguments().Length == 2 ||
(!child.IsDynamic && child.OutputPartition.Count == 1))
{
whereNode = this.PromoteConcat(source, child, x => new DryadWhereNode(predicate, queryExpr, x));
}
else
{
// The "indexed" version
// Create (x, y) => DryadWhere(x, y, predicate)
Type ptype1 = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
Type ptype2 = typeof(IEnumerable<>).MakeGenericType(typeof(IndexedValue<long>));
ParameterExpression param1 = Expression.Parameter(ptype1, HpcLinqCodeGen.MakeUniqueName("x"));
ParameterExpression param2 = Expression.Parameter(ptype2, HpcLinqCodeGen.MakeUniqueName("y"));
string targetMethod = queryExpr.Method.Name + "WithStartIndex";
MethodInfo minfo = typeof(HpcLinqEnumerable).GetMethod(targetMethod);
minfo = minfo.MakeGenericMethod(child.OutputTypes[0]);
Expression body = Expression.Call(minfo, param1, param2, predicate);
Type type = typeof(Func<,,>).MakeGenericType(ptype1, ptype2, body.Type);
LambdaExpression procFunc = Expression.Lambda(type, body, param1, param2);
child.IsForked = true;
bool isLong = (queryExpr.Method.Name == "LongWhere");
DryadQueryNode offsetNode = this.CreateOffset(isLong, queryExpr, child);
whereNode = new DryadApplyNode(procFunc, queryExpr, child, offsetNode);
}
return whereNode;
}
internal DryadQueryNode Visit(QueryNodeInfo nodeInfo)
{
Expression expression = nodeInfo.queryExpression;
if (expression.NodeType == ExpressionType.Call)
{
MethodCallExpression mcExpr = (MethodCallExpression)expression;
if (mcExpr.Method.IsStatic && TypeSystem.IsQueryOperatorCall(mcExpr))
{
return this.VisitQueryOperatorCall(nodeInfo);
}
throw DryadLinqException.Create(HpcLinqErrorCode.OperatorNotSupported,
String.Format(SR.OperatorNotSupported, mcExpr.Method.Name),
expression);
}
else if (expression.NodeType == ExpressionType.Constant)
{
DryadInputNode inputNode = new DryadInputNode(this, (ConstantExpression)expression);
if (!this.m_inputUriMap.ContainsKey(inputNode.Table.DataSourceUri.ToLower()))
{
this.m_inputUriMap.Add(inputNode.Table.DataSourceUri.ToLower(), inputNode);
}
return inputNode;
}
else
{
string errMsg = "Can't handle expression of type " + expression.NodeType;
throw DryadLinqException.Create(HpcLinqErrorCode.UnsupportedExpressionsType,
String.Format(SR.UnsupportedExpressionsType,expression.NodeType),
expression);
}
}
private DryadQueryNode VisitConcat(QueryNodeInfo source, MethodCallExpression queryExpr)
{
DryadQueryNode[] childs = new DryadQueryNode[source.children.Count];
for (int i = 0; i < source.children.Count; ++i)
{
childs[i] = this.Visit(source.children[i].child);
}
DryadQueryNode resNode = new DryadConcatNode(queryExpr, childs);
int parCount = resNode.OutputPartition.Count;
if (!resNode.IsDynamic && parCount > StaticConfig.MaxPartitionCount)
{
// Too many partitions, need to repartition
int newParCount = parCount / 2;
DryadQueryNode countNode = new DryadBasicAggregateNode(null, AggregateOpType.LongCount,
true, false, queryExpr, resNode);
DryadQueryNode mergeCountNode = new DryadMergeNode(true, false, queryExpr, countNode);
// Apply node for s => IndexedCount(s)
Type paramType = typeof(IEnumerable<>).MakeGenericType(typeof(long));
ParameterExpression param = Expression.Parameter(paramType, "s");
MethodInfo minfo = typeof(HpcLinqHelper).GetMethod("IndexedCount");
minfo = minfo.MakeGenericMethod(typeof(long));
Expression body = Expression.Call(minfo, param);
Type funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
LambdaExpression indexedCountFunc = Expression.Lambda(funcType, body, param);
DryadQueryNode indexedCountNode = new DryadApplyNode(indexedCountFunc, queryExpr, mergeCountNode);
// HashPartition(x => x.index, parCount)
param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
Expression keySelectBody = Expression.Property(param, "Index");
funcType = typeof(Func<,>).MakeGenericType(param.Type, keySelectBody.Type);
LambdaExpression keySelectExpr = Expression.Lambda(funcType, keySelectBody, param);
DryadQueryNode distCountNode = new DryadHashPartitionNode(keySelectExpr,
null,
parCount,
queryExpr,
indexedCountNode);
// Apply node for (x, y) => AddPartitionIndex(x, y, newParCount)
ParameterExpression param1 = Expression.Parameter(body.Type, "x");
Type paramType2 = typeof(IEnumerable<>).MakeGenericType(resNode.OutputTypes[0]);
ParameterExpression param2 = Expression.Parameter(paramType2, "y");
minfo = typeof(HpcLinqHelper).GetMethod("AddPartitionIndex");
minfo = minfo.MakeGenericMethod(resNode.OutputTypes[0]);
body = Expression.Call(minfo, param1, param2, Expression.Constant(newParCount));
funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
LambdaExpression addIndexFunc = Expression.Lambda(funcType, body, param1, param2);
DryadQueryNode addIndexNode = new DryadApplyNode(addIndexFunc, queryExpr, distCountNode, resNode);
// HashPartition(x => x.index, x => x.value, newParCount)
param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
body = Expression.Property(param, "Index");
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
keySelectExpr = Expression.Lambda(funcType, body, param);
body = Expression.Property(param, "Value");
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
LambdaExpression resultSelectExpr = Expression.Lambda(funcType, body, param);
resNode = new DryadHashPartitionNode(keySelectExpr,
resultSelectExpr,
null,
newParCount,
false,
queryExpr,
addIndexNode);
resNode = new DryadMergeNode(true, true, queryExpr, resNode);
}
return resNode;
}
private DryadQueryNode VisitMultiApply(QueryNodeInfo source,
LambdaExpression procLambda,
bool perPartition,
bool isFirstOnly,
MethodCallExpression queryExpr)
{
DryadQueryNode[] childs = new DryadQueryNode[source.children.Count];
for (int i = 0; i < source.children.Count; ++i)
{
childs[i] = this.Visit(source.children[i].child);
}
bool isDynamic = childs.Any(x => x.IsDynamic);
if (perPartition && !isDynamic)
{
// Homomorphic case.
if (isFirstOnly)
{
for (int i = 1; i < childs.Length; ++i)
{
childs[i] = new DryadTeeNode(childs[i].OutputTypes[0], true, queryExpr, childs[i]);
childs[i].ConOpType = ConnectionOpType.CrossProduct;
childs[i] = new DryadMergeNode(childs[0].OutputPartition.Count, queryExpr, childs[i]);
}
}
else
{
int count = childs[0].OutputPartition.Count;
for (int i = 1; i < childs.Length; ++i)
{
if (childs[i].OutputPartition.Count != count)
{
throw DryadLinqException.Create(HpcLinqErrorCode.HomomorphicApplyNeedsSamePartitionCount,
SR.HomomorphicApplyNeedsSamePartitionCount,
queryExpr);
}
}
}
}
else
{
// Non-homomorphic case.
for (int i = 0; i < childs.Length; ++i)
{
if (childs[i].IsDynamic || childs[i].OutputPartition.Count > 1)
{
childs[i] = new DryadMergeNode(true, false, queryExpr, childs[i]);
}
}
}
DryadQueryNode applyNode = new DryadApplyNode(procLambda, true, queryExpr, childs);
return applyNode;
}
private DryadQueryNode VisitContains(QueryNodeInfo source,
Expression valueExpr,
Expression comparerExpr,
bool isQuery,
Expression queryExpr)
{
DryadQueryNode child = this.Visit(source);
Type keyType = child.OutputTypes[0];
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
queryExpr);
}
DryadQueryNode resNode = this.PromoteConcat(
source, child,
x => new DryadContainsNode(valueExpr, comparerExpr, queryExpr, x));
resNode = new DryadBasicAggregateNode(null, AggregateOpType.Any, false, isQuery, queryExpr, resNode);
return resNode;
}
private DryadQueryNode VisitOrderBy(QueryNodeInfo source,
LambdaExpression keySelectExpr,
Expression comparerExpr,
bool isDescending,
Expression queryExpr)
{
DryadQueryNode child = this.Visit(source);
Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
if (comparerExpr == null && !TypeSystem.HasDefaultComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable, keyType),
queryExpr);
}
DryadQueryNode resNode = child;
if (child.OutputPartition.ParType == PartitionType.Range &&
child.OutputPartition.IsPartitionedBy(keySelectExpr, comparerExpr, isDescending))
{
// Only need to sort each partition
resNode = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending, queryExpr, child);
}
else
{
Expression isDescendingExpr = Expression.Constant(isDescending);
bool dynamicOptEnabled = (StaticConfig.DynamicOptLevel & StaticConfig.DynamicRangePartitionLevel) != 0;
if (dynamicOptEnabled || child.IsDynamic)
{
// NOTE: for MayRTM, this path should not be taken.
resNode = this.CreateRangePartition(true, keySelectExpr, null, comparerExpr,
isDescendingExpr, queryExpr, null, child);
resNode = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending, queryExpr, resNode);
}
else
{
if (child.OutputPartition.Count > 1)
{
resNode = this.CreateRangePartition(false, keySelectExpr, null, comparerExpr,
isDescendingExpr, queryExpr, null, child);
}
resNode = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending, queryExpr, resNode);
}
}
return resNode;
}
private DryadQueryNode VisitDefaultIfEmpty(QueryNodeInfo source,
Expression defaultValueExpr,
MethodCallExpression queryExpr)
{
// YY: Not very useful. We could add it later.
throw DryadLinqException.Create(HpcLinqErrorCode.OperatorNotSupported,
String.Format(SR.OperatorNotSupported, "DefaultIfEmpty"),
queryExpr);
}
private DryadQueryNode VisitQuantifier(QueryNodeInfo source,
LambdaExpression lambda,
AggregateOpType aggType,
bool isQuery,
Expression queryExpr)
{
DryadQueryNode child = this.Visit(source);
DryadQueryNode resNode = this.PromoteConcat(
source, child,
x => new DryadBasicAggregateNode(
lambda, aggType, true, isQuery, queryExpr, x));
resNode = new DryadBasicAggregateNode(null, aggType, false, isQuery, queryExpr, resNode);
return resNode;
}
private DryadQueryNode VisitDistinct(QueryNodeInfo source,
Expression comparerExpr,
Expression queryExpr)
{
DryadQueryNode child = this.Visit(source);
Type keyType = child.OutputTypes[0];
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
queryExpr);
}
object comparer = null;
if (comparerExpr != null)
{
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
comparer = evaluator.Eval(comparerExpr);
}
LambdaExpression keySelectExpr = IdentityFunction.Instance(keyType);
if (!child.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
if (child.IsDynamic || child.OutputPartition.Count > 1)
{
child = new DryadDistinctNode(true, comparerExpr, queryExpr, child);
bool isDynamic = (StaticConfig.DynamicOptLevel & StaticConfig.DynamicHashPartitionLevel) != 0;
child = new DryadHashPartitionNode(keySelectExpr,
comparerExpr,
child.OutputPartition.Count,
isDynamic,
queryExpr,
child);
child = new DryadMergeNode(false, false, queryExpr, child);
}
}
DryadQueryNode resNode = new DryadDistinctNode(false, comparerExpr, queryExpr, child);
return resNode;
}
private DryadQueryNode VisitRangePartition(QueryNodeInfo source,
LambdaExpression keySelectExpr,
Expression keysExpr,
Expression comparerExpr,
Expression isDescendingExpr,
Expression partitionCountExpr,
Expression queryExpr)
{
DryadQueryNode child = this.Visit(source);
Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
if (comparerExpr == null && !TypeSystem.HasDefaultComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable, keyType),
queryExpr);
}
DryadQueryNode resNode;
if (keysExpr == null)
{
// case: no keys are provided -- create range partitioner with auto-separator-selection
bool dynamicOptEnabled = (StaticConfig.DynamicOptLevel & StaticConfig.DynamicRangePartitionLevel) != 0;
bool useDynamic = (dynamicOptEnabled || child.IsDynamic);
// NOTE: for MayRTM, useDynamic should always be false
resNode = this.CreateRangePartition(useDynamic, keySelectExpr, null, comparerExpr, isDescendingExpr,
queryExpr, partitionCountExpr, child);
}
else
{
// case: keys are local enum (eg an array) -- create range partitioner with keys input via Object-store.
resNode = new DryadRangePartitionNode(keySelectExpr, null, keysExpr, comparerExpr,
isDescendingExpr, null, queryExpr, child);
resNode = new DryadMergeNode(false, true, queryExpr, resNode);
}
return resNode;
}
private DryadQueryNode VisitElementAt(string opName,
QueryNodeInfo source,
Expression indexExpr,
Expression queryExpr)
{
// YY: Not very useful. We could add it later.
throw DryadLinqException.Create(HpcLinqErrorCode.OperatorNotSupported,
String.Format(SR.OperatorNotSupported, opName),
queryExpr);
}
private DryadQueryNode VisitSelect(QueryNodeInfo source,
QueryNodeType nodeType,
LambdaExpression selector,
LambdaExpression resultSelector,
MethodCallExpression queryExpr)
{
DryadQueryNode selectNode;
if (selector.Type.GetGenericArguments().Length == 2)
{
// If this select's child is a groupby node, push this select into its child, if
// 1. The groupby node is not tee'd, and
// 2. The groupby node has no result selector, and
// 3. The selector is decomposable
if (!source.IsForked &&
IsGroupByWithoutResultSelector(source.queryExpression) &&
Decomposition.GetDecompositionInfoList(selector, m_codeGen) != null)
{
MethodCallExpression expr = (MethodCallExpression)source.queryExpression;
LambdaExpression keySelectExpr = HpcLinqExpression.GetLambda(expr.Arguments[1]);
// Figure out elemSelectExpr and comparerExpr
LambdaExpression elemSelectExpr = null;
Expression comparerExpr = null;
if (expr.Arguments.Count == 3)
{
elemSelectExpr = HpcLinqExpression.GetLambda(expr.Arguments[2]);
if (elemSelectExpr == null)
{
comparerExpr = expr.Arguments[2];
}
}
else if (expr.Arguments.Count == 4)
{
elemSelectExpr = HpcLinqExpression.GetLambda(expr.Arguments[2]);
comparerExpr = expr.Arguments[3];
}
// Construct new query expression by building result selector expression
// and pushing it to groupby node.
selectNode = VisitGroupBy(source.children[0].child, keySelectExpr,
elemSelectExpr, selector, comparerExpr, queryExpr);
if (nodeType == QueryNodeType.SelectMany)
{
Type selectorRetType = selector.Type.GetGenericArguments()[1];
LambdaExpression id = IdentityFunction.Instance(selectorRetType);
selectNode = new DryadSelectNode(nodeType, id, resultSelector, queryExpr, selectNode);
}
}
else
{
DryadQueryNode child = this.Visit(source);
selectNode = this.PromoteConcat(
source, child,
x => new DryadSelectNode(nodeType, selector, resultSelector, queryExpr, x));
}
}
else
{
// The "indexed" version
DryadQueryNode child = this.Visit(source);
if (!child.IsDynamic && child.OutputPartition.Count == 1)
{
selectNode = this.PromoteConcat(
source, child,
x => new DryadSelectNode(nodeType, selector, resultSelector, queryExpr, x));
}
else
{
child.IsForked = true;
// Create (x, y) => Select(x, y, selector)
Type ptype1 = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
Type ptype2 = typeof(IEnumerable<>).MakeGenericType(typeof(IndexedValue<long>));
ParameterExpression param1 = Expression.Parameter(ptype1, HpcLinqCodeGen.MakeUniqueName("x"));
ParameterExpression param2 = Expression.Parameter(ptype2, HpcLinqCodeGen.MakeUniqueName("y"));
string methodName = queryExpr.Method.Name;
Type[] selectorTypeArgs = selector.Type.GetGenericArguments();
Type typeArg2 = selectorTypeArgs[selectorTypeArgs.Length - 1];
if (nodeType == QueryNodeType.SelectMany)
{
if (resultSelector != null)
{
methodName += "Result";
}
typeArg2 = typeArg2.GetGenericArguments()[0];
}
string targetMethodName = methodName + "WithStartIndex";
MethodInfo minfo = typeof(HpcLinqEnumerable).GetMethod(targetMethodName);
Expression body;
if (resultSelector == null)
{
minfo = minfo.MakeGenericMethod(child.OutputTypes[0], typeArg2);
body = Expression.Call(minfo, param1, param2, selector);
}
else
{
minfo = minfo.MakeGenericMethod(child.OutputTypes[0], typeArg2, resultSelector.Body.Type);
body = Expression.Call(minfo, param1, param2, selector, resultSelector);
}
Type type = typeof(Func<,,>).MakeGenericType(ptype1, ptype2, body.Type);
LambdaExpression procFunc = Expression.Lambda(type, body, param1, param2);
bool isLong = methodName.StartsWith("Long", StringComparison.Ordinal);
DryadQueryNode offsetNode = this.CreateOffset(isLong, queryExpr, child);
selectNode = new DryadApplyNode(procFunc, queryExpr, child, offsetNode);
}
}
return selectNode;
}
private DryadQueryNode VisitFork(QueryNodeInfo source,
LambdaExpression forkLambda,
Expression keysExpr,
Expression queryExpr)
{
DryadQueryNode child = this.Visit(source);
return new DryadForkNode(forkLambda, keysExpr, queryExpr, child);
}
private DryadQueryNode VisitSetOperation(QueryNodeInfo source1,
QueryNodeInfo source2,
QueryNodeType nodeType,
Expression comparerExpr,
Expression queryExpr)
{
DryadQueryNode child1 = this.Visit(source1);
DryadQueryNode child2 = this.Visit(source2);
DryadQueryNode resNode = null;
Type keyType = child1.OutputTypes[0];
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
queryExpr);
}
// The comparer object:
object comparer = null;
if (comparerExpr != null)
{
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
comparer = evaluator.Eval(comparerExpr);
}
LambdaExpression keySelectExpr = IdentityFunction.Instance(keyType);
if (child1.IsDynamic || child2.IsDynamic)
{
// Well, let us do the simplest thing for now
child1 = new DryadHashPartitionNode(keySelectExpr,
null,
StaticConfig.DefaultPartitionCount,
queryExpr,
child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
child2 = new DryadHashPartitionNode(keySelectExpr,
null,
StaticConfig.DefaultPartitionCount,
queryExpr,
child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
resNode = new DryadSetOperationNode(nodeType, nodeType.ToString(), comparerExpr,
queryExpr, child1, child2);
return resNode;
}
bool isDescending1 = child1.OutputDataSetInfo.orderByInfo.IsDescending;
bool isDescending2 = child2.OutputDataSetInfo.orderByInfo.IsDescending;
// Partition child1 and child2 if needed
if (child1.OutputPartition.ParType == PartitionType.Range &&
child1.OutputPartition.HasKeys &&
child1.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
if (child2.OutputPartition.ParType != PartitionType.Range ||
!child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer) ||
child1.OutputPartition.IsSamePartition(child2.OutputPartition))
{
// Range distribute child2 using child1's partition
child2 = child1.OutputPartition.CreatePartitionNode(keySelectExpr, child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
}
}
else if (child2.OutputPartition.ParType == PartitionType.Range &&
child2.OutputPartition.HasKeys &&
child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
// Range distribute child1 using child2's partition
child1 = child2.OutputPartition.CreatePartitionNode(keySelectExpr, child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
}
else if (child1.OutputPartition.ParType == PartitionType.Hash &&
child1.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
if (child2.OutputPartition.ParType != PartitionType.Hash ||
!child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer) ||
!child1.OutputPartition.IsSamePartition(child2.OutputPartition))
{
// Hash distribute child2:
child2 = new DryadHashPartitionNode(keySelectExpr,
comparerExpr,
child1.OutputPartition.Count,
queryExpr,
child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
}
}
else if (child2.OutputPartition.ParType == PartitionType.Hash &&
child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
child1 = new DryadHashPartitionNode(keySelectExpr,
comparerExpr,
child2.OutputPartition.Count,
queryExpr,
child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
}
else
{
// No luck. Hash distribute both child1 and child2, then perform hash operation
int parCnt = Math.Max(child1.OutputPartition.Count, child2.OutputPartition.Count);
if (parCnt > 1)
{
child1 = new DryadHashPartitionNode(keySelectExpr, comparerExpr, parCnt, queryExpr, child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
child2 = new DryadHashPartitionNode(keySelectExpr, comparerExpr, parCnt, queryExpr, child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
}
}
// Perform either hash or ordered operation
string opName = "";
if (child1.IsOrderedBy(keySelectExpr, comparer))
{
if (!child1.IsOrderedBy(keySelectExpr, comparer) ||
isDescending1 != isDescending2)
{
// Sort inner if unsorted
child2 = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending1, queryExpr, child2);
}
opName = "Ordered";
}
else if (child2.IsOrderedBy(keySelectExpr, comparer))
{
if (!child1.IsOrderedBy(keySelectExpr, comparer) ||
isDescending1 != isDescending2)
{
// Sort outer if unsorted
child1 = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending2, queryExpr, child1);
}
opName = "Ordered";
}
resNode = new DryadSetOperationNode(nodeType, opName + nodeType, comparerExpr, queryExpr, child1, child2);
return resNode;
}
private DryadQueryNode VisitForkChoose(QueryNodeInfo source,
Expression indexExpr,
Expression queryExpr)
{
DryadQueryNode child = this.Visit(source);
ExpressionSimplifier<int> evaluator = new ExpressionSimplifier<int>();
int index = evaluator.Eval(indexExpr);
return ((DryadForkNode)child).Parents[index];
}
private DryadQueryNode VisitThenBy(QueryNodeInfo source,
LambdaExpression keySelectExpr,
bool isDescending,
Expression queryExpr)
{
// YY: This makes it hard to maintain OrderByInfo.
throw DryadLinqException.Create(HpcLinqErrorCode.OperatorNotSupported,
String.Format(SR.OperatorNotSupported, "ThenBy"),
queryExpr);
}
private DryadQueryNode VisitGroupBy(QueryNodeInfo source,
LambdaExpression keySelectExpr,
LambdaExpression elemSelectExpr,
LambdaExpression resultSelectExpr,
Expression comparerExpr,
Expression queryExpr)
{
DryadQueryNode child = this.Visit(source);
ExpressionInfo einfo = new ExpressionInfo(keySelectExpr);
if (einfo.IsExpensive)
{
// Any method call that is not tagged as "expensive=false" will be deemed expensive.
// if the keySelector is expensive, we rewrite the query so that the key-function is invoked only once
// and the record key passed around via a Pair<TKey,TRecord>.
// keyFunc becomes pair=>pair.Key
// elementSelector must be rewritten so that references to (record) become (pair.Value)
Type[] vkTypes = keySelectExpr.Type.GetGenericArguments();
Type pairType = typeof(Pair<,>).MakeGenericType(vkTypes[1], vkTypes[0]);
ParameterExpression pairParam = Expression.Parameter(pairType, "e");
// Add Select(x => new Pair<K,S>(key(x), x))
ParameterExpression valueParam = keySelectExpr.Parameters[0];
Expression body = Expression.New(pairType.GetConstructors()[0], keySelectExpr.Body, valueParam);
Type delegateType = typeof(Func<,>).MakeGenericType(valueParam.Type, body.Type);
LambdaExpression selectExpr = Expression.Lambda(delegateType, body, valueParam);
child = new DryadSelectNode(QueryNodeType.Select, selectExpr, null, queryExpr, child);
// Change keySelector to e => e.Key
PropertyInfo keyInfo = pairParam.Type.GetProperty("Key");
body = Expression.Property(pairParam, keyInfo);
delegateType = typeof(Func<,>).MakeGenericType(pairParam.Type, body.Type);
keySelectExpr = Expression.Lambda(delegateType, body, pairParam);
// Add or change elementSelector with e.Value
PropertyInfo valueInfo = pairParam.Type.GetProperty("Value");
body = Expression.Property(pairParam, valueInfo);
if (elemSelectExpr != null)
{
ParameterSubst subst = new ParameterSubst(elemSelectExpr.Parameters[0], body);
body = subst.Visit(elemSelectExpr.Body);
}
delegateType = typeof(Func<,>).MakeGenericType(pairParam.Type, body.Type);
elemSelectExpr = Expression.Lambda(delegateType, body, pairParam);
}
Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
queryExpr);
}
Type elemType;
if (elemSelectExpr == null)
{
elemType = keySelectExpr.Type.GetGenericArguments()[0];
}
else
{
elemType = elemSelectExpr.Type.GetGenericArguments()[1];
}
// The comparer object:
object comparer = null;
if (comparerExpr != null)
{
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
comparer = evaluator.Eval(comparerExpr);
}
LambdaExpression keySelectExpr1 = keySelectExpr;
LambdaExpression elemSelectExpr1 = elemSelectExpr;
LambdaExpression resultSelectExpr1 = resultSelectExpr;
LambdaExpression seedExpr1 = null;
LambdaExpression accumulateExpr1 = null;
List<DecompositionInfo> dInfoList = null;
if (resultSelectExpr != null)
{
dInfoList = Decomposition.GetDecompositionInfoList(resultSelectExpr, this.m_codeGen);
}
String groupByOpName = "GroupBy";
DryadQueryNode groupByNode = child;
bool isPartitioned = child.OutputPartition.IsPartitionedBy(keySelectExpr, comparer);
if (dInfoList != null)
{
// ** Decomposable GroupBy-Reduce
// This block creates the first GroupByNode and does some preparation for subsequent nodes.
if (child.IsOrderedBy(keySelectExpr, comparer))
{
groupByOpName = "OrderedGroupBy";
}
int dcnt = dInfoList.Count;
ParameterExpression keyParam;
if (resultSelectExpr.Parameters.Count == 1)
{
keyParam = Expression.Parameter(keyType, HpcLinqCodeGen.MakeUniqueName("k"));
}
else
{
keyParam = resultSelectExpr.Parameters[0];
}
// Seed:
ParameterExpression param2 = Expression.Parameter(
elemType, HpcLinqCodeGen.MakeUniqueName("e"));
Expression zeroExpr = Expression.Constant(0, typeof(int));
Expression seedBody = zeroExpr;
if (dcnt != 0)
{
LambdaExpression seed = dInfoList[dcnt-1].Seed;
ParameterSubst subst = new ParameterSubst(seed.Parameters[0], param2);
seedBody = subst.Visit(seed.Body);
for (int i = dcnt - 2; i >= 0; i--)
{
seed = dInfoList[i].Seed;
subst = new ParameterSubst(seed.Parameters[0], param2);
Expression firstExpr = subst.Visit(seed.Body);
Type newPairType = typeof(Pair<,>).MakeGenericType(firstExpr.Type, seedBody.Type);
seedBody = Expression.New(newPairType.GetConstructors()[0], firstExpr, seedBody);
}
}
LambdaExpression seedExpr = Expression.Lambda(seedBody, param2);
// Accumulate:
ParameterExpression param1 = Expression.Parameter(
seedBody.Type, HpcLinqCodeGen.MakeUniqueName("a"));
Expression accumulateBody = zeroExpr;
if (dcnt != 0)
{
accumulateBody = Decomposition.AccumulateList(param1, param2, dInfoList, 0);
}
LambdaExpression accumulateExpr = Expression.Lambda(accumulateBody, param1, param2);
// Now prepare for the merge-aggregator and/or in the secondary group-by.
// keySelectExpr1: e => e.Key
Type reducerResType = typeof(Pair<,>).MakeGenericType(keyParam.Type, accumulateBody.Type);
ParameterExpression reducerResParam = Expression.Parameter(reducerResType, "e");
PropertyInfo keyInfo = reducerResParam.Type.GetProperty("Key");
Expression body = Expression.Property(reducerResParam, keyInfo);
Type delegateType = typeof(Func<,>).MakeGenericType(reducerResParam.Type, body.Type);
keySelectExpr1 = Expression.Lambda(delegateType, body, reducerResParam);
// elemSelectExpr1: e => e.Value
PropertyInfo valueInfo = reducerResParam.Type.GetProperty("Value");
body = Expression.Property(reducerResParam, valueInfo);
delegateType = typeof(Func<,>).MakeGenericType(reducerResParam.Type, body.Type);
elemSelectExpr1 = Expression.Lambda(delegateType, body, reducerResParam);
// SeedExpr1
param2 = Expression.Parameter(elemSelectExpr1.Body.Type,
HpcLinqCodeGen.MakeUniqueName("e"));
seedExpr1 = Expression.Lambda(param2, param2);
// AccumulateExpr1
Expression recursiveAccumulateBody = zeroExpr;
if (dcnt != 0)
{
recursiveAccumulateBody = Decomposition.RecursiveAccumulateList(param1, param2, dInfoList, 0);
}
accumulateExpr1 = Expression.Lambda(recursiveAccumulateBody, param1, param2);
// resultSelectExpr1
resultSelectExpr1 = null;
// The first groupByNode.
// If the input was already correctly partitioned, this will be the only groupByNode.
bool isPartial = StaticConfig.GroupByLocalAggregationIsPartial && !isPartitioned;
groupByNode = new DryadGroupByNode(
groupByOpName, keySelectExpr, elemSelectExpr, null,
seedExpr, accumulateExpr, accumulateExpr1, comparerExpr,
isPartial, queryExpr, child);
}
else
{
// Can't do partial aggregation.
// Use sort, mergesort, and ordered groupby, if TKey implements IComparable.
if ((comparer != null && TypeSystem.IsComparer(comparer, keyType)) ||
(comparer == null && TypeSystem.HasDefaultComparer(keyType)))
{
if (!child.IsOrderedBy(keySelectExpr, comparer))
{
groupByNode = new DryadOrderByNode(keySelectExpr, comparerExpr, true, queryExpr, child);
}
groupByOpName = "OrderedGroupBy";
}
// Add a GroupByNode if it is partitioned or has elementSelector.
// If the input was already correctly partitioned, this will be the only groupByNode.
if (isPartitioned)
{
groupByNode = new DryadGroupByNode(groupByOpName,
keySelectExpr,
elemSelectExpr,
resultSelectExpr,
null, // seed
null, // accumulate
null, // recursiveAccumulate
comparerExpr,
false, // isPartial
queryExpr,
groupByNode);
}
else if (elemSelectExpr != null)
{
// Local GroupBy without resultSelector:
groupByNode = new DryadGroupByNode(groupByOpName,
keySelectExpr,
elemSelectExpr,
null, // resultSelect
null, // seed
null, // accumulate
null, // recursiveAccumulate
comparerExpr,
StaticConfig.GroupByLocalAggregationIsPartial, // isPartial
queryExpr,
groupByNode);
// keySelectExpr1: g => g.Key
ParameterExpression groupParam = Expression.Parameter(groupByNode.OutputTypes[0], "g");
PropertyInfo keyInfo = groupParam.Type.GetProperty("Key");
Expression body = Expression.Property(groupParam, keyInfo);
Type delegateType = typeof(Func<,>).MakeGenericType(groupParam.Type, body.Type);
keySelectExpr1 = Expression.Lambda(delegateType, body, groupParam);
// No elementSelector
elemSelectExpr1 = null;
// resultSelectExpr1
ParameterExpression keyParam;
Type groupType = typeof(IEnumerable<>).MakeGenericType(groupByNode.OutputTypes[0]);
groupParam = Expression.Parameter(groupType, HpcLinqCodeGen.MakeUniqueName("g"));
if (resultSelectExpr == null)
{
// resultSelectExpr1: (k, g) => MakeDryadLinqGroup(k, g)
keyParam = Expression.Parameter(keySelectExpr1.Body.Type, HpcLinqCodeGen.MakeUniqueName("k"));
MethodInfo groupingInfo = typeof(HpcLinqEnumerable).GetMethod("MakeHpcLinqGroup");
groupingInfo = groupingInfo.MakeGenericMethod(keyParam.Type, elemType);
body = Expression.Call(groupingInfo, keyParam, groupParam);
}
else
{
// resultSelectExpr1: (k, g) => resultSelectExpr(k, FlattenGroups(g))
keyParam = resultSelectExpr.Parameters[0];
MethodInfo flattenInfo = typeof(HpcLinqEnumerable).GetMethod("FlattenGroups");
flattenInfo = flattenInfo.MakeGenericMethod(keyParam.Type, elemType);
Expression groupExpr = Expression.Call(flattenInfo, groupParam);
ParameterSubst subst = new ParameterSubst(resultSelectExpr.Parameters[1], groupExpr);
body = subst.Visit(resultSelectExpr.Body);
}
delegateType = typeof(Func<,,>).MakeGenericType(keyParam.Type, groupParam.Type, body.Type);
resultSelectExpr1 = Expression.Lambda(delegateType, body, keyParam, groupParam);
}
}
// At this point, the first GroupByNode has been created.
DryadMergeNode mergeNode = null;
DryadQueryNode groupByNode1 = groupByNode;
if (!isPartitioned)
{
// Create HashPartitionNode, MergeNode, and second GroupByNode
// Note, if we are doing decomposable-GroupByReduce, there is still some work to go after this
// - attach the combiner to the first merge-node
// - attach the combiner to the merge-node
// - attach finalizer to second GroupBy
int parCount = (groupByNode.IsDynamic) ? StaticConfig.DefaultPartitionCount : groupByNode.OutputPartition.Count;
bool isDynamic = (StaticConfig.DynamicOptLevel & StaticConfig.DynamicHashPartitionLevel) != 0;
DryadQueryNode hdistNode = new DryadHashPartitionNode(keySelectExpr1, comparerExpr, parCount,
isDynamic, queryExpr, groupByNode);
// Create the Merge Node
if (groupByOpName == "OrderedGroupBy")
{
// Mergesort with the same keySelector of the hash partition
mergeNode = new DryadMergeNode(keySelectExpr1, comparerExpr, true, false, queryExpr, hdistNode);
}
else
{
// Random merge
mergeNode = new DryadMergeNode(false, false, queryExpr, hdistNode);
}
groupByNode1 = new DryadGroupByNode(groupByOpName, keySelectExpr1, elemSelectExpr1,
resultSelectExpr1, seedExpr1, accumulateExpr1,
accumulateExpr1, comparerExpr, false, queryExpr,
mergeNode);
}
// Final tidy-up for decomposable GroupBy-Reduce pattern.
// - attach combiner to first GroupByNode
// - attache combiner to MergeNode as an aggregator
// - build a SelectNode to project out results and call finalizer on them.
if (dInfoList != null)
{
// Add dynamic aggregator to the merge-node, if applicable
if (StaticConfig.GroupByDynamicReduce && !isPartitioned)
{
mergeNode.AddAggregateNode(groupByNode1);
}
// Add the final Select node
Type keyResultPairType = typeof(Pair<,>).MakeGenericType(keyType, seedExpr1.Body.Type);
ParameterExpression keyResultPairParam = Expression.Parameter(keyResultPairType,
HpcLinqCodeGen.MakeUniqueName("e"));
PropertyInfo valuePropInfo_1 = keyResultPairType.GetProperty("Value");
Expression combinedValueExpr = Expression.Property(keyResultPairParam, valuePropInfo_1);
// First, build the combinerList
int dcnt = dInfoList.Count;
Expression[] combinerList = new Expression[dcnt];
for (int i = 0; i < dcnt; i++)
{
if (i + 1 == dcnt)
{
combinerList[i] = combinedValueExpr;
}
else
{
PropertyInfo keyPropInfo = combinedValueExpr.Type.GetProperty("Key");
combinerList[i] = Expression.Property(combinedValueExpr, keyPropInfo);
PropertyInfo valuePropInfo = combinedValueExpr.Type.GetProperty("Value");
combinedValueExpr = Expression.Property(combinedValueExpr, valuePropInfo);
}
LambdaExpression finalizerExpr = dInfoList[i].FinalReducer;
if (finalizerExpr != null)
{
ParameterSubst subst = new ParameterSubst(finalizerExpr.Parameters[0], combinerList[i]);
combinerList[i] = subst.Visit(finalizerExpr.Body);
}
}
// Build the funcList
Expression[] funcList = new Expression[dcnt];
for (int i = 0; i < dcnt; i++)
{
funcList[i] = dInfoList[i].Func;
}
// Apply the substitutions
CombinerSubst combinerSubst = new CombinerSubst(resultSelectExpr, keyResultPairParam, funcList, combinerList);
Expression finalizerSelectBody = combinerSubst.Visit();
// Finally, the Select node
Type delegateType = typeof(Func<,>).MakeGenericType(keyResultPairType, finalizerSelectBody.Type);
LambdaExpression selectExpr = Expression.Lambda(delegateType, finalizerSelectBody, keyResultPairParam);
groupByNode1 = new DryadSelectNode(QueryNodeType.Select, selectExpr, null, queryExpr, groupByNode1);
}
return groupByNode1;
}
private DryadQueryNode VisitZip(QueryNodeInfo first,
QueryNodeInfo second,
LambdaExpression resultSelector,
MethodCallExpression queryExpr)
{
DryadQueryNode child1 = this.Visit(first);
DryadQueryNode child2 = this.Visit(second);
if (child1.IsDynamic || child2.IsDynamic)
{
// Well, let us for now do it on a single machine
child1 = new DryadMergeNode(true, false, queryExpr, child1);
child2 = new DryadMergeNode(true, false, queryExpr, child2);
// Apply node for (x, y) => Zip(x, y, resultSelector)
Type paramType1 = typeof(IEnumerable<>).MakeGenericType(child1.OutputTypes[0]);
ParameterExpression param1 = Expression.Parameter(paramType1, "s1");
Type paramType2 = typeof(IEnumerable<>).MakeGenericType(child2.OutputTypes[0]);
ParameterExpression param2 = Expression.Parameter(paramType2, "s2");
MethodInfo minfo = typeof(HpcLinqHelper).GetMethod("Zip");
minfo = minfo.MakeGenericMethod(child1.OutputTypes[0]);
Expression body = Expression.Call(minfo, param1, param2, resultSelector);
Type funcType = typeof(Func<,>).MakeGenericType(param1.Type, param2.Type, body.Type);
LambdaExpression procFunc = Expression.Lambda(funcType, body, param1, param2);
return new DryadApplyNode(procFunc, queryExpr, child1, child2);
}
else
{
int parCount1 = child1.OutputPartition.Count;
int parCount2 = child2.OutputPartition.Count;
// Count nodes
DryadQueryNode countNode1 = new DryadBasicAggregateNode(null, AggregateOpType.LongCount,
true, false, queryExpr, child1);
DryadQueryNode countNode2 = new DryadBasicAggregateNode(null, AggregateOpType.LongCount,
true, false, queryExpr, child2);
countNode1 = new DryadMergeNode(true, false, queryExpr, countNode1);
countNode2 = new DryadMergeNode(true, false, queryExpr, countNode2);
// Apply node for (x, y) => ZipCount(x, y)
Type paramType1 = typeof(IEnumerable<>).MakeGenericType(typeof(long));
ParameterExpression param1 = Expression.Parameter(paramType1, "x");
ParameterExpression param2 = Expression.Parameter(paramType1, "y");
MethodInfo minfo = typeof(HpcLinqHelper).GetMethod("ZipCount");
Expression body = Expression.Call(minfo, param1, param2);
Type funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
LambdaExpression zipCount = Expression.Lambda(funcType, body, param1, param2);
DryadQueryNode indexedCountNode = new DryadApplyNode(zipCount, queryExpr, countNode1, countNode2);
// HashPartition(x => x.index, parCount2)
ParameterExpression param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
Expression keySelectBody = Expression.Property(param, "Index");
funcType = typeof(Func<,>).MakeGenericType(param.Type, keySelectBody.Type);
LambdaExpression keySelectExpr = Expression.Lambda(funcType, keySelectBody, param);
DryadQueryNode distCountNode = new DryadHashPartitionNode(keySelectExpr,
null,
parCount2,
queryExpr,
indexedCountNode);
// Apply node for (x, y) => AssignPartitionIndex(x, y)
param1 = Expression.Parameter(body.Type, "x");
Type paramType2 = typeof(IEnumerable<>).MakeGenericType(child2.OutputTypes[0]);
param2 = Expression.Parameter(paramType2, "y");
minfo = typeof(HpcLinqHelper).GetMethod("AssignPartitionIndex");
minfo = minfo.MakeGenericMethod(child2.OutputTypes[0]);
body = Expression.Call(minfo, param1, param2);
funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
LambdaExpression assignIndex = Expression.Lambda(funcType, body, param1, param2);
DryadQueryNode addIndexNode = new DryadApplyNode(assignIndex, queryExpr, distCountNode, child2);
// HashPartition(x => x.index, x => x.value, parCount1)
param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
body = Expression.Property(param, "Index");
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
keySelectExpr = Expression.Lambda(funcType, body, param);
body = Expression.Property(param, "Value");
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
LambdaExpression resultSelectExpr = Expression.Lambda(funcType, body, param);
DryadQueryNode newChild2 = new DryadHashPartitionNode(keySelectExpr,
resultSelectExpr,
null,
parCount1,
false,
queryExpr,
addIndexNode);
newChild2 = new DryadMergeNode(true, true, queryExpr, newChild2);
// Finally the zip node
return new DryadZipNode(resultSelector, queryExpr, child1, newChild2);
}
}
private DryadQueryNode VisitHashPartition(QueryNodeInfo source,
LambdaExpression keySelectExpr,
Expression comparerExpr,
Expression countExpr,
Expression queryExpr)
{
DryadQueryNode child = this.Visit(source);
Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
queryExpr);
}
bool isDynamic = (StaticConfig.DynamicOptLevel & StaticConfig.DynamicHashPartitionLevel) != 0;
int nOutputPartitions;
if (countExpr != null)
{
ExpressionSimplifier<int> evaluator = new ExpressionSimplifier<int>();
nOutputPartitions = evaluator.Eval(countExpr);
isDynamic = false;
}
else
{
// Note: For MayRTM, isDynamic will never be true.
nOutputPartitions = (isDynamic) ? 1 : child.OutputPartition.Count;
}
DryadQueryNode resNode = new DryadHashPartitionNode(
keySelectExpr, null, comparerExpr, nOutputPartitions,
isDynamic, queryExpr, child);
resNode = new DryadMergeNode(false, true, queryExpr, resNode);
return resNode;
}
//@@TODO: document what the 'NodeInfo' and 'ReferencedQuery' system does
private QueryNodeInfo BuildNodeInfoGraph(Expression expression)
{
QueryNodeInfo resNodeInfo = null;
if (this.m_exprNodeInfoMap.TryGetValue(expression, out resNodeInfo))
{
return resNodeInfo;
}
MethodCallExpression mcExpr = expression as MethodCallExpression;
if (mcExpr != null && mcExpr.Method.IsStatic && TypeSystem.IsQueryOperatorCall(mcExpr))
{
switch (mcExpr.Method.Name)
{
case "Join":
case "GroupJoin":
case "Union":
case "Intersect":
case "Except":
case "Zip":
case "SequenceEqual":
case "SequenceEqualAsQuery":
{
QueryNodeInfo child1 = BuildNodeInfoGraph(mcExpr.Arguments[0]);
QueryNodeInfo child2 = BuildNodeInfoGraph(mcExpr.Arguments[1]);
resNodeInfo = new QueryNodeInfo(mcExpr, true, child1, child2);
this.BuildReferencedQuery(2, mcExpr.Arguments);
break;
}
case "Concat":
{
NewArrayExpression others = mcExpr.Arguments[1] as NewArrayExpression;
if (others == null)
{
QueryNodeInfo child1 = BuildNodeInfoGraph(mcExpr.Arguments[0]);
QueryNodeInfo child2 = BuildNodeInfoGraph(mcExpr.Arguments[1]);
resNodeInfo = new QueryNodeInfo(mcExpr, true, child1, child2);
}
else
{
QueryNodeInfo[] infos = new QueryNodeInfo[others.Expressions.Count + 1];
infos[0] = BuildNodeInfoGraph(mcExpr.Arguments[0]);
for (int i = 0; i < others.Expressions.Count; ++i)
{
infos[i + 1] = BuildNodeInfoGraph(others.Expressions[i]);
}
resNodeInfo = new QueryNodeInfo(mcExpr, true, infos);
}
break;
}
case "Apply":
case "ApplyPerPartition":
{
QueryNodeInfo child1 = BuildNodeInfoGraph(mcExpr.Arguments[0]);
if (mcExpr.Arguments.Count == 2)
{
resNodeInfo = new QueryNodeInfo(mcExpr, true, child1);
this.BuildReferencedQuery(mcExpr.Arguments[1]);
}
else
{
LambdaExpression lambda = HpcLinqExpression.GetLambda(mcExpr.Arguments[2]);
if (lambda.Parameters.Count == 2)
{
// Apply with two sources
QueryNodeInfo child2 = BuildNodeInfoGraph(mcExpr.Arguments[1]);
resNodeInfo = new QueryNodeInfo(mcExpr, true, child1, child2);
this.BuildReferencedQuery(mcExpr.Arguments[2]);
}
else
{
// Apply with multiple sources of the same type
NewArrayExpression others = (NewArrayExpression)mcExpr.Arguments[1];
QueryNodeInfo[] infos = new QueryNodeInfo[others.Expressions.Count + 1];
infos[0] = child1;
for (int i = 0; i < others.Expressions.Count; ++i)
{
infos[i + 1] = BuildNodeInfoGraph(others.Expressions[i]);
}
resNodeInfo = new QueryNodeInfo(mcExpr, true, infos);
this.BuildReferencedQuery(mcExpr.Arguments[2]);
}
}
break;
}
case "RangePartition":
{
QueryNodeInfo child1 = BuildNodeInfoGraph(mcExpr.Arguments[0]);
if (mcExpr.Arguments.Count == 6)
{
// This is a key part of handling for RangePartition( ... , IQueryable<> keysQuery, ...)
// The keys expression is established as a child[1] nodeInfo for the rangePartition node.
QueryNodeInfo child2 = BuildNodeInfoGraph(mcExpr.Arguments[2]);
resNodeInfo = new QueryNodeInfo(mcExpr, true, child1, child2);
}
else
{
resNodeInfo = new QueryNodeInfo(mcExpr, true, child1);
}
this.BuildReferencedQuery(mcExpr.Arguments[1]);
break;
}
default:
{
QueryNodeInfo child1 = BuildNodeInfoGraph(mcExpr.Arguments[0]);
resNodeInfo = new QueryNodeInfo(mcExpr, true, child1);
this.BuildReferencedQuery(1, mcExpr.Arguments);
break;
}
}
}
if (resNodeInfo == null)
{
resNodeInfo = new QueryNodeInfo(expression, false);
}
this.m_exprNodeInfoMap.Add(expression, resNodeInfo);
return resNodeInfo;
}
private DryadQueryNode VisitJoin(QueryNodeInfo outerSource,
QueryNodeInfo innerSource,
QueryNodeType nodeType,
LambdaExpression outerKeySelector,
LambdaExpression innerKeySelector,
LambdaExpression resultSelector,
Expression comparerExpr,
Expression queryExpr)
{
DryadQueryNode outerChild = this.Visit(outerSource);
DryadQueryNode innerChild = this.Visit(innerSource);
DryadQueryNode joinNode = null;
Type keyType = outerKeySelector.Type.GetGenericArguments()[1];
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
queryExpr);
}
// The comparer object:
object comparer = null;
if (comparerExpr != null)
{
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
comparer = evaluator.Eval(comparerExpr);
}
if (outerChild.IsDynamic || innerChild.IsDynamic)
{
// Well, let us do the simplest thing for now
outerChild = new DryadHashPartitionNode(outerKeySelector,
comparerExpr,
StaticConfig.DefaultPartitionCount,
queryExpr,
outerChild);
if (IsMergeNodeNeeded(outerChild))
{
outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
}
innerChild = new DryadHashPartitionNode(innerKeySelector,
comparerExpr,
StaticConfig.DefaultPartitionCount,
queryExpr,
innerChild);
if (IsMergeNodeNeeded(innerChild))
{
innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
}
joinNode = new DryadJoinNode(nodeType,
"Hash" + nodeType,
outerKeySelector,
innerKeySelector,
resultSelector,
comparerExpr,
queryExpr,
outerChild,
innerChild);
return joinNode;
}
bool isOuterDescending = outerChild.OutputDataSetInfo.orderByInfo.IsDescending;
bool isInnerDescending = innerChild.OutputDataSetInfo.orderByInfo.IsDescending;
// Partition outer and inner if needed
if (outerChild.OutputPartition.ParType == PartitionType.Range &&
outerChild.OutputPartition.HasKeys &&
outerChild.OutputPartition.IsPartitionedBy(outerKeySelector, comparer))
{
if (innerChild.OutputPartition.ParType != PartitionType.Range ||
!innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer) ||
!outerChild.OutputPartition.IsSamePartition(innerChild.OutputPartition))
{
// Range distribute inner using outer's partition.
innerChild = outerChild.OutputPartition.CreatePartitionNode(innerKeySelector, innerChild);
if (IsMergeNodeNeeded(innerChild))
{
innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
}
}
}
else if (innerChild.OutputPartition.ParType == PartitionType.Range &&
innerChild.OutputPartition.HasKeys &&
innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer))
{
// Range distribute outer using inner's partition.
outerChild = innerChild.OutputPartition.CreatePartitionNode(outerKeySelector, outerChild);
if (IsMergeNodeNeeded(outerChild))
{
outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
}
}
else if (outerChild.OutputPartition.ParType == PartitionType.Hash &&
outerChild.OutputPartition.IsPartitionedBy(outerKeySelector, comparer))
{
if (innerChild.OutputPartition.ParType != PartitionType.Hash ||
!innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer) ||
!outerChild.OutputPartition.IsSamePartition(innerChild.OutputPartition))
{
innerChild = new DryadHashPartitionNode(innerKeySelector,
comparerExpr,
outerChild.OutputPartition.Count,
queryExpr,
innerChild);
if (IsMergeNodeNeeded(innerChild))
{
innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
}
}
}
else if (innerChild.OutputPartition.ParType == PartitionType.Hash &&
innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer))
{
outerChild = new DryadHashPartitionNode(outerKeySelector,
comparerExpr,
innerChild.OutputPartition.Count,
queryExpr,
outerChild);
if (IsMergeNodeNeeded(outerChild))
{
outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
}
}
else
{
// No luck. Hash partition both outer and inner
int parCnt = Math.Max(outerChild.OutputPartition.Count, innerChild.OutputPartition.Count);
if (parCnt > 1)
{
outerChild = new DryadHashPartitionNode(outerKeySelector,
comparerExpr,
parCnt,
queryExpr,
outerChild);
if (IsMergeNodeNeeded(outerChild))
{
outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
}
innerChild = new DryadHashPartitionNode(innerKeySelector,
comparerExpr,
parCnt,
queryExpr,
innerChild);
if (IsMergeNodeNeeded(innerChild))
{
innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
}
}
}
// Perform either merge or hash join
string opName = "Hash";
if (outerChild.IsOrderedBy(outerKeySelector, comparer))
{
if (!innerChild.IsOrderedBy(innerKeySelector, comparer) ||
isOuterDescending != isInnerDescending)
{
// Sort inner if unsorted
innerChild = new DryadOrderByNode(innerKeySelector, comparerExpr,
isOuterDescending, queryExpr, innerChild);
}
opName = "Merge";
}
else if (innerChild.IsOrderedBy(innerKeySelector, comparer))
{
if (!outerChild.IsOrderedBy(outerKeySelector, comparer) ||
isOuterDescending != isInnerDescending)
{
// Sort outer if unsorted
outerChild = new DryadOrderByNode(outerKeySelector, comparerExpr,
isInnerDescending, queryExpr, outerChild);
}
opName = "Merge";
}
joinNode = new DryadJoinNode(nodeType,
opName + nodeType,
outerKeySelector,
innerKeySelector,
resultSelector,
comparerExpr,
queryExpr,
outerChild,
innerChild);
return joinNode;
}
