private DryadQueryNode VisitSetOperation(QueryNodeInfo source1,
QueryNodeInfo source2,
QueryNodeType nodeType,
Expression comparerExpr,
Expression queryExpr)
{
DryadQueryNode child1 = this.Visit(source1);
DryadQueryNode child2 = this.Visit(source2);
DryadQueryNode resNode = null;
Type keyType = child1.OutputTypes[0];
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
queryExpr);
}
// The comparer object:
object comparer = null;
if (comparerExpr != null)
{
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
comparer = evaluator.Eval(comparerExpr);
}
LambdaExpression keySelectExpr = IdentityFunction.Instance(keyType);
if (child1.IsDynamic || child2.IsDynamic)
{
// Well, let us do the simplest thing for now
child1 = new DryadHashPartitionNode(keySelectExpr,
null,
StaticConfig.DefaultPartitionCount,
queryExpr,
child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
child2 = new DryadHashPartitionNode(keySelectExpr,
null,
StaticConfig.DefaultPartitionCount,
queryExpr,
child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
resNode = new DryadSetOperationNode(nodeType, nodeType.ToString(), comparerExpr,
queryExpr, child1, child2);
return resNode;
}
bool isDescending1 = child1.OutputDataSetInfo.orderByInfo.IsDescending;
bool isDescending2 = child2.OutputDataSetInfo.orderByInfo.IsDescending;
// Partition child1 and child2 if needed
if (child1.OutputPartition.ParType == PartitionType.Range &&
child1.OutputPartition.HasKeys &&
child1.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
if (child2.OutputPartition.ParType != PartitionType.Range ||
!child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer) ||
child1.OutputPartition.IsSamePartition(child2.OutputPartition))
{
// Range distribute child2 using child1's partition
child2 = child1.OutputPartition.CreatePartitionNode(keySelectExpr, child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
}
}
else if (child2.OutputPartition.ParType == PartitionType.Range &&
child2.OutputPartition.HasKeys &&
child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
// Range distribute child1 using child2's partition
child1 = child2.OutputPartition.CreatePartitionNode(keySelectExpr, child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
}
else if (child1.OutputPartition.ParType == PartitionType.Hash &&
child1.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
if (child2.OutputPartition.ParType != PartitionType.Hash ||
!child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer) ||
!child1.OutputPartition.IsSamePartition(child2.OutputPartition))
{
// Hash distribute child2:
child2 = new DryadHashPartitionNode(keySelectExpr,
comparerExpr,
child1.OutputPartition.Count,
queryExpr,
child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
}
}
else if (child2.OutputPartition.ParType == PartitionType.Hash &&
child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
child1 = new DryadHashPartitionNode(keySelectExpr,
comparerExpr,
child2.OutputPartition.Count,
queryExpr,
child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
}
else
{
// No luck. Hash distribute both child1 and child2, then perform hash operation
int parCnt = Math.Max(child1.OutputPartition.Count, child2.OutputPartition.Count);
if (parCnt > 1)
{
child1 = new DryadHashPartitionNode(keySelectExpr, comparerExpr, parCnt, queryExpr, child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
child2 = new DryadHashPartitionNode(keySelectExpr, comparerExpr, parCnt, queryExpr, child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
}
}
// Perform either hash or ordered operation
string opName = "";
if (child1.IsOrderedBy(keySelectExpr, comparer))
{
if (!child1.IsOrderedBy(keySelectExpr, comparer) ||
isDescending1 != isDescending2)
{
// Sort inner if unsorted
child2 = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending1, queryExpr, child2);
}
opName = "Ordered";
}
else if (child2.IsOrderedBy(keySelectExpr, comparer))
{
if (!child1.IsOrderedBy(keySelectExpr, comparer) ||
isDescending1 != isDescending2)
{
// Sort outer if unsorted
child1 = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending2, queryExpr, child1);
}
opName = "Ordered";
}
resNode = new DryadSetOperationNode(nodeType, opName + nodeType, comparerExpr, queryExpr, child1, child2);
return resNode;
}
private DryadQueryNode VisitPartitionOp(string opName,
QueryNodeInfo source,
QueryNodeType nodeType,
Expression controlExpr,
MethodCallExpression queryExpr)
{
DryadQueryNode resNode;
if (nodeType == QueryNodeType.TakeWhile &&
controlExpr.Type.GetGenericArguments().Length != 2)
{
// The "indexed" version.
resNode = this.Visit(source);
// The following block used to be skipped for resNode.OutputPartition.Count == 1,
// which causes compilation error (bug 13593)
// @@TODO[p3] : implement a working optimization for nPartition==1 that calls
// directly to Linq TakeWhile.
// Note: the test is: if (resNode.IsDynamic || resNode.OutputPartition.Count > 1)
{
resNode.IsForked = true;
bool isLong = (queryExpr.Method.Name == "LongTakeWhile");
DryadQueryNode offsetNode = this.CreateOffset(isLong, queryExpr, resNode);
// Create (x, y) => GroupIndexedTakeWhile(x, y, controlExpr)
Type ptype1 = typeof(IEnumerable<>).MakeGenericType(resNode.OutputTypes[0]);
Type ptype2 = typeof(IEnumerable<>).MakeGenericType(typeof(IndexedValue<long>));
ParameterExpression param1 = Expression.Parameter(ptype1, HpcLinqCodeGen.MakeUniqueName("x"));
ParameterExpression param2 = Expression.Parameter(ptype2, HpcLinqCodeGen.MakeUniqueName("y"));
string methodName = "GroupIndexed" + queryExpr.Method.Name;
MethodInfo minfo = typeof(HpcLinqEnumerable).GetMethod(methodName);
minfo = minfo.MakeGenericMethod(resNode.OutputTypes[0]);
Expression body = Expression.Call(minfo, param1, param2, controlExpr);
Type type = typeof(Func<,,>).MakeGenericType(ptype1, ptype2, body.Type);
LambdaExpression procFunc = Expression.Lambda(type, body, param1, param2);
resNode = new DryadApplyNode(procFunc, queryExpr, resNode, offsetNode);
}
}
else if (!source.IsForked &&
(nodeType == QueryNodeType.Take || nodeType == QueryNodeType.TakeWhile) &&
(source.OperatorName == "OrderBy" || source.OperatorName == "OrderByDescending"))
{
resNode = this.Visit(source.children[0].child);
bool isDescending = (source.OperatorName == "OrderByDescending");
MethodCallExpression sourceQueryExpr = (MethodCallExpression)source.queryExpression;
LambdaExpression keySelectExpr = HpcLinqExpression.GetLambda(sourceQueryExpr.Arguments[1]);
Expression comparerExpr = null;
if (sourceQueryExpr.Arguments.Count == 3)
{
comparerExpr = sourceQueryExpr.Arguments[2];
}
resNode = this.PromoteConcat(
source.children[0].child,
resNode,
delegate(DryadQueryNode x) {
DryadQueryNode y = new DryadOrderByNode(keySelectExpr, comparerExpr,
isDescending, sourceQueryExpr, x);
return FirstStagePartitionOp(opName, nodeType, controlExpr, queryExpr, y);
});
if (resNode.IsDynamic || resNode.OutputPartition.Count > 1)
{
// Need a mergesort
resNode = new DryadMergeNode(keySelectExpr, comparerExpr, isDescending, false, sourceQueryExpr, resNode);
}
}
else
{
resNode = this.Visit(source);
if (nodeType == QueryNodeType.Take || nodeType == QueryNodeType.TakeWhile)
{
resNode = this.PromoteConcat(
source, resNode,
x => FirstStagePartitionOp(opName, nodeType, controlExpr, queryExpr, x));
}
}
resNode = new DryadPartitionOpNode(opName, nodeType, controlExpr, false, queryExpr, resNode);
return resNode;
}
private DryadQueryNode VisitOrderBy(QueryNodeInfo source,
LambdaExpression keySelectExpr,
Expression comparerExpr,
bool isDescending,
Expression queryExpr)
{
DryadQueryNode child = this.Visit(source);
Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
if (comparerExpr == null && !TypeSystem.HasDefaultComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable, keyType),
queryExpr);
}
DryadQueryNode resNode = child;
if (child.OutputPartition.ParType == PartitionType.Range &&
child.OutputPartition.IsPartitionedBy(keySelectExpr, comparerExpr, isDescending))
{
// Only need to sort each partition
resNode = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending, queryExpr, child);
}
else
{
Expression isDescendingExpr = Expression.Constant(isDescending);
bool dynamicOptEnabled = (StaticConfig.DynamicOptLevel & StaticConfig.DynamicRangePartitionLevel) != 0;
if (dynamicOptEnabled || child.IsDynamic)
{
// NOTE: for MayRTM, this path should not be taken.
resNode = this.CreateRangePartition(true, keySelectExpr, null, comparerExpr,
isDescendingExpr, queryExpr, null, child);
resNode = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending, queryExpr, resNode);
}
else
{
if (child.OutputPartition.Count > 1)
{
resNode = this.CreateRangePartition(false, keySelectExpr, null, comparerExpr,
isDescendingExpr, queryExpr, null, child);
}
resNode = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending, queryExpr, resNode);
}
}
return resNode;
}
private DryadQueryNode VisitJoin(QueryNodeInfo outerSource,
QueryNodeInfo innerSource,
QueryNodeType nodeType,
LambdaExpression outerKeySelector,
LambdaExpression innerKeySelector,
LambdaExpression resultSelector,
Expression comparerExpr,
Expression queryExpr)
{
DryadQueryNode outerChild = this.Visit(outerSource);
DryadQueryNode innerChild = this.Visit(innerSource);
DryadQueryNode joinNode = null;
Type keyType = outerKeySelector.Type.GetGenericArguments()[1];
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
queryExpr);
}
// The comparer object:
object comparer = null;
if (comparerExpr != null)
{
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
comparer = evaluator.Eval(comparerExpr);
}
if (outerChild.IsDynamic || innerChild.IsDynamic)
{
// Well, let us do the simplest thing for now
outerChild = new DryadHashPartitionNode(outerKeySelector,
comparerExpr,
StaticConfig.DefaultPartitionCount,
queryExpr,
outerChild);
if (IsMergeNodeNeeded(outerChild))
{
outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
}
innerChild = new DryadHashPartitionNode(innerKeySelector,
comparerExpr,
StaticConfig.DefaultPartitionCount,
queryExpr,
innerChild);
if (IsMergeNodeNeeded(innerChild))
{
innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
}
joinNode = new DryadJoinNode(nodeType,
"Hash" + nodeType,
outerKeySelector,
innerKeySelector,
resultSelector,
comparerExpr,
queryExpr,
outerChild,
innerChild);
return joinNode;
}
bool isOuterDescending = outerChild.OutputDataSetInfo.orderByInfo.IsDescending;
bool isInnerDescending = innerChild.OutputDataSetInfo.orderByInfo.IsDescending;
// Partition outer and inner if needed
if (outerChild.OutputPartition.ParType == PartitionType.Range &&
outerChild.OutputPartition.HasKeys &&
outerChild.OutputPartition.IsPartitionedBy(outerKeySelector, comparer))
{
if (innerChild.OutputPartition.ParType != PartitionType.Range ||
!innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer) ||
!outerChild.OutputPartition.IsSamePartition(innerChild.OutputPartition))
{
// Range distribute inner using outer's partition.
innerChild = outerChild.OutputPartition.CreatePartitionNode(innerKeySelector, innerChild);
if (IsMergeNodeNeeded(innerChild))
{
innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
}
}
}
else if (innerChild.OutputPartition.ParType == PartitionType.Range &&
innerChild.OutputPartition.HasKeys &&
innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer))
{
// Range distribute outer using inner's partition.
outerChild = innerChild.OutputPartition.CreatePartitionNode(outerKeySelector, outerChild);
if (IsMergeNodeNeeded(outerChild))
{
outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
}
}
else if (outerChild.OutputPartition.ParType == PartitionType.Hash &&
outerChild.OutputPartition.IsPartitionedBy(outerKeySelector, comparer))
{
if (innerChild.OutputPartition.ParType != PartitionType.Hash ||
!innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer) ||
!outerChild.OutputPartition.IsSamePartition(innerChild.OutputPartition))
{
innerChild = new DryadHashPartitionNode(innerKeySelector,
comparerExpr,
outerChild.OutputPartition.Count,
queryExpr,
innerChild);
if (IsMergeNodeNeeded(innerChild))
{
innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
}
}
}
else if (innerChild.OutputPartition.ParType == PartitionType.Hash &&
innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer))
{
outerChild = new DryadHashPartitionNode(outerKeySelector,
comparerExpr,
innerChild.OutputPartition.Count,
queryExpr,
outerChild);
if (IsMergeNodeNeeded(outerChild))
{
outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
}
}
else
{
// No luck. Hash partition both outer and inner
int parCnt = Math.Max(outerChild.OutputPartition.Count, innerChild.OutputPartition.Count);
if (parCnt > 1)
{
outerChild = new DryadHashPartitionNode(outerKeySelector,
comparerExpr,
parCnt,
queryExpr,
outerChild);
if (IsMergeNodeNeeded(outerChild))
{
outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
}
innerChild = new DryadHashPartitionNode(innerKeySelector,
comparerExpr,
parCnt,
queryExpr,
innerChild);
if (IsMergeNodeNeeded(innerChild))
{
innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
}
}
}
// Perform either merge or hash join
string opName = "Hash";
if (outerChild.IsOrderedBy(outerKeySelector, comparer))
{
if (!innerChild.IsOrderedBy(innerKeySelector, comparer) ||
isOuterDescending != isInnerDescending)
{
// Sort inner if unsorted
innerChild = new DryadOrderByNode(innerKeySelector, comparerExpr,
isOuterDescending, queryExpr, innerChild);
}
opName = "Merge";
}
else if (innerChild.IsOrderedBy(innerKeySelector, comparer))
{
if (!outerChild.IsOrderedBy(outerKeySelector, comparer) ||
isOuterDescending != isInnerDescending)
{
// Sort outer if unsorted
outerChild = new DryadOrderByNode(outerKeySelector, comparerExpr,
isInnerDescending, queryExpr, outerChild);
}
opName = "Merge";
}
joinNode = new DryadJoinNode(nodeType,
opName + nodeType,
outerKeySelector,
innerKeySelector,
resultSelector,
comparerExpr,
queryExpr,
outerChild,
innerChild);
return joinNode;
}
private DryadQueryNode VisitGroupBy(QueryNodeInfo source,
LambdaExpression keySelectExpr,
LambdaExpression elemSelectExpr,
LambdaExpression resultSelectExpr,
Expression comparerExpr,
Expression queryExpr)
{
DryadQueryNode child = this.Visit(source);
ExpressionInfo einfo = new ExpressionInfo(keySelectExpr);
if (einfo.IsExpensive)
{
// Any method call that is not tagged as "expensive=false" will be deemed expensive.
// if the keySelector is expensive, we rewrite the query so that the key-function is invoked only once
// and the record key passed around via a Pair<TKey,TRecord>.
// keyFunc becomes pair=>pair.Key
// elementSelector must be rewritten so that references to (record) become (pair.Value)
Type[] vkTypes = keySelectExpr.Type.GetGenericArguments();
Type pairType = typeof(Pair<,>).MakeGenericType(vkTypes[1], vkTypes[0]);
ParameterExpression pairParam = Expression.Parameter(pairType, "e");
// Add Select(x => new Pair<K,S>(key(x), x))
ParameterExpression valueParam = keySelectExpr.Parameters[0];
Expression body = Expression.New(pairType.GetConstructors()[0], keySelectExpr.Body, valueParam);
Type delegateType = typeof(Func<,>).MakeGenericType(valueParam.Type, body.Type);
LambdaExpression selectExpr = Expression.Lambda(delegateType, body, valueParam);
child = new DryadSelectNode(QueryNodeType.Select, selectExpr, null, queryExpr, child);
// Change keySelector to e => e.Key
PropertyInfo keyInfo = pairParam.Type.GetProperty("Key");
body = Expression.Property(pairParam, keyInfo);
delegateType = typeof(Func<,>).MakeGenericType(pairParam.Type, body.Type);
keySelectExpr = Expression.Lambda(delegateType, body, pairParam);
// Add or change elementSelector with e.Value
PropertyInfo valueInfo = pairParam.Type.GetProperty("Value");
body = Expression.Property(pairParam, valueInfo);
if (elemSelectExpr != null)
{
ParameterSubst subst = new ParameterSubst(elemSelectExpr.Parameters[0], body);
body = subst.Visit(elemSelectExpr.Body);
}
delegateType = typeof(Func<,>).MakeGenericType(pairParam.Type, body.Type);
elemSelectExpr = Expression.Lambda(delegateType, body, pairParam);
}
Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
queryExpr);
}
Type elemType;
if (elemSelectExpr == null)
{
elemType = keySelectExpr.Type.GetGenericArguments()[0];
}
else
{
elemType = elemSelectExpr.Type.GetGenericArguments()[1];
}
// The comparer object:
object comparer = null;
if (comparerExpr != null)
{
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
comparer = evaluator.Eval(comparerExpr);
}
LambdaExpression keySelectExpr1 = keySelectExpr;
LambdaExpression elemSelectExpr1 = elemSelectExpr;
LambdaExpression resultSelectExpr1 = resultSelectExpr;
LambdaExpression seedExpr1 = null;
LambdaExpression accumulateExpr1 = null;
List<DecompositionInfo> dInfoList = null;
if (resultSelectExpr != null)
{
dInfoList = Decomposition.GetDecompositionInfoList(resultSelectExpr, this.m_codeGen);
}
String groupByOpName = "GroupBy";
DryadQueryNode groupByNode = child;
bool isPartitioned = child.OutputPartition.IsPartitionedBy(keySelectExpr, comparer);
if (dInfoList != null)
{
// ** Decomposable GroupBy-Reduce
// This block creates the first GroupByNode and does some preparation for subsequent nodes.
if (child.IsOrderedBy(keySelectExpr, comparer))
{
groupByOpName = "OrderedGroupBy";
}
int dcnt = dInfoList.Count;
ParameterExpression keyParam;
if (resultSelectExpr.Parameters.Count == 1)
{
keyParam = Expression.Parameter(keyType, HpcLinqCodeGen.MakeUniqueName("k"));
}
else
{
keyParam = resultSelectExpr.Parameters[0];
}
// Seed:
ParameterExpression param2 = Expression.Parameter(
elemType, HpcLinqCodeGen.MakeUniqueName("e"));
Expression zeroExpr = Expression.Constant(0, typeof(int));
Expression seedBody = zeroExpr;
if (dcnt != 0)
{
LambdaExpression seed = dInfoList[dcnt-1].Seed;
ParameterSubst subst = new ParameterSubst(seed.Parameters[0], param2);
seedBody = subst.Visit(seed.Body);
for (int i = dcnt - 2; i >= 0; i--)
{
seed = dInfoList[i].Seed;
subst = new ParameterSubst(seed.Parameters[0], param2);
Expression firstExpr = subst.Visit(seed.Body);
Type newPairType = typeof(Pair<,>).MakeGenericType(firstExpr.Type, seedBody.Type);
seedBody = Expression.New(newPairType.GetConstructors()[0], firstExpr, seedBody);
}
}
LambdaExpression seedExpr = Expression.Lambda(seedBody, param2);
// Accumulate:
ParameterExpression param1 = Expression.Parameter(
seedBody.Type, HpcLinqCodeGen.MakeUniqueName("a"));
Expression accumulateBody = zeroExpr;
if (dcnt != 0)
{
accumulateBody = Decomposition.AccumulateList(param1, param2, dInfoList, 0);
}
LambdaExpression accumulateExpr = Expression.Lambda(accumulateBody, param1, param2);
// Now prepare for the merge-aggregator and/or in the secondary group-by.
// keySelectExpr1: e => e.Key
Type reducerResType = typeof(Pair<,>).MakeGenericType(keyParam.Type, accumulateBody.Type);
ParameterExpression reducerResParam = Expression.Parameter(reducerResType, "e");
PropertyInfo keyInfo = reducerResParam.Type.GetProperty("Key");
Expression body = Expression.Property(reducerResParam, keyInfo);
Type delegateType = typeof(Func<,>).MakeGenericType(reducerResParam.Type, body.Type);
keySelectExpr1 = Expression.Lambda(delegateType, body, reducerResParam);
// elemSelectExpr1: e => e.Value
PropertyInfo valueInfo = reducerResParam.Type.GetProperty("Value");
body = Expression.Property(reducerResParam, valueInfo);
delegateType = typeof(Func<,>).MakeGenericType(reducerResParam.Type, body.Type);
elemSelectExpr1 = Expression.Lambda(delegateType, body, reducerResParam);
// SeedExpr1
param2 = Expression.Parameter(elemSelectExpr1.Body.Type,
HpcLinqCodeGen.MakeUniqueName("e"));
seedExpr1 = Expression.Lambda(param2, param2);
// AccumulateExpr1
Expression recursiveAccumulateBody = zeroExpr;
if (dcnt != 0)
{
recursiveAccumulateBody = Decomposition.RecursiveAccumulateList(param1, param2, dInfoList, 0);
}
accumulateExpr1 = Expression.Lambda(recursiveAccumulateBody, param1, param2);
// resultSelectExpr1
resultSelectExpr1 = null;
// The first groupByNode.
// If the input was already correctly partitioned, this will be the only groupByNode.
bool isPartial = StaticConfig.GroupByLocalAggregationIsPartial && !isPartitioned;
groupByNode = new DryadGroupByNode(
groupByOpName, keySelectExpr, elemSelectExpr, null,
seedExpr, accumulateExpr, accumulateExpr1, comparerExpr,
isPartial, queryExpr, child);
}
else
{
// Can't do partial aggregation.
// Use sort, mergesort, and ordered groupby, if TKey implements IComparable.
if ((comparer != null && TypeSystem.IsComparer(comparer, keyType)) ||
(comparer == null && TypeSystem.HasDefaultComparer(keyType)))
{
if (!child.IsOrderedBy(keySelectExpr, comparer))
{
groupByNode = new DryadOrderByNode(keySelectExpr, comparerExpr, true, queryExpr, child);
}
groupByOpName = "OrderedGroupBy";
}
// Add a GroupByNode if it is partitioned or has elementSelector.
// If the input was already correctly partitioned, this will be the only groupByNode.
if (isPartitioned)
{
groupByNode = new DryadGroupByNode(groupByOpName,
keySelectExpr,
elemSelectExpr,
resultSelectExpr,
null, // seed
null, // accumulate
null, // recursiveAccumulate
comparerExpr,
false, // isPartial
queryExpr,
groupByNode);
}
else if (elemSelectExpr != null)
{
// Local GroupBy without resultSelector:
groupByNode = new DryadGroupByNode(groupByOpName,
keySelectExpr,
elemSelectExpr,
null, // resultSelect
null, // seed
null, // accumulate
null, // recursiveAccumulate
comparerExpr,
StaticConfig.GroupByLocalAggregationIsPartial, // isPartial
queryExpr,
groupByNode);
// keySelectExpr1: g => g.Key
ParameterExpression groupParam = Expression.Parameter(groupByNode.OutputTypes[0], "g");
PropertyInfo keyInfo = groupParam.Type.GetProperty("Key");
Expression body = Expression.Property(groupParam, keyInfo);
Type delegateType = typeof(Func<,>).MakeGenericType(groupParam.Type, body.Type);
keySelectExpr1 = Expression.Lambda(delegateType, body, groupParam);
// No elementSelector
elemSelectExpr1 = null;
// resultSelectExpr1
ParameterExpression keyParam;
Type groupType = typeof(IEnumerable<>).MakeGenericType(groupByNode.OutputTypes[0]);
groupParam = Expression.Parameter(groupType, HpcLinqCodeGen.MakeUniqueName("g"));
if (resultSelectExpr == null)
{
// resultSelectExpr1: (k, g) => MakeDryadLinqGroup(k, g)
keyParam = Expression.Parameter(keySelectExpr1.Body.Type, HpcLinqCodeGen.MakeUniqueName("k"));
MethodInfo groupingInfo = typeof(HpcLinqEnumerable).GetMethod("MakeHpcLinqGroup");
groupingInfo = groupingInfo.MakeGenericMethod(keyParam.Type, elemType);
body = Expression.Call(groupingInfo, keyParam, groupParam);
}
else
{
// resultSelectExpr1: (k, g) => resultSelectExpr(k, FlattenGroups(g))
keyParam = resultSelectExpr.Parameters[0];
MethodInfo flattenInfo = typeof(HpcLinqEnumerable).GetMethod("FlattenGroups");
flattenInfo = flattenInfo.MakeGenericMethod(keyParam.Type, elemType);
Expression groupExpr = Expression.Call(flattenInfo, groupParam);
ParameterSubst subst = new ParameterSubst(resultSelectExpr.Parameters[1], groupExpr);
body = subst.Visit(resultSelectExpr.Body);
}
delegateType = typeof(Func<,,>).MakeGenericType(keyParam.Type, groupParam.Type, body.Type);
resultSelectExpr1 = Expression.Lambda(delegateType, body, keyParam, groupParam);
}
}
// At this point, the first GroupByNode has been created.
DryadMergeNode mergeNode = null;
DryadQueryNode groupByNode1 = groupByNode;
if (!isPartitioned)
{
// Create HashPartitionNode, MergeNode, and second GroupByNode
// Note, if we are doing decomposable-GroupByReduce, there is still some work to go after this
// - attach the combiner to the first merge-node
// - attach the combiner to the merge-node
// - attach finalizer to second GroupBy
int parCount = (groupByNode.IsDynamic) ? StaticConfig.DefaultPartitionCount : groupByNode.OutputPartition.Count;
bool isDynamic = (StaticConfig.DynamicOptLevel & StaticConfig.DynamicHashPartitionLevel) != 0;
DryadQueryNode hdistNode = new DryadHashPartitionNode(keySelectExpr1, comparerExpr, parCount,
isDynamic, queryExpr, groupByNode);
// Create the Merge Node
if (groupByOpName == "OrderedGroupBy")
{
// Mergesort with the same keySelector of the hash partition
mergeNode = new DryadMergeNode(keySelectExpr1, comparerExpr, true, false, queryExpr, hdistNode);
}
else
{
// Random merge
mergeNode = new DryadMergeNode(false, false, queryExpr, hdistNode);
}
groupByNode1 = new DryadGroupByNode(groupByOpName, keySelectExpr1, elemSelectExpr1,
resultSelectExpr1, seedExpr1, accumulateExpr1,
accumulateExpr1, comparerExpr, false, queryExpr,
mergeNode);
}
// Final tidy-up for decomposable GroupBy-Reduce pattern.
// - attach combiner to first GroupByNode
// - attache combiner to MergeNode as an aggregator
// - build a SelectNode to project out results and call finalizer on them.
if (dInfoList != null)
{
// Add dynamic aggregator to the merge-node, if applicable
if (StaticConfig.GroupByDynamicReduce && !isPartitioned)
{
mergeNode.AddAggregateNode(groupByNode1);
}
// Add the final Select node
Type keyResultPairType = typeof(Pair<,>).MakeGenericType(keyType, seedExpr1.Body.Type);
ParameterExpression keyResultPairParam = Expression.Parameter(keyResultPairType,
HpcLinqCodeGen.MakeUniqueName("e"));
PropertyInfo valuePropInfo_1 = keyResultPairType.GetProperty("Value");
Expression combinedValueExpr = Expression.Property(keyResultPairParam, valuePropInfo_1);
// First, build the combinerList
int dcnt = dInfoList.Count;
Expression[] combinerList = new Expression[dcnt];
for (int i = 0; i < dcnt; i++)
{
if (i + 1 == dcnt)
{
combinerList[i] = combinedValueExpr;
}
else
{
PropertyInfo keyPropInfo = combinedValueExpr.Type.GetProperty("Key");
combinerList[i] = Expression.Property(combinedValueExpr, keyPropInfo);
PropertyInfo valuePropInfo = combinedValueExpr.Type.GetProperty("Value");
combinedValueExpr = Expression.Property(combinedValueExpr, valuePropInfo);
}
LambdaExpression finalizerExpr = dInfoList[i].FinalReducer;
if (finalizerExpr != null)
{
ParameterSubst subst = new ParameterSubst(finalizerExpr.Parameters[0], combinerList[i]);
combinerList[i] = subst.Visit(finalizerExpr.Body);
}
}
// Build the funcList
Expression[] funcList = new Expression[dcnt];
for (int i = 0; i < dcnt; i++)
{
funcList[i] = dInfoList[i].Func;
}
// Apply the substitutions
CombinerSubst combinerSubst = new CombinerSubst(resultSelectExpr, keyResultPairParam, funcList, combinerList);
Expression finalizerSelectBody = combinerSubst.Visit();
// Finally, the Select node
Type delegateType = typeof(Func<,>).MakeGenericType(keyResultPairType, finalizerSelectBody.Type);
LambdaExpression selectExpr = Expression.Lambda(delegateType, finalizerSelectBody, keyResultPairParam);
groupByNode1 = new DryadSelectNode(QueryNodeType.Select, selectExpr, null, queryExpr, groupByNode1);
}
return groupByNode1;
}
internal virtual string Visit(DryadOrderByNode node,
CodeMemberMethod vertexMethod,
string[] readerNames,
string[] writerNames)
{
return node.AddVertexCode(vertexMethod, readerNames, writerNames);
}
