// Precondition: idx < dInfoList.Count
internal static Expression RecursiveAccumulateList(Expression valueExpr1,
Expression valueExpr2,
List <DecompositionInfo> dInfoList,
int idx)
{
LambdaExpression recursiveAccumulateExpr = dInfoList[idx].RecursiveAccumulator;
if (dInfoList.Count == idx + 1)
{
ParameterSubst subst = new ParameterSubst(recursiveAccumulateExpr.Parameters[0], valueExpr1);
Expression resultExpr = subst.Visit(recursiveAccumulateExpr.Body);
subst = new ParameterSubst(recursiveAccumulateExpr.Parameters[1], valueExpr2);
return(subst.Visit(resultExpr));
}
else
{
PropertyInfo keyPropInfo1 = valueExpr1.Type.GetProperty("Key");
Expression keyValueExpr1 = Expression.Property(valueExpr1, keyPropInfo1);
PropertyInfo keyPropInfo2 = valueExpr2.Type.GetProperty("Key");
Expression keyValueExpr2 = Expression.Property(valueExpr2, keyPropInfo2);
ParameterSubst subst = new ParameterSubst(recursiveAccumulateExpr.Parameters[0], keyValueExpr1);
Expression expr1 = subst.Visit(recursiveAccumulateExpr.Body);
subst = new ParameterSubst(recursiveAccumulateExpr.Parameters[1], keyValueExpr2);
expr1 = subst.Visit(expr1);
PropertyInfo valuePropInfo1 = valueExpr1.Type.GetProperty("Value");
Expression valueValueExpr1 = Expression.Property(valueExpr1, valuePropInfo1);
PropertyInfo valuePropInfo2 = valueExpr2.Type.GetProperty("Value");
Expression valueValueExpr2 = Expression.Property(valueExpr2, valuePropInfo2);
Expression expr2 = RecursiveAccumulateList(valueValueExpr1, valueValueExpr2, dInfoList, idx + 1);
Type pairType = typeof(Pair <,>).MakeGenericType(expr1.Type, expr2.Type);
return(Expression.New(pairType.GetConstructors()[0], expr1, expr2));
}
}
private DryadQueryNode VisitGroupBy(QueryNodeInfo source,
LambdaExpression keySelectExpr,
LambdaExpression elemSelectExpr,
LambdaExpression resultSelectExpr,
Expression comparerExpr,
Expression queryExpr)
{
DryadQueryNode child = this.Visit(source);
ExpressionInfo einfo = new ExpressionInfo(keySelectExpr);
if (einfo.IsExpensive)
{
// Any method call that is not tagged as "expensive=false" will be deemed expensive.
// if the keySelector is expensive, we rewrite the query so that the key-function is invoked only once
// and the record key passed around via a Pair<TKey,TRecord>.
// keyFunc becomes pair=>pair.Key
// elementSelector must be rewritten so that references to (record) become (pair.Value)
Type[] vkTypes = keySelectExpr.Type.GetGenericArguments();
Type pairType = typeof(Pair<,>).MakeGenericType(vkTypes[1], vkTypes[0]);
ParameterExpression pairParam = Expression.Parameter(pairType, "e");
// Add Select(x => new Pair<K,S>(key(x), x))
ParameterExpression valueParam = keySelectExpr.Parameters[0];
Expression body = Expression.New(pairType.GetConstructors()[0], keySelectExpr.Body, valueParam);
Type delegateType = typeof(Func<,>).MakeGenericType(valueParam.Type, body.Type);
LambdaExpression selectExpr = Expression.Lambda(delegateType, body, valueParam);
child = new DryadSelectNode(QueryNodeType.Select, selectExpr, null, queryExpr, child);
// Change keySelector to e => e.Key
PropertyInfo keyInfo = pairParam.Type.GetProperty("Key");
body = Expression.Property(pairParam, keyInfo);
delegateType = typeof(Func<,>).MakeGenericType(pairParam.Type, body.Type);
keySelectExpr = Expression.Lambda(delegateType, body, pairParam);
// Add or change elementSelector with e.Value
PropertyInfo valueInfo = pairParam.Type.GetProperty("Value");
body = Expression.Property(pairParam, valueInfo);
if (elemSelectExpr != null)
{
ParameterSubst subst = new ParameterSubst(elemSelectExpr.Parameters[0], body);
body = subst.Visit(elemSelectExpr.Body);
}
delegateType = typeof(Func<,>).MakeGenericType(pairParam.Type, body.Type);
elemSelectExpr = Expression.Lambda(delegateType, body, pairParam);
}
Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
queryExpr);
}
Type elemType;
if (elemSelectExpr == null)
{
elemType = keySelectExpr.Type.GetGenericArguments()[0];
}
else
{
elemType = elemSelectExpr.Type.GetGenericArguments()[1];
}
// The comparer object:
object comparer = null;
if (comparerExpr != null)
{
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
comparer = evaluator.Eval(comparerExpr);
}
LambdaExpression keySelectExpr1 = keySelectExpr;
LambdaExpression elemSelectExpr1 = elemSelectExpr;
LambdaExpression resultSelectExpr1 = resultSelectExpr;
LambdaExpression seedExpr1 = null;
LambdaExpression accumulateExpr1 = null;
List<DecompositionInfo> dInfoList = null;
if (resultSelectExpr != null)
{
dInfoList = Decomposition.GetDecompositionInfoList(resultSelectExpr, this.m_codeGen);
}
String groupByOpName = "GroupBy";
DryadQueryNode groupByNode = child;
bool isPartitioned = child.OutputPartition.IsPartitionedBy(keySelectExpr, comparer);
if (dInfoList != null)
{
// ** Decomposable GroupBy-Reduce
// This block creates the first GroupByNode and does some preparation for subsequent nodes.
if (child.IsOrderedBy(keySelectExpr, comparer))
{
groupByOpName = "OrderedGroupBy";
}
int dcnt = dInfoList.Count;
ParameterExpression keyParam;
if (resultSelectExpr.Parameters.Count == 1)
{
keyParam = Expression.Parameter(keyType, HpcLinqCodeGen.MakeUniqueName("k"));
}
else
{
keyParam = resultSelectExpr.Parameters[0];
}
// Seed:
ParameterExpression param2 = Expression.Parameter(
elemType, HpcLinqCodeGen.MakeUniqueName("e"));
Expression zeroExpr = Expression.Constant(0, typeof(int));
Expression seedBody = zeroExpr;
if (dcnt != 0)
{
LambdaExpression seed = dInfoList[dcnt-1].Seed;
ParameterSubst subst = new ParameterSubst(seed.Parameters[0], param2);
seedBody = subst.Visit(seed.Body);
for (int i = dcnt - 2; i >= 0; i--)
{
seed = dInfoList[i].Seed;
subst = new ParameterSubst(seed.Parameters[0], param2);
Expression firstExpr = subst.Visit(seed.Body);
Type newPairType = typeof(Pair<,>).MakeGenericType(firstExpr.Type, seedBody.Type);
seedBody = Expression.New(newPairType.GetConstructors()[0], firstExpr, seedBody);
}
}
LambdaExpression seedExpr = Expression.Lambda(seedBody, param2);
// Accumulate:
ParameterExpression param1 = Expression.Parameter(
seedBody.Type, HpcLinqCodeGen.MakeUniqueName("a"));
Expression accumulateBody = zeroExpr;
if (dcnt != 0)
{
accumulateBody = Decomposition.AccumulateList(param1, param2, dInfoList, 0);
}
LambdaExpression accumulateExpr = Expression.Lambda(accumulateBody, param1, param2);
// Now prepare for the merge-aggregator and/or in the secondary group-by.
// keySelectExpr1: e => e.Key
Type reducerResType = typeof(Pair<,>).MakeGenericType(keyParam.Type, accumulateBody.Type);
ParameterExpression reducerResParam = Expression.Parameter(reducerResType, "e");
PropertyInfo keyInfo = reducerResParam.Type.GetProperty("Key");
Expression body = Expression.Property(reducerResParam, keyInfo);
Type delegateType = typeof(Func<,>).MakeGenericType(reducerResParam.Type, body.Type);
keySelectExpr1 = Expression.Lambda(delegateType, body, reducerResParam);
// elemSelectExpr1: e => e.Value
PropertyInfo valueInfo = reducerResParam.Type.GetProperty("Value");
body = Expression.Property(reducerResParam, valueInfo);
delegateType = typeof(Func<,>).MakeGenericType(reducerResParam.Type, body.Type);
elemSelectExpr1 = Expression.Lambda(delegateType, body, reducerResParam);
// SeedExpr1
param2 = Expression.Parameter(elemSelectExpr1.Body.Type,
HpcLinqCodeGen.MakeUniqueName("e"));
seedExpr1 = Expression.Lambda(param2, param2);
// AccumulateExpr1
Expression recursiveAccumulateBody = zeroExpr;
if (dcnt != 0)
{
recursiveAccumulateBody = Decomposition.RecursiveAccumulateList(param1, param2, dInfoList, 0);
}
accumulateExpr1 = Expression.Lambda(recursiveAccumulateBody, param1, param2);
// resultSelectExpr1
resultSelectExpr1 = null;
// The first groupByNode.
// If the input was already correctly partitioned, this will be the only groupByNode.
bool isPartial = StaticConfig.GroupByLocalAggregationIsPartial && !isPartitioned;
groupByNode = new DryadGroupByNode(
groupByOpName, keySelectExpr, elemSelectExpr, null,
seedExpr, accumulateExpr, accumulateExpr1, comparerExpr,
isPartial, queryExpr, child);
}
else
{
// Can't do partial aggregation.
// Use sort, mergesort, and ordered groupby, if TKey implements IComparable.
if ((comparer != null && TypeSystem.IsComparer(comparer, keyType)) ||
(comparer == null && TypeSystem.HasDefaultComparer(keyType)))
{
if (!child.IsOrderedBy(keySelectExpr, comparer))
{
groupByNode = new DryadOrderByNode(keySelectExpr, comparerExpr, true, queryExpr, child);
}
groupByOpName = "OrderedGroupBy";
}
// Add a GroupByNode if it is partitioned or has elementSelector.
// If the input was already correctly partitioned, this will be the only groupByNode.
if (isPartitioned)
{
groupByNode = new DryadGroupByNode(groupByOpName,
keySelectExpr,
elemSelectExpr,
resultSelectExpr,
null, // seed
null, // accumulate
null, // recursiveAccumulate
comparerExpr,
false, // isPartial
queryExpr,
groupByNode);
}
else if (elemSelectExpr != null)
{
// Local GroupBy without resultSelector:
groupByNode = new DryadGroupByNode(groupByOpName,
keySelectExpr,
elemSelectExpr,
null, // resultSelect
null, // seed
null, // accumulate
null, // recursiveAccumulate
comparerExpr,
StaticConfig.GroupByLocalAggregationIsPartial, // isPartial
queryExpr,
groupByNode);
// keySelectExpr1: g => g.Key
ParameterExpression groupParam = Expression.Parameter(groupByNode.OutputTypes[0], "g");
PropertyInfo keyInfo = groupParam.Type.GetProperty("Key");
Expression body = Expression.Property(groupParam, keyInfo);
Type delegateType = typeof(Func<,>).MakeGenericType(groupParam.Type, body.Type);
keySelectExpr1 = Expression.Lambda(delegateType, body, groupParam);
// No elementSelector
elemSelectExpr1 = null;
// resultSelectExpr1
ParameterExpression keyParam;
Type groupType = typeof(IEnumerable<>).MakeGenericType(groupByNode.OutputTypes[0]);
groupParam = Expression.Parameter(groupType, HpcLinqCodeGen.MakeUniqueName("g"));
if (resultSelectExpr == null)
{
// resultSelectExpr1: (k, g) => MakeDryadLinqGroup(k, g)
keyParam = Expression.Parameter(keySelectExpr1.Body.Type, HpcLinqCodeGen.MakeUniqueName("k"));
MethodInfo groupingInfo = typeof(HpcLinqEnumerable).GetMethod("MakeHpcLinqGroup");
groupingInfo = groupingInfo.MakeGenericMethod(keyParam.Type, elemType);
body = Expression.Call(groupingInfo, keyParam, groupParam);
}
else
{
// resultSelectExpr1: (k, g) => resultSelectExpr(k, FlattenGroups(g))
keyParam = resultSelectExpr.Parameters[0];
MethodInfo flattenInfo = typeof(HpcLinqEnumerable).GetMethod("FlattenGroups");
flattenInfo = flattenInfo.MakeGenericMethod(keyParam.Type, elemType);
Expression groupExpr = Expression.Call(flattenInfo, groupParam);
ParameterSubst subst = new ParameterSubst(resultSelectExpr.Parameters[1], groupExpr);
body = subst.Visit(resultSelectExpr.Body);
}
delegateType = typeof(Func<,,>).MakeGenericType(keyParam.Type, groupParam.Type, body.Type);
resultSelectExpr1 = Expression.Lambda(delegateType, body, keyParam, groupParam);
}
}
// At this point, the first GroupByNode has been created.
DryadMergeNode mergeNode = null;
DryadQueryNode groupByNode1 = groupByNode;
if (!isPartitioned)
{
// Create HashPartitionNode, MergeNode, and second GroupByNode
// Note, if we are doing decomposable-GroupByReduce, there is still some work to go after this
// - attach the combiner to the first merge-node
// - attach the combiner to the merge-node
// - attach finalizer to second GroupBy
int parCount = (groupByNode.IsDynamic) ? StaticConfig.DefaultPartitionCount : groupByNode.OutputPartition.Count;
bool isDynamic = (StaticConfig.DynamicOptLevel & StaticConfig.DynamicHashPartitionLevel) != 0;
DryadQueryNode hdistNode = new DryadHashPartitionNode(keySelectExpr1, comparerExpr, parCount,
isDynamic, queryExpr, groupByNode);
// Create the Merge Node
if (groupByOpName == "OrderedGroupBy")
{
// Mergesort with the same keySelector of the hash partition
mergeNode = new DryadMergeNode(keySelectExpr1, comparerExpr, true, false, queryExpr, hdistNode);
}
else
{
// Random merge
mergeNode = new DryadMergeNode(false, false, queryExpr, hdistNode);
}
groupByNode1 = new DryadGroupByNode(groupByOpName, keySelectExpr1, elemSelectExpr1,
resultSelectExpr1, seedExpr1, accumulateExpr1,
accumulateExpr1, comparerExpr, false, queryExpr,
mergeNode);
}
// Final tidy-up for decomposable GroupBy-Reduce pattern.
// - attach combiner to first GroupByNode
// - attache combiner to MergeNode as an aggregator
// - build a SelectNode to project out results and call finalizer on them.
if (dInfoList != null)
{
// Add dynamic aggregator to the merge-node, if applicable
if (StaticConfig.GroupByDynamicReduce && !isPartitioned)
{
mergeNode.AddAggregateNode(groupByNode1);
}
// Add the final Select node
Type keyResultPairType = typeof(Pair<,>).MakeGenericType(keyType, seedExpr1.Body.Type);
ParameterExpression keyResultPairParam = Expression.Parameter(keyResultPairType,
HpcLinqCodeGen.MakeUniqueName("e"));
PropertyInfo valuePropInfo_1 = keyResultPairType.GetProperty("Value");
Expression combinedValueExpr = Expression.Property(keyResultPairParam, valuePropInfo_1);
// First, build the combinerList
int dcnt = dInfoList.Count;
Expression[] combinerList = new Expression[dcnt];
for (int i = 0; i < dcnt; i++)
{
if (i + 1 == dcnt)
{
combinerList[i] = combinedValueExpr;
}
else
{
PropertyInfo keyPropInfo = combinedValueExpr.Type.GetProperty("Key");
combinerList[i] = Expression.Property(combinedValueExpr, keyPropInfo);
PropertyInfo valuePropInfo = combinedValueExpr.Type.GetProperty("Value");
combinedValueExpr = Expression.Property(combinedValueExpr, valuePropInfo);
}
LambdaExpression finalizerExpr = dInfoList[i].FinalReducer;
if (finalizerExpr != null)
{
ParameterSubst subst = new ParameterSubst(finalizerExpr.Parameters[0], combinerList[i]);
combinerList[i] = subst.Visit(finalizerExpr.Body);
}
}
// Build the funcList
Expression[] funcList = new Expression[dcnt];
for (int i = 0; i < dcnt; i++)
{
funcList[i] = dInfoList[i].Func;
}
// Apply the substitutions
CombinerSubst combinerSubst = new CombinerSubst(resultSelectExpr, keyResultPairParam, funcList, combinerList);
Expression finalizerSelectBody = combinerSubst.Visit();
// Finally, the Select node
Type delegateType = typeof(Func<,>).MakeGenericType(keyResultPairType, finalizerSelectBody.Type);
LambdaExpression selectExpr = Expression.Lambda(delegateType, finalizerSelectBody, keyResultPairParam);
groupByNode1 = new DryadSelectNode(QueryNodeType.Select, selectExpr, null, queryExpr, groupByNode1);
}
return groupByNode1;
}
private static DecompositionInfo GetDecompositionInfo(ParameterExpression groupParam,
MethodCallExpression mcExpr,
HpcLinqCodeGen codeGen)
{
if (mcExpr.Arguments.Count == 0 || mcExpr.Arguments[0] != groupParam)
{
return(null);
}
for (int i = 1; i < mcExpr.Arguments.Count; i++)
{
if (HpcLinqExpression.Contains(groupParam, mcExpr.Arguments[i]))
{
return(null);
}
}
ExpressionSimplifier <object> evaluator = new ExpressionSimplifier <object>();
Type[] paramTypeArgs = groupParam.Type.GetGenericArguments();
Type sourceElemType = paramTypeArgs[paramTypeArgs.Length - 1];
Type resultType = mcExpr.Type;
Type decomposerType = null;
DecomposableAttribute decomposableAttrib = AttributeSystem.GetDecomposableAttrib(mcExpr);
if (decomposableAttrib != null)
{
decomposerType = decomposableAttrib.DecompositionType;
}
else
{
MethodInfo mInfo = mcExpr.Method;
if (mInfo.DeclaringType == typeof(System.Linq.Enumerable) ||
mInfo.DeclaringType == typeof(System.Linq.Queryable))
{
// For built-in decomposable operators.
switch (mInfo.Name)
{
case "Count":
case "LongCount":
{
Type outputType;
Expression body;
if (mInfo.Name == "Count")
{
outputType = typeof(Int32);
body = Expression.Constant(1, outputType);
}
else
{
outputType = typeof(Int64);
body = Expression.Constant((long)1, outputType);
}
ParameterExpression param1 = Expression.Parameter(outputType, "a");
ParameterExpression param2 = Expression.Parameter(sourceElemType, "e");
LambdaExpression seedExpr = Expression.Lambda(body, param2);
body = Expression.AddChecked(param1, body);
LambdaExpression accumulateExpr = Expression.Lambda(body, param1, param2);
param2 = Expression.Parameter(outputType, "b");
body = Expression.AddChecked(param1, param2);
LambdaExpression recursiveAccumulateExpr = Expression.Lambda(body, param1, param2);
return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null));
}
case "Any":
{
ParameterExpression param1 = Expression.Parameter(typeof(bool), "a");
ParameterExpression param2;
Expression body;
if (mcExpr.Arguments.Count == 1)
{
param2 = Expression.Parameter(sourceElemType, "e");
body = Expression.Constant(true, typeof(bool));
}
else
{
LambdaExpression predExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[1]);
param2 = predExpr.Parameters[0];
body = predExpr.Body;
}
LambdaExpression seedExpr = Expression.Lambda(body, param2);
LambdaExpression accumulateExpr = Expression.Lambda(Expression.Or(param1, body), param1, param2);
param2 = Expression.Parameter(typeof(bool), "b");
body = Expression.Or(param1, param2);
LambdaExpression recursiveAccumulateExpr = Expression.Lambda(body, param1, param2);
return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null));
}
case "All":
{
ParameterExpression param1 = Expression.Parameter(typeof(bool), "a");
LambdaExpression predExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[1]);
ParameterExpression param2 = predExpr.Parameters[0];
Expression body = predExpr.Body;
LambdaExpression seedExpr = Expression.Lambda(body, param2);
LambdaExpression accumulateExpr = Expression.Lambda(Expression.And(param1, body), param1, param2);
param2 = Expression.Parameter(typeof(bool), "b");
body = Expression.And(param1, param2);
LambdaExpression recursiveAccumulateExpr = Expression.Lambda(body, param1, param2);
return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null));
}
case "First":
{
ParameterExpression param1 = Expression.Parameter(sourceElemType, "a");
ParameterExpression param2 = Expression.Parameter(sourceElemType, "e");
LambdaExpression seedExpr = Expression.Lambda(param2, param2);
LambdaExpression accumulateExpr = Expression.Lambda(param1, param1, param2);
LambdaExpression recursiveAccumulateExpr = accumulateExpr;
return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null));
}
case "Last":
{
ParameterExpression param1 = Expression.Parameter(sourceElemType, "a");
ParameterExpression param2 = Expression.Parameter(sourceElemType, "e");
LambdaExpression seedExpr = Expression.Lambda(param2, param2);
LambdaExpression accumulateExpr = Expression.Lambda(param2, param1, param2);
LambdaExpression recursiveAccumulateExpr = accumulateExpr;
return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null));
}
case "Sum":
{
ParameterExpression param1;
ParameterExpression param2;
Expression arg2;
if (mInfo.GetParameters().Length == 1)
{
param2 = Expression.Parameter(sourceElemType, "e");
arg2 = param2;
}
else
{
LambdaExpression selectExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[1]);
param2 = selectExpr.Parameters[0];
arg2 = selectExpr.Body;
}
Expression abody, sbody;
if (arg2.Type.IsGenericType)
{
param1 = Expression.Parameter(arg2.Type.GetGenericArguments()[0], "a");
MethodInfo accumulateInfo = typeof(HpcLinqVertex).GetMethod(
"SumAccumulate",
new Type[] { param1.Type, arg2.Type });
sbody = Expression.Constant(0, param1.Type);
sbody = Expression.Call(accumulateInfo, sbody, arg2);
abody = Expression.Call(accumulateInfo, param1, arg2);
}
else
{
param1 = Expression.Parameter(arg2.Type, "a");
sbody = arg2;
abody = Expression.AddChecked(param1, arg2);
}
LambdaExpression seedExpr = Expression.Lambda(sbody, param2);
LambdaExpression accumulateExpr = Expression.Lambda(abody, param1, param2);
param2 = Expression.Parameter(param1.Type, "b");
Expression rbody = Expression.AddChecked(param1, param2);
LambdaExpression recursiveAccumulateExpr = Expression.Lambda(rbody, param1, param2);
Expression fbody = Expression.Convert(param1, arg2.Type);
LambdaExpression finalReduceExpr = Expression.Lambda(fbody, param1);
return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr,
recursiveAccumulateExpr, finalReduceExpr));
}
case "Max":
case "Min":
{
ParameterExpression param2;
Expression abody;
if (mInfo.GetParameters().Length == 1)
{
param2 = Expression.Parameter(sourceElemType, "e");
abody = param2;
}
else
{
LambdaExpression selectExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[1]);
param2 = selectExpr.Parameters[0];
abody = selectExpr.Body;
}
ParameterExpression param1 = Expression.Parameter(abody.Type, "a");
Expression sbody = abody;
MethodInfo accumulateInfo;
string methodName = (mInfo.Name == "Max") ? "MaxAccumulate" : "MinAccumulate";
if (mInfo.IsGenericMethod && (mInfo.GetParameters().Length == 1))
{
accumulateInfo = typeof(HpcLinqVertex).GetMethod(methodName + "Generic");
accumulateInfo = accumulateInfo.MakeGenericMethod(sourceElemType);
}
else
{
accumulateInfo = typeof(HpcLinqVertex).GetMethod(
methodName,
new Type[] { param1.Type, abody.Type });
}
abody = Expression.Call(accumulateInfo, param1, abody);
LambdaExpression seedExpr = Expression.Lambda(sbody, param2);
LambdaExpression accumulateExpr = Expression.Lambda(abody, param1, param2);
param2 = Expression.Parameter(param1.Type, "b");
Expression rbody = Expression.Call(accumulateInfo, param1, param2);
LambdaExpression recursiveAccumulateExpr = Expression.Lambda(rbody, param1, param2);
return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null));
}
case "Aggregate":
{
ParameterExpression elemParam = Expression.Parameter(sourceElemType, "e");
LambdaExpression accumulateExpr;
LambdaExpression seedExpr;
if (mcExpr.Arguments.Count == 2)
{
accumulateExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[1]);
seedExpr = Expression.Lambda(elemParam, elemParam);
}
else
{
accumulateExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[2]);
object seedVal = evaluator.Eval(mcExpr.Arguments[1]);
Expression body = Expression.Constant(seedVal, seedVal.GetType());
ParameterSubst subst = new ParameterSubst(accumulateExpr.Parameters[0], body);
body = subst.Visit(accumulateExpr.Body);
seedExpr = Expression.Lambda(body, accumulateExpr.Parameters[1]);
}
if (!HpcLinqExpression.IsAssociative(accumulateExpr))
{
return(null);
}
LambdaExpression recursiveAccumulateExpr = HpcLinqExpression.GetAssociativeCombiner(accumulateExpr);
return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null));
}
case "Average":
{
ParameterExpression param2;
Expression abody;
if (mInfo.GetParameters().Length == 1)
{
param2 = Expression.Parameter(sourceElemType, "e");
abody = param2;
}
else
{
LambdaExpression selectExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[1]);
param2 = selectExpr.Parameters[0];
abody = selectExpr.Body;
}
Type aggValueType = abody.Type;
if (aggValueType == typeof(int) ||
aggValueType == typeof(int?))
{
aggValueType = typeof(long);
}
else if (aggValueType == typeof(long?))
{
aggValueType = typeof(long);
}
else if (aggValueType == typeof(float) ||
aggValueType == typeof(float?))
{
aggValueType = typeof(double);
}
else if (aggValueType == typeof(double?))
{
aggValueType = typeof(double);
}
else if (aggValueType == typeof(decimal?))
{
aggValueType = typeof(decimal);
}
Type sumAndCountType = typeof(AggregateValue <>).MakeGenericType(aggValueType);
ParameterExpression param1 = Expression.Parameter(sumAndCountType, "a");
MethodInfo accumulateInfo = typeof(HpcLinqVertex).GetMethod(
"AverageAccumulate",
new Type[] { sumAndCountType, abody.Type });
// Seed:
Expression sbody = Expression.New(sumAndCountType);
sbody = Expression.Call(accumulateInfo, sbody, abody);
LambdaExpression seedExpr = Expression.Lambda(sbody, param2);
// Accumulate:
abody = Expression.Call(accumulateInfo, param1, abody);
LambdaExpression accumulateExpr = Expression.Lambda(abody, param1, param2);
// RecursiveAccumulate:
param2 = Expression.Parameter(param1.Type, "b");
PropertyInfo valueInfo = sumAndCountType.GetProperty("Value");
PropertyInfo countInfo = sumAndCountType.GetProperty("Count");
Expression sumExpr1 = Expression.Property(param1, valueInfo);
Expression countExpr1 = Expression.Property(param1, countInfo);
Expression sumExpr2 = Expression.Property(param2, valueInfo);
Expression countExpr2 = Expression.Property(param2, countInfo);
Expression sumExpr = Expression.AddChecked(sumExpr1, sumExpr2);
Expression countExpr = Expression.AddChecked(countExpr1, countExpr2);
ConstructorInfo cinfo = sumAndCountType.GetConstructor(new Type[] { sumExpr.Type, countExpr.Type });
Expression rbody = Expression.New(cinfo, sumExpr, countExpr);
LambdaExpression recursiveAccumulateExpr = Expression.Lambda(rbody, param1, param2);
// FinalReduce:
if (sumExpr1.Type == typeof(long))
{
sumExpr1 = Expression.Convert(sumExpr1, typeof(double));
}
Expression fbody = Expression.Divide(sumExpr1, countExpr1);
fbody = Expression.Convert(fbody, resultType);
if (resultType.IsGenericType)
{
Expression zeroExpr = Expression.Constant(0, typeof(long));
Expression condExpr = Expression.GreaterThan(countExpr1, zeroExpr);
Expression nullExpr = Expression.Constant(null, resultType);
fbody = Expression.Condition(condExpr, fbody, nullExpr);
}
LambdaExpression finalReduceExpr = Expression.Lambda(fbody, param1);
return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, finalReduceExpr));
}
case "Contains":
{
decomposerType = typeof(ContainsDecomposition <>).MakeGenericType(sourceElemType);
break;
}
case "Distinct":
{
decomposerType = typeof(DistinctDecomposition <>).MakeGenericType(sourceElemType);
break;
}
default:
{
return(null);
}
}
}
}
if (decomposerType == null)
{
return(null);
}
Type implementedInterface = null;
Type[] interfaces = decomposerType.GetInterfaces();
foreach (Type intf in interfaces)
{
if (intf.GetGenericTypeDefinition() == typeof(IDecomposable <, ,>))
{
if (implementedInterface != null)
{
throw new DryadLinqException("Decomposition class can implement only one decomposable interface.");
}
implementedInterface = intf;
}
}
if (implementedInterface == null ||
implementedInterface.GetGenericArguments().Length != 3)
{
throw new DryadLinqException("Decomposition class " + decomposerType.FullName +
"must implement IDecomposable<,,>");
}
// The second type of the implemented interface definition is the accumulatorType.
Type accumulatorType = implementedInterface.GetGenericArguments()[1];
// Now check that all the types match up.
Type decomposerInterface = typeof(IDecomposable <, ,>).MakeGenericType(
sourceElemType, accumulatorType, resultType);
if (!decomposerInterface.IsAssignableFrom(decomposerType))
{
throw new DryadLinqException("Decomposition class must match the function that it decorates.");
}
if (decomposerType.ContainsGenericParameters)
{
if (decomposerType.GetGenericArguments().Length != 1 ||
!decomposerType.GetGenericArguments()[0].IsGenericParameter)
{
throw new DryadLinqException(decomposerType.Name + " must match the function it annotates.");
}
decomposerType = decomposerType.MakeGenericType(sourceElemType);
}
if (decomposerType.GetConstructor(Type.EmptyTypes) == null)
{
throw new DryadLinqException("Decomposition class must have a default constructor.");
}
// Add to the codegen a call of the static Initializer of decomposerType
Expression[] args = new Expression[mcExpr.Arguments.Count - 1];
for (int i = 0; i < args.Length; i++)
{
args[i] = Expression.Convert(mcExpr.Arguments[i + 1], typeof(object));
}
Expression stateExpr = Expression.NewArrayInit(typeof(object), args);
string decomposerName = codeGen.AddDecompositionInitializer(decomposerType, stateExpr);
ParameterExpression decomposer = Expression.Parameter(decomposerType, decomposerName);
// Seed: TSource => TAccumulate
MethodInfo seedInfo1 = decomposerType.GetMethod("Seed");
ParameterExpression p2 = Expression.Parameter(sourceElemType, "e");
Expression sbody1 = Expression.Call(decomposer, seedInfo1, p2);
LambdaExpression seedExpr1 = Expression.Lambda(sbody1, p2);
// Accumulate: (TAccumulate, TSource) => TAccumulate
MethodInfo accumulateInfo1 = decomposerType.GetMethod("Accumulate");
ParameterExpression p1 = Expression.Parameter(accumulatorType, "a");
Expression abody1 = Expression.Call(decomposer, accumulateInfo1, p1, p2);
LambdaExpression accumulateExpr1 = Expression.Lambda(abody1, p1, p2);
// RecursiveAccumulate: (TAccumulate, TAccumulate) => TAccumulate
MethodInfo recursiveAccumulateInfo1 = decomposerType.GetMethod("RecursiveAccumulate");
p2 = Expression.Parameter(accumulatorType, "e");
Expression rbody1 = Expression.Call(decomposer, recursiveAccumulateInfo1, p1, p2);
LambdaExpression recursiveAccumulateExpr1 = Expression.Lambda(rbody1, p1, p2);
// FinalReduce: TAccumulate => TResult
MethodInfo finalReduceInfo1 = decomposerType.GetMethod("FinalReduce");
Expression fbody1 = Expression.Call(decomposer, finalReduceInfo1, p1);
LambdaExpression finalReduceExpr1 = Expression.Lambda(fbody1, p1);
return(new DecompositionInfo(mcExpr, seedExpr1, accumulateExpr1, recursiveAccumulateExpr1, finalReduceExpr1));
}
