// Precondition: idx < dInfoList.Count internal static Expression RecursiveAccumulateList(Expression valueExpr1, Expression valueExpr2, List <DecompositionInfo> dInfoList, int idx) { LambdaExpression recursiveAccumulateExpr = dInfoList[idx].RecursiveAccumulator; if (dInfoList.Count == idx + 1) { ParameterSubst subst = new ParameterSubst(recursiveAccumulateExpr.Parameters[0], valueExpr1); Expression resultExpr = subst.Visit(recursiveAccumulateExpr.Body); subst = new ParameterSubst(recursiveAccumulateExpr.Parameters[1], valueExpr2); return(subst.Visit(resultExpr)); } else { PropertyInfo keyPropInfo1 = valueExpr1.Type.GetProperty("Key"); Expression keyValueExpr1 = Expression.Property(valueExpr1, keyPropInfo1); PropertyInfo keyPropInfo2 = valueExpr2.Type.GetProperty("Key"); Expression keyValueExpr2 = Expression.Property(valueExpr2, keyPropInfo2); ParameterSubst subst = new ParameterSubst(recursiveAccumulateExpr.Parameters[0], keyValueExpr1); Expression expr1 = subst.Visit(recursiveAccumulateExpr.Body); subst = new ParameterSubst(recursiveAccumulateExpr.Parameters[1], keyValueExpr2); expr1 = subst.Visit(expr1); PropertyInfo valuePropInfo1 = valueExpr1.Type.GetProperty("Value"); Expression valueValueExpr1 = Expression.Property(valueExpr1, valuePropInfo1); PropertyInfo valuePropInfo2 = valueExpr2.Type.GetProperty("Value"); Expression valueValueExpr2 = Expression.Property(valueExpr2, valuePropInfo2); Expression expr2 = RecursiveAccumulateList(valueValueExpr1, valueValueExpr2, dInfoList, idx + 1); Type pairType = typeof(Pair <,>).MakeGenericType(expr1.Type, expr2.Type); return(Expression.New(pairType.GetConstructors()[0], expr1, expr2)); } }
private DryadQueryNode VisitGroupBy(QueryNodeInfo source, LambdaExpression keySelectExpr, LambdaExpression elemSelectExpr, LambdaExpression resultSelectExpr, Expression comparerExpr, Expression queryExpr) { DryadQueryNode child = this.Visit(source); ExpressionInfo einfo = new ExpressionInfo(keySelectExpr); if (einfo.IsExpensive) { // Any method call that is not tagged as "expensive=false" will be deemed expensive. // if the keySelector is expensive, we rewrite the query so that the key-function is invoked only once // and the record key passed around via a Pair<TKey,TRecord>. // keyFunc becomes pair=>pair.Key // elementSelector must be rewritten so that references to (record) become (pair.Value) Type[] vkTypes = keySelectExpr.Type.GetGenericArguments(); Type pairType = typeof(Pair<,>).MakeGenericType(vkTypes[1], vkTypes[0]); ParameterExpression pairParam = Expression.Parameter(pairType, "e"); // Add Select(x => new Pair<K,S>(key(x), x)) ParameterExpression valueParam = keySelectExpr.Parameters[0]; Expression body = Expression.New(pairType.GetConstructors()[0], keySelectExpr.Body, valueParam); Type delegateType = typeof(Func<,>).MakeGenericType(valueParam.Type, body.Type); LambdaExpression selectExpr = Expression.Lambda(delegateType, body, valueParam); child = new DryadSelectNode(QueryNodeType.Select, selectExpr, null, queryExpr, child); // Change keySelector to e => e.Key PropertyInfo keyInfo = pairParam.Type.GetProperty("Key"); body = Expression.Property(pairParam, keyInfo); delegateType = typeof(Func<,>).MakeGenericType(pairParam.Type, body.Type); keySelectExpr = Expression.Lambda(delegateType, body, pairParam); // Add or change elementSelector with e.Value PropertyInfo valueInfo = pairParam.Type.GetProperty("Value"); body = Expression.Property(pairParam, valueInfo); if (elemSelectExpr != null) { ParameterSubst subst = new ParameterSubst(elemSelectExpr.Parameters[0], body); body = subst.Visit(elemSelectExpr.Body); } delegateType = typeof(Func<,>).MakeGenericType(pairParam.Type, body.Type); elemSelectExpr = Expression.Lambda(delegateType, body, pairParam); } Type keyType = keySelectExpr.Type.GetGenericArguments()[1]; if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType)) { throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType), queryExpr); } Type elemType; if (elemSelectExpr == null) { elemType = keySelectExpr.Type.GetGenericArguments()[0]; } else { elemType = elemSelectExpr.Type.GetGenericArguments()[1]; } // The comparer object: object comparer = null; if (comparerExpr != null) { ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>(); comparer = evaluator.Eval(comparerExpr); } LambdaExpression keySelectExpr1 = keySelectExpr; LambdaExpression elemSelectExpr1 = elemSelectExpr; LambdaExpression resultSelectExpr1 = resultSelectExpr; LambdaExpression seedExpr1 = null; LambdaExpression accumulateExpr1 = null; List<DecompositionInfo> dInfoList = null; if (resultSelectExpr != null) { dInfoList = Decomposition.GetDecompositionInfoList(resultSelectExpr, this.m_codeGen); } String groupByOpName = "GroupBy"; DryadQueryNode groupByNode = child; bool isPartitioned = child.OutputPartition.IsPartitionedBy(keySelectExpr, comparer); if (dInfoList != null) { // ** Decomposable GroupBy-Reduce // This block creates the first GroupByNode and does some preparation for subsequent nodes. if (child.IsOrderedBy(keySelectExpr, comparer)) { groupByOpName = "OrderedGroupBy"; } int dcnt = dInfoList.Count; ParameterExpression keyParam; if (resultSelectExpr.Parameters.Count == 1) { keyParam = Expression.Parameter(keyType, HpcLinqCodeGen.MakeUniqueName("k")); } else { keyParam = resultSelectExpr.Parameters[0]; } // Seed: ParameterExpression param2 = Expression.Parameter( elemType, HpcLinqCodeGen.MakeUniqueName("e")); Expression zeroExpr = Expression.Constant(0, typeof(int)); Expression seedBody = zeroExpr; if (dcnt != 0) { LambdaExpression seed = dInfoList[dcnt-1].Seed; ParameterSubst subst = new ParameterSubst(seed.Parameters[0], param2); seedBody = subst.Visit(seed.Body); for (int i = dcnt - 2; i >= 0; i--) { seed = dInfoList[i].Seed; subst = new ParameterSubst(seed.Parameters[0], param2); Expression firstExpr = subst.Visit(seed.Body); Type newPairType = typeof(Pair<,>).MakeGenericType(firstExpr.Type, seedBody.Type); seedBody = Expression.New(newPairType.GetConstructors()[0], firstExpr, seedBody); } } LambdaExpression seedExpr = Expression.Lambda(seedBody, param2); // Accumulate: ParameterExpression param1 = Expression.Parameter( seedBody.Type, HpcLinqCodeGen.MakeUniqueName("a")); Expression accumulateBody = zeroExpr; if (dcnt != 0) { accumulateBody = Decomposition.AccumulateList(param1, param2, dInfoList, 0); } LambdaExpression accumulateExpr = Expression.Lambda(accumulateBody, param1, param2); // Now prepare for the merge-aggregator and/or in the secondary group-by. // keySelectExpr1: e => e.Key Type reducerResType = typeof(Pair<,>).MakeGenericType(keyParam.Type, accumulateBody.Type); ParameterExpression reducerResParam = Expression.Parameter(reducerResType, "e"); PropertyInfo keyInfo = reducerResParam.Type.GetProperty("Key"); Expression body = Expression.Property(reducerResParam, keyInfo); Type delegateType = typeof(Func<,>).MakeGenericType(reducerResParam.Type, body.Type); keySelectExpr1 = Expression.Lambda(delegateType, body, reducerResParam); // elemSelectExpr1: e => e.Value PropertyInfo valueInfo = reducerResParam.Type.GetProperty("Value"); body = Expression.Property(reducerResParam, valueInfo); delegateType = typeof(Func<,>).MakeGenericType(reducerResParam.Type, body.Type); elemSelectExpr1 = Expression.Lambda(delegateType, body, reducerResParam); // SeedExpr1 param2 = Expression.Parameter(elemSelectExpr1.Body.Type, HpcLinqCodeGen.MakeUniqueName("e")); seedExpr1 = Expression.Lambda(param2, param2); // AccumulateExpr1 Expression recursiveAccumulateBody = zeroExpr; if (dcnt != 0) { recursiveAccumulateBody = Decomposition.RecursiveAccumulateList(param1, param2, dInfoList, 0); } accumulateExpr1 = Expression.Lambda(recursiveAccumulateBody, param1, param2); // resultSelectExpr1 resultSelectExpr1 = null; // The first groupByNode. // If the input was already correctly partitioned, this will be the only groupByNode. bool isPartial = StaticConfig.GroupByLocalAggregationIsPartial && !isPartitioned; groupByNode = new DryadGroupByNode( groupByOpName, keySelectExpr, elemSelectExpr, null, seedExpr, accumulateExpr, accumulateExpr1, comparerExpr, isPartial, queryExpr, child); } else { // Can't do partial aggregation. // Use sort, mergesort, and ordered groupby, if TKey implements IComparable. if ((comparer != null && TypeSystem.IsComparer(comparer, keyType)) || (comparer == null && TypeSystem.HasDefaultComparer(keyType))) { if (!child.IsOrderedBy(keySelectExpr, comparer)) { groupByNode = new DryadOrderByNode(keySelectExpr, comparerExpr, true, queryExpr, child); } groupByOpName = "OrderedGroupBy"; } // Add a GroupByNode if it is partitioned or has elementSelector. // If the input was already correctly partitioned, this will be the only groupByNode. if (isPartitioned) { groupByNode = new DryadGroupByNode(groupByOpName, keySelectExpr, elemSelectExpr, resultSelectExpr, null, // seed null, // accumulate null, // recursiveAccumulate comparerExpr, false, // isPartial queryExpr, groupByNode); } else if (elemSelectExpr != null) { // Local GroupBy without resultSelector: groupByNode = new DryadGroupByNode(groupByOpName, keySelectExpr, elemSelectExpr, null, // resultSelect null, // seed null, // accumulate null, // recursiveAccumulate comparerExpr, StaticConfig.GroupByLocalAggregationIsPartial, // isPartial queryExpr, groupByNode); // keySelectExpr1: g => g.Key ParameterExpression groupParam = Expression.Parameter(groupByNode.OutputTypes[0], "g"); PropertyInfo keyInfo = groupParam.Type.GetProperty("Key"); Expression body = Expression.Property(groupParam, keyInfo); Type delegateType = typeof(Func<,>).MakeGenericType(groupParam.Type, body.Type); keySelectExpr1 = Expression.Lambda(delegateType, body, groupParam); // No elementSelector elemSelectExpr1 = null; // resultSelectExpr1 ParameterExpression keyParam; Type groupType = typeof(IEnumerable<>).MakeGenericType(groupByNode.OutputTypes[0]); groupParam = Expression.Parameter(groupType, HpcLinqCodeGen.MakeUniqueName("g")); if (resultSelectExpr == null) { // resultSelectExpr1: (k, g) => MakeDryadLinqGroup(k, g) keyParam = Expression.Parameter(keySelectExpr1.Body.Type, HpcLinqCodeGen.MakeUniqueName("k")); MethodInfo groupingInfo = typeof(HpcLinqEnumerable).GetMethod("MakeHpcLinqGroup"); groupingInfo = groupingInfo.MakeGenericMethod(keyParam.Type, elemType); body = Expression.Call(groupingInfo, keyParam, groupParam); } else { // resultSelectExpr1: (k, g) => resultSelectExpr(k, FlattenGroups(g)) keyParam = resultSelectExpr.Parameters[0]; MethodInfo flattenInfo = typeof(HpcLinqEnumerable).GetMethod("FlattenGroups"); flattenInfo = flattenInfo.MakeGenericMethod(keyParam.Type, elemType); Expression groupExpr = Expression.Call(flattenInfo, groupParam); ParameterSubst subst = new ParameterSubst(resultSelectExpr.Parameters[1], groupExpr); body = subst.Visit(resultSelectExpr.Body); } delegateType = typeof(Func<,,>).MakeGenericType(keyParam.Type, groupParam.Type, body.Type); resultSelectExpr1 = Expression.Lambda(delegateType, body, keyParam, groupParam); } } // At this point, the first GroupByNode has been created. DryadMergeNode mergeNode = null; DryadQueryNode groupByNode1 = groupByNode; if (!isPartitioned) { // Create HashPartitionNode, MergeNode, and second GroupByNode // Note, if we are doing decomposable-GroupByReduce, there is still some work to go after this // - attach the combiner to the first merge-node // - attach the combiner to the merge-node // - attach finalizer to second GroupBy int parCount = (groupByNode.IsDynamic) ? StaticConfig.DefaultPartitionCount : groupByNode.OutputPartition.Count; bool isDynamic = (StaticConfig.DynamicOptLevel & StaticConfig.DynamicHashPartitionLevel) != 0; DryadQueryNode hdistNode = new DryadHashPartitionNode(keySelectExpr1, comparerExpr, parCount, isDynamic, queryExpr, groupByNode); // Create the Merge Node if (groupByOpName == "OrderedGroupBy") { // Mergesort with the same keySelector of the hash partition mergeNode = new DryadMergeNode(keySelectExpr1, comparerExpr, true, false, queryExpr, hdistNode); } else { // Random merge mergeNode = new DryadMergeNode(false, false, queryExpr, hdistNode); } groupByNode1 = new DryadGroupByNode(groupByOpName, keySelectExpr1, elemSelectExpr1, resultSelectExpr1, seedExpr1, accumulateExpr1, accumulateExpr1, comparerExpr, false, queryExpr, mergeNode); } // Final tidy-up for decomposable GroupBy-Reduce pattern. // - attach combiner to first GroupByNode // - attache combiner to MergeNode as an aggregator // - build a SelectNode to project out results and call finalizer on them. if (dInfoList != null) { // Add dynamic aggregator to the merge-node, if applicable if (StaticConfig.GroupByDynamicReduce && !isPartitioned) { mergeNode.AddAggregateNode(groupByNode1); } // Add the final Select node Type keyResultPairType = typeof(Pair<,>).MakeGenericType(keyType, seedExpr1.Body.Type); ParameterExpression keyResultPairParam = Expression.Parameter(keyResultPairType, HpcLinqCodeGen.MakeUniqueName("e")); PropertyInfo valuePropInfo_1 = keyResultPairType.GetProperty("Value"); Expression combinedValueExpr = Expression.Property(keyResultPairParam, valuePropInfo_1); // First, build the combinerList int dcnt = dInfoList.Count; Expression[] combinerList = new Expression[dcnt]; for (int i = 0; i < dcnt; i++) { if (i + 1 == dcnt) { combinerList[i] = combinedValueExpr; } else { PropertyInfo keyPropInfo = combinedValueExpr.Type.GetProperty("Key"); combinerList[i] = Expression.Property(combinedValueExpr, keyPropInfo); PropertyInfo valuePropInfo = combinedValueExpr.Type.GetProperty("Value"); combinedValueExpr = Expression.Property(combinedValueExpr, valuePropInfo); } LambdaExpression finalizerExpr = dInfoList[i].FinalReducer; if (finalizerExpr != null) { ParameterSubst subst = new ParameterSubst(finalizerExpr.Parameters[0], combinerList[i]); combinerList[i] = subst.Visit(finalizerExpr.Body); } } // Build the funcList Expression[] funcList = new Expression[dcnt]; for (int i = 0; i < dcnt; i++) { funcList[i] = dInfoList[i].Func; } // Apply the substitutions CombinerSubst combinerSubst = new CombinerSubst(resultSelectExpr, keyResultPairParam, funcList, combinerList); Expression finalizerSelectBody = combinerSubst.Visit(); // Finally, the Select node Type delegateType = typeof(Func<,>).MakeGenericType(keyResultPairType, finalizerSelectBody.Type); LambdaExpression selectExpr = Expression.Lambda(delegateType, finalizerSelectBody, keyResultPairParam); groupByNode1 = new DryadSelectNode(QueryNodeType.Select, selectExpr, null, queryExpr, groupByNode1); } return groupByNode1; }
private static DecompositionInfo GetDecompositionInfo(ParameterExpression groupParam, MethodCallExpression mcExpr, HpcLinqCodeGen codeGen) { if (mcExpr.Arguments.Count == 0 || mcExpr.Arguments[0] != groupParam) { return(null); } for (int i = 1; i < mcExpr.Arguments.Count; i++) { if (HpcLinqExpression.Contains(groupParam, mcExpr.Arguments[i])) { return(null); } } ExpressionSimplifier <object> evaluator = new ExpressionSimplifier <object>(); Type[] paramTypeArgs = groupParam.Type.GetGenericArguments(); Type sourceElemType = paramTypeArgs[paramTypeArgs.Length - 1]; Type resultType = mcExpr.Type; Type decomposerType = null; DecomposableAttribute decomposableAttrib = AttributeSystem.GetDecomposableAttrib(mcExpr); if (decomposableAttrib != null) { decomposerType = decomposableAttrib.DecompositionType; } else { MethodInfo mInfo = mcExpr.Method; if (mInfo.DeclaringType == typeof(System.Linq.Enumerable) || mInfo.DeclaringType == typeof(System.Linq.Queryable)) { // For built-in decomposable operators. switch (mInfo.Name) { case "Count": case "LongCount": { Type outputType; Expression body; if (mInfo.Name == "Count") { outputType = typeof(Int32); body = Expression.Constant(1, outputType); } else { outputType = typeof(Int64); body = Expression.Constant((long)1, outputType); } ParameterExpression param1 = Expression.Parameter(outputType, "a"); ParameterExpression param2 = Expression.Parameter(sourceElemType, "e"); LambdaExpression seedExpr = Expression.Lambda(body, param2); body = Expression.AddChecked(param1, body); LambdaExpression accumulateExpr = Expression.Lambda(body, param1, param2); param2 = Expression.Parameter(outputType, "b"); body = Expression.AddChecked(param1, param2); LambdaExpression recursiveAccumulateExpr = Expression.Lambda(body, param1, param2); return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null)); } case "Any": { ParameterExpression param1 = Expression.Parameter(typeof(bool), "a"); ParameterExpression param2; Expression body; if (mcExpr.Arguments.Count == 1) { param2 = Expression.Parameter(sourceElemType, "e"); body = Expression.Constant(true, typeof(bool)); } else { LambdaExpression predExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[1]); param2 = predExpr.Parameters[0]; body = predExpr.Body; } LambdaExpression seedExpr = Expression.Lambda(body, param2); LambdaExpression accumulateExpr = Expression.Lambda(Expression.Or(param1, body), param1, param2); param2 = Expression.Parameter(typeof(bool), "b"); body = Expression.Or(param1, param2); LambdaExpression recursiveAccumulateExpr = Expression.Lambda(body, param1, param2); return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null)); } case "All": { ParameterExpression param1 = Expression.Parameter(typeof(bool), "a"); LambdaExpression predExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[1]); ParameterExpression param2 = predExpr.Parameters[0]; Expression body = predExpr.Body; LambdaExpression seedExpr = Expression.Lambda(body, param2); LambdaExpression accumulateExpr = Expression.Lambda(Expression.And(param1, body), param1, param2); param2 = Expression.Parameter(typeof(bool), "b"); body = Expression.And(param1, param2); LambdaExpression recursiveAccumulateExpr = Expression.Lambda(body, param1, param2); return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null)); } case "First": { ParameterExpression param1 = Expression.Parameter(sourceElemType, "a"); ParameterExpression param2 = Expression.Parameter(sourceElemType, "e"); LambdaExpression seedExpr = Expression.Lambda(param2, param2); LambdaExpression accumulateExpr = Expression.Lambda(param1, param1, param2); LambdaExpression recursiveAccumulateExpr = accumulateExpr; return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null)); } case "Last": { ParameterExpression param1 = Expression.Parameter(sourceElemType, "a"); ParameterExpression param2 = Expression.Parameter(sourceElemType, "e"); LambdaExpression seedExpr = Expression.Lambda(param2, param2); LambdaExpression accumulateExpr = Expression.Lambda(param2, param1, param2); LambdaExpression recursiveAccumulateExpr = accumulateExpr; return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null)); } case "Sum": { ParameterExpression param1; ParameterExpression param2; Expression arg2; if (mInfo.GetParameters().Length == 1) { param2 = Expression.Parameter(sourceElemType, "e"); arg2 = param2; } else { LambdaExpression selectExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[1]); param2 = selectExpr.Parameters[0]; arg2 = selectExpr.Body; } Expression abody, sbody; if (arg2.Type.IsGenericType) { param1 = Expression.Parameter(arg2.Type.GetGenericArguments()[0], "a"); MethodInfo accumulateInfo = typeof(HpcLinqVertex).GetMethod( "SumAccumulate", new Type[] { param1.Type, arg2.Type }); sbody = Expression.Constant(0, param1.Type); sbody = Expression.Call(accumulateInfo, sbody, arg2); abody = Expression.Call(accumulateInfo, param1, arg2); } else { param1 = Expression.Parameter(arg2.Type, "a"); sbody = arg2; abody = Expression.AddChecked(param1, arg2); } LambdaExpression seedExpr = Expression.Lambda(sbody, param2); LambdaExpression accumulateExpr = Expression.Lambda(abody, param1, param2); param2 = Expression.Parameter(param1.Type, "b"); Expression rbody = Expression.AddChecked(param1, param2); LambdaExpression recursiveAccumulateExpr = Expression.Lambda(rbody, param1, param2); Expression fbody = Expression.Convert(param1, arg2.Type); LambdaExpression finalReduceExpr = Expression.Lambda(fbody, param1); return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, finalReduceExpr)); } case "Max": case "Min": { ParameterExpression param2; Expression abody; if (mInfo.GetParameters().Length == 1) { param2 = Expression.Parameter(sourceElemType, "e"); abody = param2; } else { LambdaExpression selectExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[1]); param2 = selectExpr.Parameters[0]; abody = selectExpr.Body; } ParameterExpression param1 = Expression.Parameter(abody.Type, "a"); Expression sbody = abody; MethodInfo accumulateInfo; string methodName = (mInfo.Name == "Max") ? "MaxAccumulate" : "MinAccumulate"; if (mInfo.IsGenericMethod && (mInfo.GetParameters().Length == 1)) { accumulateInfo = typeof(HpcLinqVertex).GetMethod(methodName + "Generic"); accumulateInfo = accumulateInfo.MakeGenericMethod(sourceElemType); } else { accumulateInfo = typeof(HpcLinqVertex).GetMethod( methodName, new Type[] { param1.Type, abody.Type }); } abody = Expression.Call(accumulateInfo, param1, abody); LambdaExpression seedExpr = Expression.Lambda(sbody, param2); LambdaExpression accumulateExpr = Expression.Lambda(abody, param1, param2); param2 = Expression.Parameter(param1.Type, "b"); Expression rbody = Expression.Call(accumulateInfo, param1, param2); LambdaExpression recursiveAccumulateExpr = Expression.Lambda(rbody, param1, param2); return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null)); } case "Aggregate": { ParameterExpression elemParam = Expression.Parameter(sourceElemType, "e"); LambdaExpression accumulateExpr; LambdaExpression seedExpr; if (mcExpr.Arguments.Count == 2) { accumulateExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[1]); seedExpr = Expression.Lambda(elemParam, elemParam); } else { accumulateExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[2]); object seedVal = evaluator.Eval(mcExpr.Arguments[1]); Expression body = Expression.Constant(seedVal, seedVal.GetType()); ParameterSubst subst = new ParameterSubst(accumulateExpr.Parameters[0], body); body = subst.Visit(accumulateExpr.Body); seedExpr = Expression.Lambda(body, accumulateExpr.Parameters[1]); } if (!HpcLinqExpression.IsAssociative(accumulateExpr)) { return(null); } LambdaExpression recursiveAccumulateExpr = HpcLinqExpression.GetAssociativeCombiner(accumulateExpr); return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null)); } case "Average": { ParameterExpression param2; Expression abody; if (mInfo.GetParameters().Length == 1) { param2 = Expression.Parameter(sourceElemType, "e"); abody = param2; } else { LambdaExpression selectExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[1]); param2 = selectExpr.Parameters[0]; abody = selectExpr.Body; } Type aggValueType = abody.Type; if (aggValueType == typeof(int) || aggValueType == typeof(int?)) { aggValueType = typeof(long); } else if (aggValueType == typeof(long?)) { aggValueType = typeof(long); } else if (aggValueType == typeof(float) || aggValueType == typeof(float?)) { aggValueType = typeof(double); } else if (aggValueType == typeof(double?)) { aggValueType = typeof(double); } else if (aggValueType == typeof(decimal?)) { aggValueType = typeof(decimal); } Type sumAndCountType = typeof(AggregateValue <>).MakeGenericType(aggValueType); ParameterExpression param1 = Expression.Parameter(sumAndCountType, "a"); MethodInfo accumulateInfo = typeof(HpcLinqVertex).GetMethod( "AverageAccumulate", new Type[] { sumAndCountType, abody.Type }); // Seed: Expression sbody = Expression.New(sumAndCountType); sbody = Expression.Call(accumulateInfo, sbody, abody); LambdaExpression seedExpr = Expression.Lambda(sbody, param2); // Accumulate: abody = Expression.Call(accumulateInfo, param1, abody); LambdaExpression accumulateExpr = Expression.Lambda(abody, param1, param2); // RecursiveAccumulate: param2 = Expression.Parameter(param1.Type, "b"); PropertyInfo valueInfo = sumAndCountType.GetProperty("Value"); PropertyInfo countInfo = sumAndCountType.GetProperty("Count"); Expression sumExpr1 = Expression.Property(param1, valueInfo); Expression countExpr1 = Expression.Property(param1, countInfo); Expression sumExpr2 = Expression.Property(param2, valueInfo); Expression countExpr2 = Expression.Property(param2, countInfo); Expression sumExpr = Expression.AddChecked(sumExpr1, sumExpr2); Expression countExpr = Expression.AddChecked(countExpr1, countExpr2); ConstructorInfo cinfo = sumAndCountType.GetConstructor(new Type[] { sumExpr.Type, countExpr.Type }); Expression rbody = Expression.New(cinfo, sumExpr, countExpr); LambdaExpression recursiveAccumulateExpr = Expression.Lambda(rbody, param1, param2); // FinalReduce: if (sumExpr1.Type == typeof(long)) { sumExpr1 = Expression.Convert(sumExpr1, typeof(double)); } Expression fbody = Expression.Divide(sumExpr1, countExpr1); fbody = Expression.Convert(fbody, resultType); if (resultType.IsGenericType) { Expression zeroExpr = Expression.Constant(0, typeof(long)); Expression condExpr = Expression.GreaterThan(countExpr1, zeroExpr); Expression nullExpr = Expression.Constant(null, resultType); fbody = Expression.Condition(condExpr, fbody, nullExpr); } LambdaExpression finalReduceExpr = Expression.Lambda(fbody, param1); return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, finalReduceExpr)); } case "Contains": { decomposerType = typeof(ContainsDecomposition <>).MakeGenericType(sourceElemType); break; } case "Distinct": { decomposerType = typeof(DistinctDecomposition <>).MakeGenericType(sourceElemType); break; } default: { return(null); } } } } if (decomposerType == null) { return(null); } Type implementedInterface = null; Type[] interfaces = decomposerType.GetInterfaces(); foreach (Type intf in interfaces) { if (intf.GetGenericTypeDefinition() == typeof(IDecomposable <, ,>)) { if (implementedInterface != null) { throw new DryadLinqException("Decomposition class can implement only one decomposable interface."); } implementedInterface = intf; } } if (implementedInterface == null || implementedInterface.GetGenericArguments().Length != 3) { throw new DryadLinqException("Decomposition class " + decomposerType.FullName + "must implement IDecomposable<,,>"); } // The second type of the implemented interface definition is the accumulatorType. Type accumulatorType = implementedInterface.GetGenericArguments()[1]; // Now check that all the types match up. Type decomposerInterface = typeof(IDecomposable <, ,>).MakeGenericType( sourceElemType, accumulatorType, resultType); if (!decomposerInterface.IsAssignableFrom(decomposerType)) { throw new DryadLinqException("Decomposition class must match the function that it decorates."); } if (decomposerType.ContainsGenericParameters) { if (decomposerType.GetGenericArguments().Length != 1 || !decomposerType.GetGenericArguments()[0].IsGenericParameter) { throw new DryadLinqException(decomposerType.Name + " must match the function it annotates."); } decomposerType = decomposerType.MakeGenericType(sourceElemType); } if (decomposerType.GetConstructor(Type.EmptyTypes) == null) { throw new DryadLinqException("Decomposition class must have a default constructor."); } // Add to the codegen a call of the static Initializer of decomposerType Expression[] args = new Expression[mcExpr.Arguments.Count - 1]; for (int i = 0; i < args.Length; i++) { args[i] = Expression.Convert(mcExpr.Arguments[i + 1], typeof(object)); } Expression stateExpr = Expression.NewArrayInit(typeof(object), args); string decomposerName = codeGen.AddDecompositionInitializer(decomposerType, stateExpr); ParameterExpression decomposer = Expression.Parameter(decomposerType, decomposerName); // Seed: TSource => TAccumulate MethodInfo seedInfo1 = decomposerType.GetMethod("Seed"); ParameterExpression p2 = Expression.Parameter(sourceElemType, "e"); Expression sbody1 = Expression.Call(decomposer, seedInfo1, p2); LambdaExpression seedExpr1 = Expression.Lambda(sbody1, p2); // Accumulate: (TAccumulate, TSource) => TAccumulate MethodInfo accumulateInfo1 = decomposerType.GetMethod("Accumulate"); ParameterExpression p1 = Expression.Parameter(accumulatorType, "a"); Expression abody1 = Expression.Call(decomposer, accumulateInfo1, p1, p2); LambdaExpression accumulateExpr1 = Expression.Lambda(abody1, p1, p2); // RecursiveAccumulate: (TAccumulate, TAccumulate) => TAccumulate MethodInfo recursiveAccumulateInfo1 = decomposerType.GetMethod("RecursiveAccumulate"); p2 = Expression.Parameter(accumulatorType, "e"); Expression rbody1 = Expression.Call(decomposer, recursiveAccumulateInfo1, p1, p2); LambdaExpression recursiveAccumulateExpr1 = Expression.Lambda(rbody1, p1, p2); // FinalReduce: TAccumulate => TResult MethodInfo finalReduceInfo1 = decomposerType.GetMethod("FinalReduce"); Expression fbody1 = Expression.Call(decomposer, finalReduceInfo1, p1); LambdaExpression finalReduceExpr1 = Expression.Lambda(fbody1, p1); return(new DecompositionInfo(mcExpr, seedExpr1, accumulateExpr1, recursiveAccumulateExpr1, finalReduceExpr1)); }