GetDecompositionInfoList(LambdaExpression resultSelectExpr, DryadLinqCodeGen codeGen) { ParameterExpression keyParam; ParameterExpression groupParam; if (resultSelectExpr.Parameters.Count == 1) { keyParam = null; groupParam = resultSelectExpr.Parameters[0]; } else { Debug.Assert(resultSelectExpr.Parameters.Count == 2); keyParam = resultSelectExpr.Parameters[0]; groupParam = resultSelectExpr.Parameters[1]; } List <DecompositionInfo> infoList = new List <DecompositionInfo>(1); bool isDecomposed = GetDecompositionInfoList(keyParam, groupParam, resultSelectExpr.Body, infoList, codeGen); if (isDecomposed) { return(infoList); } return(null); }
public override void Ingress <T>(DryadLinqContext context, IEnumerable <T> source, Uri dataSetUri, DryadLinqMetaData metaData, CompressionScheme compressionScheme, bool isTemp = false) { string account, key, container, blob; AzureUtils.FromAzureUri(dataSetUri, out account, out key, out container, out blob); if (compressionScheme != CompressionScheme.None) { throw new DryadLinqException("Not implemented: writing to Azure temporary storage with compression enabled"); } AzureDfsClient client = new AzureDfsClient(account, key, container); DryadLinqFactory <T> factory = (DryadLinqFactory <T>)DryadLinqCodeGen.GetFactory(context, typeof(T)); using (Stream stream = client.GetFileStreamWriterAsync(blob).Result) { DryadLinqBlockStream nativeStream = new DryadLinqBlockStream(stream); DryadLinqRecordWriter <T> writer = factory.MakeWriter(nativeStream); foreach (T rec in source) { writer.WriteRecordSync(rec); } writer.Close(); } }
private string MakeProcessServiceConfig() { var configDoc = new XDocument(); var docElement = new XElement("PeloponneseConfig"); var psElement = new XElement("ProcessService"); var psPortElement = new XElement("Port"); psPortElement.Value = "8471"; psElement.Add(psPortElement); var psPrefixElement = new XElement("Prefix"); psPrefixElement.Value = "/peloponnese/dpservice/"; psElement.Add(psPrefixElement); var envElement = new XElement("Environment"); psElement.Add(envElement); docElement.Add(psElement); configDoc.Add(docElement); string psConfigPath = DryadLinqCodeGen.GetPathForGeneratedFile("psConfig.xml", null); configDoc.Save(psConfigPath); return(psConfigPath); }
internal static DryadLinqRecordReader <TRecord> MakeDryadRecordReader <TRecord>(DryadLinqContext context, string readPath) { DryadLinqFactory <TRecord> factory = (DryadLinqFactory <TRecord>)DryadLinqCodeGen.GetFactory(context, typeof(TRecord)); NativeBlockStream nativeStream = ReflectionHelper.CreateDryadLinqFileStream(readPath, FileMode.Open, FileAccess.Read); // ??? NativeBlockStream nativeStream = ReflectionHelper.CreateDryadLinqFileStream(readPath, FileMode.Open, FileAccess.Read, DscCompressionScheme.None); DryadLinqRecordReader <TRecord> reader = factory.MakeReader(nativeStream); return(reader); }
internal TableEnumerator(DryadLinqContext context, DataProvider dataProvider, Uri dataSetUri) { this.m_current = default(T); this.m_factory = (DryadLinqFactory <T>)DryadLinqCodeGen.GetFactory(context, typeof(T)); Stream stream = dataProvider.Egress(context, dataSetUri); DryadLinqBlockStream nativeStream = new DryadLinqBlockStream(stream); this.m_reader = this.m_factory.MakeReader(nativeStream); this.m_reader.StartWorker(); }
private string GenerateConfig() { var psConfigPath = MakeProcessServiceConfig(); var configPath = DryadLinqCodeGen.GetPathForGeneratedFile("ppmConfig.xml", null); var configDoc = MakeConfig(psConfigPath, null); configDoc.Save(configPath); return(configPath); }
public override void Ingress <T>(DryadLinqContext context, IEnumerable <T> source, Uri dataSetUri, DryadLinqMetaData metaData, CompressionScheme compressionScheme, bool isTemp, Expression <Action <IEnumerable <T>, Stream> > serializer) { string fileName = dataSetUri.LocalPath; if (!String.IsNullOrEmpty(dataSetUri.Host)) { fileName = @"\\" + dataSetUri.Host + fileName; } // Write the partition: string partDir = Path.GetDirectoryName(fileName); partDir = Path.Combine(partDir, DryadLinqUtil.MakeUniqueName()); Directory.CreateDirectory(partDir); string uncPath = Path.Combine(partDir, "Part"); string partitionPath = uncPath + ".00000000"; DryadLinqFactory <T> factory = (DryadLinqFactory <T>)DryadLinqCodeGen.GetFactory(context, typeof(T)); using (FileStream fstream = new FileStream(partitionPath, FileMode.CreateNew, FileAccess.Write)) { if (serializer == null) { DryadLinqFileBlockStream nativeStream = new DryadLinqFileBlockStream(fstream, compressionScheme); DryadLinqRecordWriter <T> writer = factory.MakeWriter(nativeStream); foreach (T rec in source) { writer.WriteRecordSync(rec); } writer.Close(); } else { Action <IEnumerable <T>, Stream> serializerFunc = serializer.Compile(); serializerFunc(source, fstream); } } // Write the partfile: long partSize = new FileInfo(partitionPath).Length; using (StreamWriter writer = File.CreateText(fileName)) { writer.WriteLine(uncPath); writer.WriteLine("1"); writer.WriteLine("{0},{1}", 0, partSize); } }
private XDocument GenerateConfig() { string psConfigPath = MakeProcessServiceConfig(); // this will cause the psConfig to be uploaded to the DFS during MakeConfig AddLocalFile(psConfigPath); var configDoc = MakeConfig(psConfigPath); string configPath = DryadLinqCodeGen.GetPathForGeneratedFile("ppmConfig.xml", null); configDoc.Save(configPath); return(MakeLauncherConfig(configPath)); }
private string MakeProcessServiceConfig() { var configDoc = new XDocument(); var docElement = new XElement("PeloponneseConfig"); var psElement = new XElement("ProcessService"); var psPortElement = new XElement("Port"); psPortElement.Value = "8472"; psElement.Add(psPortElement); var psPrefixElement = new XElement("Prefix"); psPrefixElement.Value = "/peloponnese/dpservice/"; psElement.Add(psPrefixElement); var environment = new Dictionary <string, string>(); environment.Add("PATH", Environment.GetEnvironmentVariable("PATH") + ";" + Context.PeloponneseHomeDirectory); var jarPath = Path.Combine(Context.PeloponneseHomeDirectory, "Microsoft.Research.Peloponnese.HadoopBridge.jar"); environment.Add("PELOPONNESE_ADDITIONAL_CLASSPATH", jarPath); environment.Add("DRYAD_THREADS_PER_WORKER", Context.ThreadsPerWorker.ToString()); var envElement = new XElement("Environment"); foreach (var e in environment) { var varElement = new XElement("Variable"); varElement.SetAttributeValue("var", e.Key); varElement.Value = e.Value; envElement.Add(varElement); } psElement.Add(envElement); docElement.Add(psElement); configDoc.Add(docElement); string psConfigPath = DryadLinqCodeGen.GetPathForGeneratedFile("psConfig.xml", null); configDoc.Save(psConfigPath); return(psConfigPath); }
public override void Ingress <T>(DryadLinqContext context, IEnumerable <T> source, Uri dataSetUri, DryadLinqMetaData metaData, CompressionScheme compressionScheme, bool isTemp = false) { // Write the partition: string partDir = context.PartitionUncPath; if (partDir == null) { partDir = Path.GetDirectoryName(dataSetUri.LocalPath); } if (!Path.IsPathRooted(partDir)) { partDir = Path.Combine("/", partDir); } partDir = Path.Combine(partDir, DryadLinqUtil.MakeUniqueName()); Directory.CreateDirectory(partDir); string partPath = Path.Combine(partDir, "Part"); string partFilePath = partPath + ".00000000"; DryadLinqFactory <T> factory = (DryadLinqFactory <T>)DryadLinqCodeGen.GetFactory(context, typeof(T)); using (FileStream fstream = new FileStream(partFilePath, FileMode.CreateNew, FileAccess.Write)) { DryadLinqFileBlockStream nativeStream = new DryadLinqFileBlockStream(fstream, compressionScheme); DryadLinqRecordWriter <T> writer = factory.MakeWriter(nativeStream); foreach (T rec in source) { writer.WriteRecordSync(rec); } writer.Close(); } // Write the partfile: FileInfo finfo = new FileInfo(partFilePath); using (StreamWriter writer = File.CreateText(dataSetUri.LocalPath)) { writer.WriteLine(partPath); writer.WriteLine("1"); writer.WriteLine("{0},{1},{2}", 0, finfo.Length, Environment.MachineName); } }
private XDocument GenerateConfig() { XElement peloponneseResource = ConfigHelpers.MakePeloponneseResourceGroup( Context.Cluster.DfsClient, Context.PeloponneseHomeDirectory); string psConfigPath = MakeProcessServiceConfig(); // this will cause the psConfig to be uploaded to the DFS during MakeConfig AddLocalFile(psConfigPath); var configDoc = MakeConfig(psConfigPath, peloponneseResource); string configPath = DryadLinqCodeGen.GetPathForGeneratedFile("ppmConfig.xml", null); configDoc.Save(configPath); return(MakeLauncherConfig(configPath, peloponneseResource)); }
private string MakeProcessServiceConfig() { var configDoc = new XDocument(); var docElement = new XElement("PeloponneseConfig"); var psElement = new XElement("ProcessService"); var psPortElement = new XElement("Port"); psPortElement.Value = "8471"; psElement.Add(psPortElement); var psPrefixElement = new XElement("Prefix"); psPrefixElement.Value = "/peloponnese/dpservice/"; psElement.Add(psPrefixElement); var environment = new Dictionary <string, string>(); environment.Add(Constants.LoggingLevelEnvVar, Constants.LoggingStringFromLevel((int)Context.RuntimeLoggingLevel).ToString()); environment.Add("DRYAD_THREADS_PER_WORKER", Context.ThreadsPerWorker.ToString()); var envElement = new XElement("Environment"); foreach (var e in environment) { var varElement = new XElement("Variable"); varElement.SetAttributeValue("var", e.Key); varElement.Value = e.Value; envElement.Add(varElement); } psElement.Add(envElement); docElement.Add(psElement); configDoc.Add(docElement); string psConfigPath = DryadLinqCodeGen.GetPathForGeneratedFile("psConfig.xml", null); configDoc.Save(psConfigPath); return(psConfigPath); }
public override void Ingress <T>(DryadLinqContext context, IEnumerable <T> source, Uri dataSetUri, DryadLinqMetaData metaData, CompressionScheme outputScheme, bool isTemp, Expression <Action <IEnumerable <T>, Stream> > serializer) { DryadLinqFactory <T> factory = (DryadLinqFactory <T>)DryadLinqCodeGen.GetFactory(context, typeof(T)); using (Stream stream = context.GetHdfsClient.GetDfsStreamWriter(dataSetUri)) { DryadLinqBlockStream nativeStream = new DryadLinqBlockStream(stream); DryadLinqRecordWriter <T> writer = factory.MakeWriter(nativeStream); foreach (T rec in source) { writer.WriteRecordSync(rec); } writer.Close(); } }
private static bool GetDecompositionInfoList(ParameterExpression keyParam, ParameterExpression groupParam, MemberBinding mbinding, List <DecompositionInfo> infoList, DryadLinqCodeGen codeGen) { if (mbinding is MemberAssignment) { Expression expr = ((MemberAssignment)mbinding).Expression; return(GetDecompositionInfoList(keyParam, groupParam, expr, infoList, codeGen)); } else if (mbinding is MemberMemberBinding) { foreach (MemberBinding mb in ((MemberMemberBinding)mbinding).Bindings) { bool isDecomposed = GetDecompositionInfoList(keyParam, groupParam, mb, infoList, codeGen); if (!isDecomposed) { return(false); } } } else if (mbinding is MemberListBinding) { foreach (ElementInit ei in ((MemberListBinding)mbinding).Initializers) { foreach (Expression arg in ei.Arguments) { bool isDecomposed = GetDecompositionInfoList(keyParam, groupParam, arg, infoList, codeGen); if (!isDecomposed) { return(false); } } } } return(true); }
private static bool GetDecompositionInfoList(ParameterExpression keyParam, ParameterExpression groupParam, Expression expr, List <DecompositionInfo> infoList, DryadLinqCodeGen codeGen) { IEnumerable <Expression> argList = null; if (DryadLinqExpression.IsConstant(expr)) { return(true); } else if (expr is BinaryExpression) { BinaryExpression be = (BinaryExpression)expr; argList = new Expression[] { be.Left, be.Right }; } else if (expr is UnaryExpression) { UnaryExpression ue = (UnaryExpression)expr; return(GetDecompositionInfoList(keyParam, groupParam, ue.Operand, infoList, codeGen)); } else if (expr is ConditionalExpression) { ConditionalExpression ce = (ConditionalExpression)expr; argList = new Expression[] { ce.Test, ce.IfTrue, ce.IfFalse }; } else if (expr is MethodCallExpression) { MethodCallExpression mcExpr = (MethodCallExpression)expr; DecompositionInfo dinfo = GetDecompositionInfo(groupParam, mcExpr, codeGen); if (dinfo != null) { infoList.Add(dinfo); return(true); } if (mcExpr.Object != null) { bool isDecomposed = GetDecompositionInfoList(keyParam, groupParam, mcExpr.Object, infoList, codeGen); if (!isDecomposed) { return(false); } } argList = mcExpr.Arguments; } else if (expr is NewExpression) { argList = ((NewExpression)expr).Arguments; } else if (expr is NewArrayExpression) { argList = ((NewArrayExpression)expr).Expressions; } else if (expr is ListInitExpression) { ListInitExpression li = (ListInitExpression)expr; bool isDecomposed = GetDecompositionInfoList(keyParam, groupParam, li.NewExpression, infoList, codeGen); for (int i = 0, n = li.Initializers.Count; i < n; i++) { ElementInit ei = li.Initializers[i]; foreach (Expression arg in ei.Arguments) { isDecomposed = GetDecompositionInfoList(keyParam, groupParam, arg, infoList, codeGen); if (!isDecomposed) { return(false); } } } return(true); } else if (expr is MemberInitExpression) { MemberInitExpression mi = (MemberInitExpression)expr; bool isDecomposed = GetDecompositionInfoList(keyParam, groupParam, mi.NewExpression, infoList, codeGen); if (!isDecomposed) { return(false); } foreach (MemberBinding mb in mi.Bindings) { isDecomposed = GetDecompositionInfoList(keyParam, groupParam, mb, infoList, codeGen); if (!isDecomposed) { return(false); } } return(true); } else if (keyParam == null) { while (expr is MemberExpression) { MemberExpression me = (MemberExpression)expr; if (me.Expression == groupParam && me.Member.Name == "Key") { return(true); } expr = me.Expression; } return(false); } else { while (expr is MemberExpression) { expr = ((MemberExpression)expr).Expression; } return(expr == keyParam); } foreach (var argExpr in argList) { bool isDecomposed = GetDecompositionInfoList(keyParam, groupParam, argExpr, infoList, codeGen); if (!isDecomposed) { return(false); } } return(true); }
static DryadLinqClientLog() { string path = DryadLinqCodeGen.GetPathForGeneratedFile(CLIENT_LOG_FILENAME, null); s_writer = new StreamWriter(path); }
internal static string GetClientSideObjectStorePath() { return(DryadLinqCodeGen.GetPathForGeneratedFile(ObjectStoreFileName, null)); }
private static DecompositionInfo GetDecompositionInfo(ParameterExpression groupParam, MethodCallExpression mcExpr, DryadLinqCodeGen codeGen) { if (mcExpr.Arguments.Count == 0 || mcExpr.Arguments[0] != groupParam) { return(null); } for (int i = 1; i < mcExpr.Arguments.Count; i++) { if (DryadLinqExpression.Contains(groupParam, mcExpr.Arguments[i])) { return(null); } } ExpressionSimplifier <object> evaluator = new ExpressionSimplifier <object>(); Type[] paramTypeArgs = groupParam.Type.GetGenericArguments(); Type sourceElemType = paramTypeArgs[paramTypeArgs.Length - 1]; Type resultType = mcExpr.Type; Type decomposerType = null; DecomposableAttribute decomposableAttrib = AttributeSystem.GetDecomposableAttrib(mcExpr); if (decomposableAttrib != null) { decomposerType = decomposableAttrib.DecompositionType; } else { MethodInfo mInfo = mcExpr.Method; if (mInfo.DeclaringType == typeof(System.Linq.Enumerable) || mInfo.DeclaringType == typeof(System.Linq.Queryable)) { // For built-in decomposable operators. switch (mInfo.Name) { case "Count": case "LongCount": { Type outputType; Expression body; if (mInfo.Name == "Count") { outputType = typeof(Int32); body = Expression.Constant(1, outputType); } else { outputType = typeof(Int64); body = Expression.Constant((long)1, outputType); } ParameterExpression param1 = Expression.Parameter(outputType, "a"); ParameterExpression param2 = Expression.Parameter(sourceElemType, "e"); LambdaExpression seedExpr = Expression.Lambda(body, param2); body = Expression.AddChecked(param1, body); LambdaExpression accumulateExpr = Expression.Lambda(body, param1, param2); param2 = Expression.Parameter(outputType, "b"); body = Expression.AddChecked(param1, param2); LambdaExpression recursiveAccumulateExpr = Expression.Lambda(body, param1, param2); return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null)); } case "Any": { ParameterExpression param1 = Expression.Parameter(typeof(bool), "a"); ParameterExpression param2; Expression body; if (mcExpr.Arguments.Count == 1) { param2 = Expression.Parameter(sourceElemType, "e"); body = Expression.Constant(true, typeof(bool)); } else { LambdaExpression predExpr = DryadLinqExpression.GetLambda(mcExpr.Arguments[1]); param2 = predExpr.Parameters[0]; body = predExpr.Body; } LambdaExpression seedExpr = Expression.Lambda(body, param2); LambdaExpression accumulateExpr = Expression.Lambda(Expression.Or(param1, body), param1, param2); param2 = Expression.Parameter(typeof(bool), "b"); body = Expression.Or(param1, param2); LambdaExpression recursiveAccumulateExpr = Expression.Lambda(body, param1, param2); return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null)); } case "All": { ParameterExpression param1 = Expression.Parameter(typeof(bool), "a"); LambdaExpression predExpr = DryadLinqExpression.GetLambda(mcExpr.Arguments[1]); ParameterExpression param2 = predExpr.Parameters[0]; Expression body = predExpr.Body; LambdaExpression seedExpr = Expression.Lambda(body, param2); LambdaExpression accumulateExpr = Expression.Lambda(Expression.And(param1, body), param1, param2); param2 = Expression.Parameter(typeof(bool), "b"); body = Expression.And(param1, param2); LambdaExpression recursiveAccumulateExpr = Expression.Lambda(body, param1, param2); return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null)); } case "First": { ParameterExpression param1 = Expression.Parameter(sourceElemType, "a"); ParameterExpression param2 = Expression.Parameter(sourceElemType, "e"); LambdaExpression seedExpr = Expression.Lambda(param2, param2); LambdaExpression accumulateExpr = Expression.Lambda(param1, param1, param2); LambdaExpression recursiveAccumulateExpr = accumulateExpr; return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null)); } case "Last": { ParameterExpression param1 = Expression.Parameter(sourceElemType, "a"); ParameterExpression param2 = Expression.Parameter(sourceElemType, "e"); LambdaExpression seedExpr = Expression.Lambda(param2, param2); LambdaExpression accumulateExpr = Expression.Lambda(param2, param1, param2); LambdaExpression recursiveAccumulateExpr = accumulateExpr; return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null)); } case "Sum": { ParameterExpression param1; ParameterExpression param2; Expression arg2; if (mInfo.GetParameters().Length == 1) { param2 = Expression.Parameter(sourceElemType, "e"); arg2 = param2; } else { LambdaExpression selectExpr = DryadLinqExpression.GetLambda(mcExpr.Arguments[1]); param2 = selectExpr.Parameters[0]; arg2 = selectExpr.Body; } Expression abody, sbody; if (arg2.Type.IsGenericType) { param1 = Expression.Parameter(arg2.Type.GetGenericArguments()[0], "a"); MethodInfo accumulateInfo = typeof(DryadLinqVertex).GetMethod( "SumAccumulate", new Type[] { param1.Type, arg2.Type }); sbody = Expression.Constant(0, param1.Type); sbody = Expression.Call(accumulateInfo, sbody, arg2); abody = Expression.Call(accumulateInfo, param1, arg2); } else { param1 = Expression.Parameter(arg2.Type, "a"); sbody = arg2; abody = Expression.AddChecked(param1, arg2); } LambdaExpression seedExpr = Expression.Lambda(sbody, param2); LambdaExpression accumulateExpr = Expression.Lambda(abody, param1, param2); param2 = Expression.Parameter(param1.Type, "b"); Expression rbody = Expression.AddChecked(param1, param2); LambdaExpression recursiveAccumulateExpr = Expression.Lambda(rbody, param1, param2); Expression fbody = Expression.Convert(param1, arg2.Type); LambdaExpression finalReduceExpr = Expression.Lambda(fbody, param1); return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, finalReduceExpr)); } case "Max": case "Min": { ParameterExpression param2; Expression abody; if (mInfo.GetParameters().Length == 1) { param2 = Expression.Parameter(sourceElemType, "e"); abody = param2; } else { LambdaExpression selectExpr = DryadLinqExpression.GetLambda(mcExpr.Arguments[1]); param2 = selectExpr.Parameters[0]; abody = selectExpr.Body; } ParameterExpression param1 = Expression.Parameter(abody.Type, "a"); Expression sbody = abody; MethodInfo accumulateInfo; string methodName = (mInfo.Name == "Max") ? "MaxAccumulate" : "MinAccumulate"; if (mInfo.IsGenericMethod && (mInfo.GetParameters().Length == 1)) { accumulateInfo = typeof(DryadLinqVertex).GetMethod(methodName + "Generic"); accumulateInfo = accumulateInfo.MakeGenericMethod(sourceElemType); } else { accumulateInfo = typeof(DryadLinqVertex).GetMethod( methodName, new Type[] { param1.Type, abody.Type }); } abody = Expression.Call(accumulateInfo, param1, abody); LambdaExpression seedExpr = Expression.Lambda(sbody, param2); LambdaExpression accumulateExpr = Expression.Lambda(abody, param1, param2); param2 = Expression.Parameter(param1.Type, "b"); Expression rbody = Expression.Call(accumulateInfo, param1, param2); LambdaExpression recursiveAccumulateExpr = Expression.Lambda(rbody, param1, param2); return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null)); } case "Aggregate": { ParameterExpression elemParam = Expression.Parameter(sourceElemType, "e"); LambdaExpression accumulateExpr; LambdaExpression seedExpr; if (mcExpr.Arguments.Count == 2) { accumulateExpr = DryadLinqExpression.GetLambda(mcExpr.Arguments[1]); seedExpr = Expression.Lambda(elemParam, elemParam); } else { accumulateExpr = DryadLinqExpression.GetLambda(mcExpr.Arguments[2]); object seedVal = evaluator.Eval(mcExpr.Arguments[1]); Expression body = Expression.Constant(seedVal, seedVal.GetType()); ParameterSubst subst = new ParameterSubst(accumulateExpr.Parameters[0], body); body = subst.Visit(accumulateExpr.Body); seedExpr = Expression.Lambda(body, accumulateExpr.Parameters[1]); } if (!DryadLinqExpression.IsAssociative(accumulateExpr)) { return(null); } LambdaExpression recursiveAccumulateExpr = DryadLinqExpression.GetAssociativeCombiner(accumulateExpr); return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null)); } case "Average": { ParameterExpression param2; Expression abody; if (mInfo.GetParameters().Length == 1) { param2 = Expression.Parameter(sourceElemType, "e"); abody = param2; } else { LambdaExpression selectExpr = DryadLinqExpression.GetLambda(mcExpr.Arguments[1]); param2 = selectExpr.Parameters[0]; abody = selectExpr.Body; } Type aggValueType = abody.Type; if (aggValueType == typeof(int) || aggValueType == typeof(int?)) { aggValueType = typeof(long); } else if (aggValueType == typeof(long?)) { aggValueType = typeof(long); } else if (aggValueType == typeof(float) || aggValueType == typeof(float?)) { aggValueType = typeof(double); } else if (aggValueType == typeof(double?)) { aggValueType = typeof(double); } else if (aggValueType == typeof(decimal?)) { aggValueType = typeof(decimal); } Type sumAndCountType = typeof(AggregateValue <>).MakeGenericType(aggValueType); ParameterExpression param1 = Expression.Parameter(sumAndCountType, "a"); MethodInfo accumulateInfo = typeof(DryadLinqVertex).GetMethod( "AverageAccumulate", new Type[] { sumAndCountType, abody.Type }); // Seed: Expression sbody = Expression.New(sumAndCountType); sbody = Expression.Call(accumulateInfo, sbody, abody); LambdaExpression seedExpr = Expression.Lambda(sbody, param2); // Accumulate: abody = Expression.Call(accumulateInfo, param1, abody); LambdaExpression accumulateExpr = Expression.Lambda(abody, param1, param2); // RecursiveAccumulate: param2 = Expression.Parameter(param1.Type, "b"); PropertyInfo valueInfo = sumAndCountType.GetProperty("Value"); PropertyInfo countInfo = sumAndCountType.GetProperty("Count"); Expression sumExpr1 = Expression.Property(param1, valueInfo); Expression countExpr1 = Expression.Property(param1, countInfo); Expression sumExpr2 = Expression.Property(param2, valueInfo); Expression countExpr2 = Expression.Property(param2, countInfo); Expression sumExpr = Expression.AddChecked(sumExpr1, sumExpr2); Expression countExpr = Expression.AddChecked(countExpr1, countExpr2); ConstructorInfo cinfo = sumAndCountType.GetConstructor(new Type[] { sumExpr.Type, countExpr.Type }); Expression rbody = Expression.New(cinfo, sumExpr, countExpr); LambdaExpression recursiveAccumulateExpr = Expression.Lambda(rbody, param1, param2); // FinalReduce: if (sumExpr1.Type == typeof(long)) { sumExpr1 = Expression.Convert(sumExpr1, typeof(double)); } Expression fbody = Expression.Divide(sumExpr1, countExpr1); fbody = Expression.Convert(fbody, resultType); if (resultType.IsGenericType) { Expression zeroExpr = Expression.Constant(0, typeof(long)); Expression condExpr = Expression.GreaterThan(countExpr1, zeroExpr); Expression nullExpr = Expression.Constant(null, resultType); fbody = Expression.Condition(condExpr, fbody, nullExpr); } LambdaExpression finalReduceExpr = Expression.Lambda(fbody, param1); return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, finalReduceExpr)); } case "Contains": { decomposerType = typeof(ContainsDecomposition <>).MakeGenericType(sourceElemType); break; } case "Distinct": { decomposerType = typeof(DistinctDecomposition <>).MakeGenericType(sourceElemType); break; } default: { return(null); } } } } if (decomposerType == null) { return(null); } Type implementedInterface = null; Type[] interfaces = decomposerType.GetInterfaces(); foreach (Type intf in interfaces) { if (intf.GetGenericTypeDefinition() == typeof(IDecomposable <, ,>)) { if (implementedInterface != null) { throw new DryadLinqException("Decomposition class can implement only one decomposable interface."); } implementedInterface = intf; } } if (implementedInterface == null || implementedInterface.GetGenericArguments().Length != 3) { throw new DryadLinqException("Decomposition class " + decomposerType.FullName + "must implement IDecomposable<,,>"); } // The second type of the implemented interface definition is the accumulatorType. Type accumulatorType = implementedInterface.GetGenericArguments()[1]; // Now check that all the types match up. Type decomposerInterface = typeof(IDecomposable <, ,>).MakeGenericType( sourceElemType, accumulatorType, resultType); if (!decomposerInterface.IsAssignableFrom(decomposerType)) { throw new DryadLinqException("Decomposition class must match the function that it decorates."); } if (decomposerType.ContainsGenericParameters) { if (decomposerType.GetGenericArguments().Length != 1 || !decomposerType.GetGenericArguments()[0].IsGenericParameter) { throw new DryadLinqException(decomposerType.Name + " must match the function it annotates."); } decomposerType = decomposerType.MakeGenericType(sourceElemType); } if (decomposerType.GetConstructor(Type.EmptyTypes) == null) { throw new DryadLinqException("Decomposition class must have a default constructor."); } // Add to the codegen a call of the static Initializer of decomposerType Expression[] args = new Expression[mcExpr.Arguments.Count - 1]; for (int i = 0; i < args.Length; i++) { args[i] = Expression.Convert(mcExpr.Arguments[i + 1], typeof(object)); } Expression stateExpr = Expression.NewArrayInit(typeof(object), args); string decomposerName = codeGen.AddDecompositionInitializer(decomposerType, stateExpr); ParameterExpression decomposer = Expression.Parameter(decomposerType, decomposerName); // Seed: TSource => TAccumulate MethodInfo seedInfo1 = decomposerType.GetMethod("Seed"); ParameterExpression p2 = Expression.Parameter(sourceElemType, "e"); Expression sbody1 = Expression.Call(decomposer, seedInfo1, p2); LambdaExpression seedExpr1 = Expression.Lambda(sbody1, p2); // Accumulate: (TAccumulate, TSource) => TAccumulate MethodInfo accumulateInfo1 = decomposerType.GetMethod("Accumulate"); ParameterExpression p1 = Expression.Parameter(accumulatorType, "a"); Expression abody1 = Expression.Call(decomposer, accumulateInfo1, p1, p2); LambdaExpression accumulateExpr1 = Expression.Lambda(abody1, p1, p2); // RecursiveAccumulate: (TAccumulate, TAccumulate) => TAccumulate MethodInfo recursiveAccumulateInfo1 = decomposerType.GetMethod("RecursiveAccumulate"); p2 = Expression.Parameter(accumulatorType, "e"); Expression rbody1 = Expression.Call(decomposer, recursiveAccumulateInfo1, p1, p2); LambdaExpression recursiveAccumulateExpr1 = Expression.Lambda(rbody1, p1, p2); // FinalReduce: TAccumulate => TResult MethodInfo finalReduceInfo1 = decomposerType.GetMethod("FinalReduce"); Expression fbody1 = Expression.Call(decomposer, finalReduceInfo1, p1); LambdaExpression finalReduceExpr1 = Expression.Lambda(fbody1, p1); return(new DecompositionInfo(mcExpr, seedExpr1, accumulateExpr1, recursiveAccumulateExpr1, finalReduceExpr1)); }