// This constructor is specifically to support enumeration of a query. // It assumes that the Expressions all terminate with a ToStore node. internal DryadLinqQueryGen(DryadLinqContext context, VertexCodeGen vertexCodeGen, Expression queryExpr, Uri tableUri, bool isTempOutput) { this.m_queryExprs = new Expression[] { queryExpr }; this.m_outputTableUris = new Uri[] { tableUri }; this.m_isTempOutput = new bool[] { isTempOutput }; this.m_context = context; this.Initialize(vertexCodeGen); }
internal HpcLinqQueryGen(HpcLinqContext context, VertexCodeGen vertexCodeGen, Expression queryExpr, string tableUri, bool isTempOutput) { this.m_queryExprs = new Expression[] { queryExpr }; string fullTableUri = tableUri; this.m_outputTableUris = new string[] { fullTableUri }; this.m_isTempOutput = new bool[] { isTempOutput }; this.m_context = context; this.Initialize(vertexCodeGen); }
private void Initialize(VertexCodeGen vertexCodeGen) { this.m_codeGen = new HpcLinqCodeGen(this.m_context, vertexCodeGen); this.m_queryPlan1 = null; this.m_queryPlan2 = null; this.m_queryPlan3 = null; this.m_DryadLinqProgram = null; this.m_queryPlan1 = null; this.m_exprNodeInfoMap = new Dictionary<Expression, QueryNodeInfo>(); this.m_referencedQueryMap = new Dictionary<Expression, QueryNodeInfo>(); this.m_inputUriMap = new Dictionary<string, DryadInputNode>(); this.m_outputUriMap = new Dictionary<string, DryadOutputNode>(); this.queryExecutor = new JobExecutor(this.m_context); // Initialize the data structures for the output tables this.m_outputTypes = new Type[this.m_queryExprs.Length]; this.m_outputDatapaths = new string[this.m_queryExprs.Length]; this.m_queryNodeInfos = new QueryNodeInfo[this.m_queryExprs.Length]; for (int i = 0; i < this.m_queryExprs.Length; i++) { this.m_queryNodeInfos[i] = this.BuildNodeInfoGraph(this.m_queryExprs[i]); this.m_queryNodeInfos[i] = new QueryNodeInfo(this.m_queryExprs[i], false, this.m_queryNodeInfos[i]); this.m_outputDatapaths[i] = DataPath.GetDataPath(this.m_outputTableUris[i]); Dictionary<string, string> args = DataPath.GetArguments(this.m_outputTableUris[i]); if (!(DataPath.IsDsc(this.m_outputDatapaths[i]) || DataPath.IsHdfs(this.m_outputDatapaths[i]))) { throw new DryadLinqException(HpcLinqErrorCode.UnrecognizedDataSource, String.Format(SR.UnrecognizedDataSource, this.m_outputTableUris[i])); } } }
// This constructor is specifically to support Materialize() calls. // it assumes that the Expressions all terminate with a ToDsc node. internal HpcLinqQueryGen(HpcLinqContext context, VertexCodeGen vertexCodeGen, Expression[] qlist) { this.m_queryExprs = new Expression[qlist.Length]; this.m_outputTableUris = new string[qlist.Length]; this.m_isTempOutput = new bool[qlist.Length]; this.m_context = context; for (int i = 0; i < this.m_queryExprs.Length; i++) { MethodCallExpression mcExpr = (MethodCallExpression)qlist[i]; string tableUri; this.m_queryExprs[i] = mcExpr.Arguments[0]; //this block supports the scenario: q-nonToDsc if (mcExpr.Method.Name == ReflectedNames.DryadLinqIQueryable_AnonymousDscPlaceholder) { ExpressionSimplifier<string> e1 = new ExpressionSimplifier<string>(); tableUri = e1.Eval(mcExpr.Arguments[1]); this.m_isTempOutput[i] = true; } //this block supports the scenario: q.ToDsc() else if (mcExpr.Method.Name == ReflectedNames.DryadLinqIQueryable_ToDscWorker) { DscService dsc = context.DscService; ExpressionSimplifier<string> e2 = new ExpressionSimplifier<string>(); string streamName = e2.Eval(mcExpr.Arguments[2]); tableUri = DataPath.MakeDscStreamUri(dsc, streamName); this.m_isTempOutput[i] = false; } //this block supports the scenario: q.ToHdfs() else if (mcExpr.Method.Name == ReflectedNames.DryadLinqIQueryable_ToHdfsWorker) { string hdfsHeadNode = context.HdfsService; ExpressionSimplifier<string> e2 = new ExpressionSimplifier<string>(); string streamName = e2.Eval(mcExpr.Arguments[2]); tableUri = DataPath.MakeHdfsStreamUri(hdfsHeadNode, streamName); this.m_isTempOutput[i] = false; } else { throw new InvalidOperationException(); // should not occur. } this.m_outputTableUris[i] = tableUri; } this.Initialize(vertexCodeGen); }
internal DryadLinqCodeGen(DryadLinqContext context, VertexCodeGen vertexCodeGen) { this.m_context = context; this.m_vertexCodeGen = vertexCodeGen; this.m_loadedVertexAssembly = null; this.m_dryadLinqUnit = new CodeCompileUnit(); // Create a namespace this.m_dryadCodeSpace = new CodeNamespace(TargetNamespace); this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System")); this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Collections")); this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Collections.Generic")); this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Text")); this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Linq")); this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Linq.Expressions")); this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Diagnostics")); this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Runtime.Serialization")); this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Data.SqlTypes")); this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Data.Linq")); this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Data.Linq.Mapping")); this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("Microsoft.Research.DryadLinq")); this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("Microsoft.Research.DryadLinq.Internal")); this.m_dryadLinqUnit.Namespaces.Add(this.m_dryadCodeSpace); // Create the class for all the DryadLinq extension methods this.m_dryadExtensionClass = new CodeTypeDeclaration(ExtensionClassName); this.m_dryadExtensionClass.IsClass = true; this.m_dryadExtensionClass.IsPartial = true; this.m_dryadExtensionClass.TypeAttributes = TypeAttributes.Public; this.m_dryadCodeSpace.Types.Add(this.m_dryadExtensionClass); // Create the static constructor for the vertex extension class this.m_extensionStaticCtor = new CodeTypeConstructor(); this.m_dryadExtensionClass.Members.Add(this.m_extensionStaticCtor); // Create the class for all the DryadLinq vertex methods this.m_dryadVertexClass = new CodeTypeDeclaration(VertexClassName); this.m_dryadVertexClass.IsClass = true; this.m_dryadVertexClass.TypeAttributes = TypeAttributes.Public | TypeAttributes.Sealed; this.m_dryadCodeSpace.Types.Add(this.m_dryadVertexClass); this.AddCopyResourcesMethod(); // The set of input/output channel datatypes this.m_dryadDataTypes = new HashSet<Type>(); this.m_dryadDataTypes.Add(typeof(byte)); this.m_dryadDataTypes.Add(typeof(sbyte)); this.m_dryadDataTypes.Add(typeof(bool)); this.m_dryadDataTypes.Add(typeof(char)); this.m_dryadDataTypes.Add(typeof(short)); this.m_dryadDataTypes.Add(typeof(ushort)); this.m_dryadDataTypes.Add(typeof(int)); this.m_dryadDataTypes.Add(typeof(uint)); this.m_dryadDataTypes.Add(typeof(long)); this.m_dryadDataTypes.Add(typeof(ulong)); this.m_dryadDataTypes.Add(typeof(float)); this.m_dryadDataTypes.Add(typeof(decimal)); this.m_dryadDataTypes.Add(typeof(double)); this.m_dryadDataTypes.Add(typeof(DateTime)); this.m_dryadDataTypes.Add(typeof(string)); this.m_dryadDataTypes.Add(typeof(LineRecord)); this.m_dryadDataTypes.Add(typeof(SqlDateTime)); this.m_dryadDataTypes.Add(typeof(Guid)); // The set of datatypes we have added serialization methods this.m_serializationDatatypes = new HashSet<Type>(); this.m_fieldToStaticName = new Dictionary<FieldInfo, string>(); this.m_staticFieldDefined = new HashSet<string>(); this.m_typeToSerializerName = new Dictionary<Type, string>(); this.m_anonymousTypeToName = new Dictionary<Type, string>(); this.m_nameToAlias = new Dictionary<string, string>(); }
// This constructor is specifically to support SubmitAndWait() calls. // It assumes that the Expressions all terminate with a ToStore node. internal DryadLinqQueryGen(DryadLinqContext context, VertexCodeGen vertexCodeGen, Expression[] qlist) { this.m_queryExprs = new Expression[qlist.Length]; this.m_outputTableUris = new Uri[qlist.Length]; this.m_serializers = new Expression[qlist.Length]; this.m_deserializers = new Expression[qlist.Length]; this.m_isTempOutput = new bool[qlist.Length]; this.m_context = context; for (int i = 0; i < this.m_queryExprs.Length; i++) { MethodCallExpression mcExpr = qlist[i] as MethodCallExpression; if (mcExpr != null && mcExpr.Method.Name == ReflectedNames.DLQ_ToStoreInternal) { this.m_queryExprs[i] = mcExpr.Arguments[0]; ExpressionSimplifier<Uri> e2 = new ExpressionSimplifier<Uri>(); this.m_outputTableUris[i] = e2.Eval(mcExpr.Arguments[1]); ExpressionSimplifier<bool> e3 = new ExpressionSimplifier<bool>(); this.m_isTempOutput[i] = e3.Eval(mcExpr.Arguments[2]); this.m_serializers[i] = mcExpr.Arguments[3]; this.m_deserializers[i] = mcExpr.Arguments[4]; } else if (mcExpr != null && mcExpr.Method.Name == ReflectedNames.DLQ_ToStoreInternalAux) { this.m_queryExprs[i] = mcExpr.Arguments[0]; ExpressionSimplifier<Uri> e2 = new ExpressionSimplifier<Uri>(); this.m_outputTableUris[i] = e2.Eval(mcExpr.Arguments[1]); ExpressionSimplifier<bool> e3 = new ExpressionSimplifier<bool>(); this.m_isTempOutput[i] = e3.Eval(mcExpr.Arguments[2]); } else { throw new DryadLinqException("Internal error: The method must be " + ReflectedNames.DLQ_ToStoreInternal); } } this.Initialize(vertexCodeGen); }
private void Initialize(VertexCodeGen vertexCodeGen) { this.m_codeGen = new DryadLinqCodeGen(this.m_context, vertexCodeGen); this.m_queryPlan1 = null; this.m_queryPlan2 = null; this.m_queryPlan3 = null; this.m_DryadLinqProgram = null; this.m_queryPlan1 = null; this.m_exprNodeInfoMap = new Dictionary<Expression, QueryNodeInfo>(); this.m_referencedQueryMap = new Dictionary<Expression, QueryNodeInfo>(); this.m_inputUriMap = new Dictionary<string, DLinqInputNode>(); this.m_outputUriMap = new Dictionary<string, DLinqOutputNode>(); this.m_queryExecutor = new DryadLinqJobExecutor(this.m_context); // Initialize the data structures for the output tables this.m_outputTypes = new Type[this.m_queryExprs.Length]; this.m_queryNodeInfos = new QueryNodeInfo[this.m_queryExprs.Length]; for (int i = 0; i < this.m_queryExprs.Length; i++) { this.m_queryNodeInfos[i] = this.BuildNodeInfoGraph(this.m_queryExprs[i]); this.m_queryNodeInfos[i] = new DummyQueryNodeInfo(this.m_queryExprs[i], false, this.m_queryNodeInfos[i]); if (!DataPath.IsValidDataPath(this.m_outputTableUris[i])) { throw new DryadLinqException(DryadLinqErrorCode.UnrecognizedDataSource, String.Format(SR.UnrecognizedDataSource, this.m_outputTableUris[i].AbsoluteUri)); } } }