// This constructor is specifically to support enumeration of a query.
// It assumes that the Expressions all terminate with a ToStore node.
internal DryadLinqQueryGen(DryadLinqContext context,
VertexCodeGen vertexCodeGen,
Expression queryExpr,
Uri tableUri,
bool isTempOutput)
{
this.m_queryExprs = new Expression[] { queryExpr };
this.m_outputTableUris = new Uri[] { tableUri };
this.m_isTempOutput = new bool[] { isTempOutput };
this.m_context = context;
this.Initialize(vertexCodeGen);
}
internal HpcLinqQueryGen(HpcLinqContext context,
VertexCodeGen vertexCodeGen,
Expression queryExpr,
string tableUri,
bool isTempOutput)
{
this.m_queryExprs = new Expression[] { queryExpr };
string fullTableUri = tableUri;
this.m_outputTableUris = new string[] { fullTableUri };
this.m_isTempOutput = new bool[] { isTempOutput };
this.m_context = context;
this.Initialize(vertexCodeGen);
}
private void Initialize(VertexCodeGen vertexCodeGen)
{
this.m_codeGen = new HpcLinqCodeGen(this.m_context, vertexCodeGen);
this.m_queryPlan1 = null;
this.m_queryPlan2 = null;
this.m_queryPlan3 = null;
this.m_DryadLinqProgram = null;
this.m_queryPlan1 = null;
this.m_exprNodeInfoMap = new Dictionary<Expression, QueryNodeInfo>();
this.m_referencedQueryMap = new Dictionary<Expression, QueryNodeInfo>();
this.m_inputUriMap = new Dictionary<string, DryadInputNode>();
this.m_outputUriMap = new Dictionary<string, DryadOutputNode>();
this.queryExecutor = new JobExecutor(this.m_context);
// Initialize the data structures for the output tables
this.m_outputTypes = new Type[this.m_queryExprs.Length];
this.m_outputDatapaths = new string[this.m_queryExprs.Length];
this.m_queryNodeInfos = new QueryNodeInfo[this.m_queryExprs.Length];
for (int i = 0; i < this.m_queryExprs.Length; i++)
{
this.m_queryNodeInfos[i] = this.BuildNodeInfoGraph(this.m_queryExprs[i]);
this.m_queryNodeInfos[i] = new QueryNodeInfo(this.m_queryExprs[i], false, this.m_queryNodeInfos[i]);
this.m_outputDatapaths[i] = DataPath.GetDataPath(this.m_outputTableUris[i]);
Dictionary<string, string> args = DataPath.GetArguments(this.m_outputTableUris[i]);
if (!(DataPath.IsDsc(this.m_outputDatapaths[i]) || DataPath.IsHdfs(this.m_outputDatapaths[i])))
{
throw new DryadLinqException(HpcLinqErrorCode.UnrecognizedDataSource,
String.Format(SR.UnrecognizedDataSource, this.m_outputTableUris[i]));
}
}
}
// This constructor is specifically to support Materialize() calls.
// it assumes that the Expressions all terminate with a ToDsc node.
internal HpcLinqQueryGen(HpcLinqContext context,
VertexCodeGen vertexCodeGen,
Expression[] qlist)
{
this.m_queryExprs = new Expression[qlist.Length];
this.m_outputTableUris = new string[qlist.Length];
this.m_isTempOutput = new bool[qlist.Length];
this.m_context = context;
for (int i = 0; i < this.m_queryExprs.Length; i++)
{
MethodCallExpression mcExpr = (MethodCallExpression)qlist[i];
string tableUri;
this.m_queryExprs[i] = mcExpr.Arguments[0];
//this block supports the scenario: q-nonToDsc
if (mcExpr.Method.Name == ReflectedNames.DryadLinqIQueryable_AnonymousDscPlaceholder)
{
ExpressionSimplifier<string> e1 = new ExpressionSimplifier<string>();
tableUri = e1.Eval(mcExpr.Arguments[1]);
this.m_isTempOutput[i] = true;
}
//this block supports the scenario: q.ToDsc()
else if (mcExpr.Method.Name == ReflectedNames.DryadLinqIQueryable_ToDscWorker)
{
DscService dsc = context.DscService;
ExpressionSimplifier<string> e2 = new ExpressionSimplifier<string>();
string streamName = e2.Eval(mcExpr.Arguments[2]);
tableUri = DataPath.MakeDscStreamUri(dsc, streamName);
this.m_isTempOutput[i] = false;
}
//this block supports the scenario: q.ToHdfs()
else if (mcExpr.Method.Name == ReflectedNames.DryadLinqIQueryable_ToHdfsWorker)
{
string hdfsHeadNode = context.HdfsService;
ExpressionSimplifier<string> e2 = new ExpressionSimplifier<string>();
string streamName = e2.Eval(mcExpr.Arguments[2]);
tableUri = DataPath.MakeHdfsStreamUri(hdfsHeadNode, streamName);
this.m_isTempOutput[i] = false;
}
else {
throw new InvalidOperationException(); // should not occur.
}
this.m_outputTableUris[i] = tableUri;
}
this.Initialize(vertexCodeGen);
}
internal DryadLinqCodeGen(DryadLinqContext context, VertexCodeGen vertexCodeGen)
{
this.m_context = context;
this.m_vertexCodeGen = vertexCodeGen;
this.m_loadedVertexAssembly = null;
this.m_dryadLinqUnit = new CodeCompileUnit();
// Create a namespace
this.m_dryadCodeSpace = new CodeNamespace(TargetNamespace);
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Collections"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Collections.Generic"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Text"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Linq"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Linq.Expressions"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Diagnostics"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Runtime.Serialization"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Data.SqlTypes"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Data.Linq"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Data.Linq.Mapping"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("Microsoft.Research.DryadLinq"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("Microsoft.Research.DryadLinq.Internal"));
this.m_dryadLinqUnit.Namespaces.Add(this.m_dryadCodeSpace);
// Create the class for all the DryadLinq extension methods
this.m_dryadExtensionClass = new CodeTypeDeclaration(ExtensionClassName);
this.m_dryadExtensionClass.IsClass = true;
this.m_dryadExtensionClass.IsPartial = true;
this.m_dryadExtensionClass.TypeAttributes = TypeAttributes.Public;
this.m_dryadCodeSpace.Types.Add(this.m_dryadExtensionClass);
// Create the static constructor for the vertex extension class
this.m_extensionStaticCtor = new CodeTypeConstructor();
this.m_dryadExtensionClass.Members.Add(this.m_extensionStaticCtor);
// Create the class for all the DryadLinq vertex methods
this.m_dryadVertexClass = new CodeTypeDeclaration(VertexClassName);
this.m_dryadVertexClass.IsClass = true;
this.m_dryadVertexClass.TypeAttributes = TypeAttributes.Public | TypeAttributes.Sealed;
this.m_dryadCodeSpace.Types.Add(this.m_dryadVertexClass);
this.AddCopyResourcesMethod();
// The set of input/output channel datatypes
this.m_dryadDataTypes = new HashSet<Type>();
this.m_dryadDataTypes.Add(typeof(byte));
this.m_dryadDataTypes.Add(typeof(sbyte));
this.m_dryadDataTypes.Add(typeof(bool));
this.m_dryadDataTypes.Add(typeof(char));
this.m_dryadDataTypes.Add(typeof(short));
this.m_dryadDataTypes.Add(typeof(ushort));
this.m_dryadDataTypes.Add(typeof(int));
this.m_dryadDataTypes.Add(typeof(uint));
this.m_dryadDataTypes.Add(typeof(long));
this.m_dryadDataTypes.Add(typeof(ulong));
this.m_dryadDataTypes.Add(typeof(float));
this.m_dryadDataTypes.Add(typeof(decimal));
this.m_dryadDataTypes.Add(typeof(double));
this.m_dryadDataTypes.Add(typeof(DateTime));
this.m_dryadDataTypes.Add(typeof(string));
this.m_dryadDataTypes.Add(typeof(LineRecord));
this.m_dryadDataTypes.Add(typeof(SqlDateTime));
this.m_dryadDataTypes.Add(typeof(Guid));
// The set of datatypes we have added serialization methods
this.m_serializationDatatypes = new HashSet<Type>();
this.m_fieldToStaticName = new Dictionary<FieldInfo, string>();
this.m_staticFieldDefined = new HashSet<string>();
this.m_typeToSerializerName = new Dictionary<Type, string>();
this.m_anonymousTypeToName = new Dictionary<Type, string>();
this.m_nameToAlias = new Dictionary<string, string>();
}
// This constructor is specifically to support SubmitAndWait() calls.
// It assumes that the Expressions all terminate with a ToStore node.
internal DryadLinqQueryGen(DryadLinqContext context,
VertexCodeGen vertexCodeGen,
Expression[] qlist)
{
this.m_queryExprs = new Expression[qlist.Length];
this.m_outputTableUris = new Uri[qlist.Length];
this.m_serializers = new Expression[qlist.Length];
this.m_deserializers = new Expression[qlist.Length];
this.m_isTempOutput = new bool[qlist.Length];
this.m_context = context;
for (int i = 0; i < this.m_queryExprs.Length; i++)
{
MethodCallExpression mcExpr = qlist[i] as MethodCallExpression;
if (mcExpr != null && mcExpr.Method.Name == ReflectedNames.DLQ_ToStoreInternal)
{
this.m_queryExprs[i] = mcExpr.Arguments[0];
ExpressionSimplifier<Uri> e2 = new ExpressionSimplifier<Uri>();
this.m_outputTableUris[i] = e2.Eval(mcExpr.Arguments[1]);
ExpressionSimplifier<bool> e3 = new ExpressionSimplifier<bool>();
this.m_isTempOutput[i] = e3.Eval(mcExpr.Arguments[2]);
this.m_serializers[i] = mcExpr.Arguments[3];
this.m_deserializers[i] = mcExpr.Arguments[4];
}
else if (mcExpr != null && mcExpr.Method.Name == ReflectedNames.DLQ_ToStoreInternalAux)
{
this.m_queryExprs[i] = mcExpr.Arguments[0];
ExpressionSimplifier<Uri> e2 = new ExpressionSimplifier<Uri>();
this.m_outputTableUris[i] = e2.Eval(mcExpr.Arguments[1]);
ExpressionSimplifier<bool> e3 = new ExpressionSimplifier<bool>();
this.m_isTempOutput[i] = e3.Eval(mcExpr.Arguments[2]);
}
else
{
throw new DryadLinqException("Internal error: The method must be " + ReflectedNames.DLQ_ToStoreInternal);
}
}
this.Initialize(vertexCodeGen);
}
private void Initialize(VertexCodeGen vertexCodeGen)
{
this.m_codeGen = new DryadLinqCodeGen(this.m_context, vertexCodeGen);
this.m_queryPlan1 = null;
this.m_queryPlan2 = null;
this.m_queryPlan3 = null;
this.m_DryadLinqProgram = null;
this.m_queryPlan1 = null;
this.m_exprNodeInfoMap = new Dictionary<Expression, QueryNodeInfo>();
this.m_referencedQueryMap = new Dictionary<Expression, QueryNodeInfo>();
this.m_inputUriMap = new Dictionary<string, DLinqInputNode>();
this.m_outputUriMap = new Dictionary<string, DLinqOutputNode>();
this.m_queryExecutor = new DryadLinqJobExecutor(this.m_context);
// Initialize the data structures for the output tables
this.m_outputTypes = new Type[this.m_queryExprs.Length];
this.m_queryNodeInfos = new QueryNodeInfo[this.m_queryExprs.Length];
for (int i = 0; i < this.m_queryExprs.Length; i++)
{
this.m_queryNodeInfos[i] = this.BuildNodeInfoGraph(this.m_queryExprs[i]);
this.m_queryNodeInfos[i] = new DummyQueryNodeInfo(this.m_queryExprs[i], false, this.m_queryNodeInfos[i]);
if (!DataPath.IsValidDataPath(this.m_outputTableUris[i]))
{
throw new DryadLinqException(DryadLinqErrorCode.UnrecognizedDataSource,
String.Format(SR.UnrecognizedDataSource,
this.m_outputTableUris[i].AbsoluteUri));
}
}
}
