internal HpcJobSubmission(HpcLinqContext context)
{
this.m_context = context;
this.m_status = JobStatus.NotSubmitted;
//@@TODO[P0] pass the runtime to the DryadJobSubmission so that it can use the scheduler instance.
//@@TODO: Merge DryadJobSubmission into Ms.Hpc.Linq. Until then make sure Context is not disposed before DryadJobSubmission.
this.m_job = new DryadJobSubmission(m_context.GetIScheduler());
}
internal HpcLinqQueryGen(HpcLinqContext context,
VertexCodeGen vertexCodeGen,
Expression queryExpr,
string tableUri,
bool isTempOutput)
{
this.m_queryExprs = new Expression[] { queryExpr };
string fullTableUri = tableUri;
this.m_outputTableUris = new string[] { fullTableUri };
this.m_isTempOutput = new bool[] { isTempOutput };
this.m_context = context;
this.Initialize(vertexCodeGen);
}
// This constructor is specifically to support Materialize() calls.
// it assumes that the Expressions all terminate with a ToDsc node.
internal HpcLinqQueryGen(HpcLinqContext context,
VertexCodeGen vertexCodeGen,
Expression[] qlist)
{
this.m_queryExprs = new Expression[qlist.Length];
this.m_outputTableUris = new string[qlist.Length];
this.m_isTempOutput = new bool[qlist.Length];
this.m_context = context;
for (int i = 0; i < this.m_queryExprs.Length; i++)
{
MethodCallExpression mcExpr = (MethodCallExpression)qlist[i];
string tableUri;
this.m_queryExprs[i] = mcExpr.Arguments[0];
//this block supports the scenario: q-nonToDsc
if (mcExpr.Method.Name == ReflectedNames.DryadLinqIQueryable_AnonymousDscPlaceholder)
{
ExpressionSimplifier<string> e1 = new ExpressionSimplifier<string>();
tableUri = e1.Eval(mcExpr.Arguments[1]);
this.m_isTempOutput[i] = true;
}
//this block supports the scenario: q.ToDsc()
else if (mcExpr.Method.Name == ReflectedNames.DryadLinqIQueryable_ToDscWorker)
{
DscService dsc = context.DscService;
ExpressionSimplifier<string> e2 = new ExpressionSimplifier<string>();
string streamName = e2.Eval(mcExpr.Arguments[2]);
tableUri = DataPath.MakeDscStreamUri(dsc, streamName);
this.m_isTempOutput[i] = false;
}
//this block supports the scenario: q.ToHdfs()
else if (mcExpr.Method.Name == ReflectedNames.DryadLinqIQueryable_ToHdfsWorker)
{
string hdfsHeadNode = context.HdfsService;
ExpressionSimplifier<string> e2 = new ExpressionSimplifier<string>();
string streamName = e2.Eval(mcExpr.Arguments[2]);
tableUri = DataPath.MakeHdfsStreamUri(hdfsHeadNode, streamName);
this.m_isTempOutput[i] = false;
}
else {
throw new InvalidOperationException(); // should not occur.
}
this.m_outputTableUris[i] = tableUri;
}
this.Initialize(vertexCodeGen);
}
internal static string MakeUniqueTemporaryHdfsFileSetUri(HpcLinqContext context)
{
string uri = DataPath.MakeHdfsStreamUri(context.Configuration.HdfsNameNode, MakeUniqueTemporaryDscFileSetName());
return uri;
}
internal static string MakeUniqueTemporaryDscFileSetUri(HpcLinqContext context)
{
string uri = DataPath.MakeDscStreamUri(context.DscService.HostName, MakeUniqueTemporaryDscFileSetName());
return uri;
}
// create DryadLinqMetaData from a query OutputNode
internal static DryadLinqMetaData FromOutputNode(HpcLinqContext context, DryadOutputNode node)
{
DryadLinqMetaData metaData = new DryadLinqMetaData();
if (! (DataPath.IsDsc(node.MetaDataUri) || DataPath.IsHdfs(node.MetaDataUri)) )
{
throw new InvalidOperationException();
}
metaData.m_context = context;
metaData.m_dscStreamName = node.MetaDataUri;
metaData.m_elemType = node.OutputTypes[0];
metaData.m_compressionScheme = node.OutputCompressionScheme;
//metaData.m_version = context.ClientVersion;
//metaData.InitializeFlags();
//metaData.m_fp = 0UL;
//metaData.m_dataSetInfo = node.OutputDataSetInfo;
return metaData;
}
// Load a DryadLinqMetaData from an existing dsc stream.
internal static DryadLinqMetaData FromDscStream(HpcLinqContext context, string dscStreamName)
{
DryadLinqMetaData metaData;
try
{
DscFileSet fs = context.DscService.GetFileSet(dscStreamName);
metaData = new DryadLinqMetaData();
metaData.m_context = context;
metaData.m_dscStreamName = dscStreamName;
//metaData.m_fp = 0L;
//metaData.m_dataSetInfo = null;
byte[] metaDataBytes;
//record-type
metaDataBytes = fs.GetMetadata(DryadLinqMetaData.RECORD_TYPE_NAME);
if (metaDataBytes != null)
{
string recordTypeString = Encoding.UTF8.GetString(metaDataBytes);
metaData.m_elemType = Type.GetType(recordTypeString);
}
//Compression-scheme
metaData.m_compressionScheme = fs.CompressionScheme;
}
catch (Exception e)
{
throw new DryadLinqException(HpcLinqErrorCode.ErrorReadingMetadata,
String.Format(SR.ErrorReadingMetadata), e);
}
return metaData;
}
internal static DryadLinqMetaData ForLocalDebug(HpcLinqContext context,
Type recordType,
string dscStreamName,
DscCompressionScheme compressionScheme)
{
DryadLinqMetaData metaData = new DryadLinqMetaData();
metaData.m_context = context;
metaData.m_dscStreamName = dscStreamName;
metaData.m_elemType = recordType;
metaData.m_compressionScheme = compressionScheme;
//metaData.m_version = context.ClientVersion;
//metaData.InitializeFlags();
//metaData.m_fp = 0UL;
//metaData.m_dataSetInfo = node.OutputDataSetInfo;
return metaData;
}
public static object GetFactory(HpcLinqContext context, Type type)
{
lock (s_codeGenLock)
{
if (s_TypeToFactory.ContainsKey(type))
{
return s_TypeToFactory[type];
}
HpcLinqCodeGen codeGen = new HpcLinqCodeGen(context, new VertexCodeGen());
codeGen.AddDryadCodeForType(type);
// build assembly, and load into memory, because we'll next instantiate
// the factory type out of the generated assembly.
codeGen.BuildAssembly(true);
string factoryTypeFullName = TargetNamespace + "." + HpcLinqFactoryClassName(type);
object factory = codeGen.m_loadedVertexAssembly.CreateInstance(factoryTypeFullName);
s_TypeToFactory.Add(type, factory);
return factory;
}
}
internal HpcLinqCodeGen(HpcLinqContext context, VertexCodeGen vertexCodeGen)
{
this.m_context = context;
this.m_vertexCodeGen = vertexCodeGen;
this.m_loadedVertexAssembly = null;
this.m_dryadLinqUnit = new CodeCompileUnit();
// Create a namespace
this.m_dryadCodeSpace = new CodeNamespace(TargetNamespace);
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Collections"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Collections.Generic"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Text"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Linq"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Linq.Expressions"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Diagnostics"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Runtime.Serialization"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Data.SqlTypes"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Data.Linq"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Data.Linq.Mapping"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("Microsoft.Research.DryadLinq"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("Microsoft.Research.DryadLinq.Internal"));
this.m_dryadLinqUnit.Namespaces.Add(this.m_dryadCodeSpace);
// Create the class for all the Dryad extension methods
this.m_dryadExtensionClass = new CodeTypeDeclaration(ExtensionClassName);
this.m_dryadExtensionClass.IsClass = true;
this.m_dryadExtensionClass.IsPartial = true;
this.m_dryadExtensionClass.TypeAttributes = TypeAttributes.Public;
this.m_dryadCodeSpace.Types.Add(this.m_dryadExtensionClass);
// Create the static constructor for the vertex extension class
this.m_extensionStaticCtor = new CodeTypeConstructor();
this.m_dryadExtensionClass.Members.Add(this.m_extensionStaticCtor);
// Create the class for all the Dryad vertex methods
this.m_dryadVertexClass = new CodeTypeDeclaration(VertexClassName);
this.m_dryadVertexClass.IsClass = true;
this.m_dryadVertexClass.TypeAttributes = TypeAttributes.Public | TypeAttributes.Sealed;
this.m_dryadCodeSpace.Types.Add(this.m_dryadVertexClass);
this.AddCopyResourcesMethod();
// The set of input/output channel datatypes
this.m_dryadDataTypes = new HashSet<Type>();
this.m_dryadDataTypes.Add(typeof(byte));
this.m_dryadDataTypes.Add(typeof(sbyte));
this.m_dryadDataTypes.Add(typeof(bool));
this.m_dryadDataTypes.Add(typeof(char));
this.m_dryadDataTypes.Add(typeof(short));
this.m_dryadDataTypes.Add(typeof(ushort));
this.m_dryadDataTypes.Add(typeof(int));
this.m_dryadDataTypes.Add(typeof(uint));
this.m_dryadDataTypes.Add(typeof(long));
this.m_dryadDataTypes.Add(typeof(ulong));
this.m_dryadDataTypes.Add(typeof(float));
this.m_dryadDataTypes.Add(typeof(decimal));
this.m_dryadDataTypes.Add(typeof(double));
this.m_dryadDataTypes.Add(typeof(DateTime));
this.m_dryadDataTypes.Add(typeof(string));
this.m_dryadDataTypes.Add(typeof(LineRecord));
this.m_dryadDataTypes.Add(typeof(SqlDateTime));
this.m_dryadDataTypes.Add(typeof(Guid));
// The set of datatypes we have added serialization methods
this.m_serializationDatatypes = new HashSet<Type>();
this.m_fieldToStaticName = new Dictionary<FieldInfo, string>();
this.m_staticFieldDefined = new HashSet<string>();
this.m_typeToSerializerName = new Dictionary<Type, string>();
this.m_anonymousTypeToName = new Dictionary<Type, string>();
this.m_nameToAlias = new Dictionary<string, string>();
}
internal static bool RecordCanBeNull(HpcLinqContext context, Type type)
{
if (type == null || type.IsValueType) return false;
object[] attribs = type.GetCustomAttributes(typeof(NullableAttribute), true);
if (attribs.Length == 0)
{
return StaticConfig.AllowNullRecords;
}
return ((NullableAttribute)attribs[0]).CanBeNull;
}
internal static bool DoAutoTypeInference(HpcLinqContext context, Type type)
{
if (!StaticConfig.AllowAutoTypeInference) return false;
object[] a = type.GetCustomAttributes(typeof(AutoTypeInferenceAttribute), true);
return (a.Length != 0);
}
public HpcJobSubmission(HpcLinqContext context)
{
m_context = context;
}
public YarnJobSubmission(HpcLinqContext context)
{
m_context = context;
m_status = JobStatus.NotSubmitted;
m_wc = new WebClient();
}
internal DryadOutputNode(HpcLinqContext context,
string outputUri,
bool isTempOutput,
DscCompressionScheme outputScheme,
Expression queryExpr,
DryadQueryNode child)
: base(QueryNodeType.OutputTable, child.QueryGen, queryExpr, child)
{
if (TypeSystem.IsTypeOrAnyGenericParamsAnonymous(child.OutputTypes[0]))
{
throw DryadLinqException.Create(HpcLinqErrorCode.OutputTypeCannotBeAnonymous,
SR.OutputTypeCannotBeAnonymous,
queryExpr);
}
this.m_context = context;
this.m_outputUri = outputUri;
this.m_outputType = child.OutputTypes[0];
this.m_outputDataSetInfo = child.OutputDataSetInfo;
this.m_partitionCount = child.OutputDataSetInfo.partitionInfo.Count;
this.m_dynamicManager = DynamicManager.Splitter;
this.m_outputCompressionScheme = outputScheme;
this.m_isTempOutput = isTempOutput;
}
/// <summary>
/// Create a new job executor object.
/// </summary>
public JobExecutor(HpcLinqContext context)
{
// use a new job submission object for each query
// this.errorMsg = "";
this.m_context = context;
this.currentStatus = JobStatus.NotSubmitted;
if (context.Runtime is HpcQueryRuntime)
{
YarnJobSubmission job = new YarnJobSubmission(context);
// job.LocalJM = false;
job.Initialize();
this.executionKind = ExecutionKind.JobScheduler;
this.jobSubmission = job;
}
else
{
throw new DryadLinqException(HpcLinqErrorCode.UnsupportedSchedulerType,
String.Format(SR.UnsupportedSchedulerType, context.Runtime));
}
#if REMOVE
case SchedulerKind.LocalJM:
{
HpcJobSubmission job = new HpcJobSubmission(runtime);
job.LocalJM = true;
job.Initialize();
this.executionKind = ExecutionKind.JobScheduler;
this.jobSubmission = job;
DryadLinq.SchedulerType = SchedulerKind.Hpc;
break;
}
#endif
}
