// Load a DryadLinqMetaData from an existing dsc stream.
internal static DryadLinqMetaData FromDscStream(HpcLinqContext context, string dscStreamName)
{
DryadLinqMetaData metaData;
try
{
DscFileSet fs = context.DscService.GetFileSet(dscStreamName);
metaData = new DryadLinqMetaData();
metaData.m_context = context;
metaData.m_dscStreamName = dscStreamName;
//metaData.m_fp = 0L;
//metaData.m_dataSetInfo = null;
byte[] metaDataBytes;
//record-type
metaDataBytes = fs.GetMetadata(DryadLinqMetaData.RECORD_TYPE_NAME);
if (metaDataBytes != null)
{
string recordTypeString = Encoding.UTF8.GetString(metaDataBytes);
metaData.m_elemType = Type.GetType(recordTypeString);
}
//Compression-scheme
metaData.m_compressionScheme = fs.CompressionScheme;
}
catch (Exception e)
{
throw new DryadLinqException(HpcLinqErrorCode.ErrorReadingMetadata,
String.Format(SR.ErrorReadingMetadata), e);
}
return(metaData);
}
internal static HpcLinqContext GetContext(DryadLinqProviderBase provider)
{
HpcLinqContext context = provider.Context;
Debug.Assert(context != null, "A context should always be associated with a HpcLinqQuery<T>");
context.ThrowIfDisposed();
return(context);
}
internal static bool DoAutoTypeInference(HpcLinqContext context, Type type)
{
if (!StaticConfig.AllowAutoTypeInference)
{
return(false);
}
object[] a = type.GetCustomAttributes(typeof(AutoTypeInferenceAttribute), true);
return(a.Length != 0);
}
/// <summary>
/// Get the DSC file set specified by a URI.
/// </summary>
/// <typeparam name="T">The record type of the table.</typeparam>
/// <param name="dscFileSetUri">The URI of a DscFileSet.</param>
/// <returns>A query object representing the dsc file set data.</returns>
internal static DryadLinqQuery <T> GetPartitionedTable <T>(HpcLinqContext context, string dscFileSetUri)
{
Dictionary <string, string> args = DataPath.GetArguments(dscFileSetUri);
DataProvider dataProvider = new DataProvider();
DryadLinqProvider queryProvider = new DryadLinqProvider(context);
return(new DryadLinqQuery <T>(null, queryProvider, dataProvider, dscFileSetUri));
}
internal HpcJobSubmission(HpcLinqContext context)
{
this.m_context = context;
this.m_status = JobStatus.NotSubmitted;
//@@TODO[P0] pass the runtime to the DryadJobSubmission so that it can use the scheduler instance.
//@@TODO: Merge DryadJobSubmission into Ms.Hpc.Linq. Until then make sure Context is not disposed before DryadJobSubmission.
this.m_job = new DryadJobSubmission(m_context.GetIScheduler());
}
internal static bool RecordCanBeNull(HpcLinqContext context, Type type)
{
if (type == null || type.IsValueType)
{
return(false);
}
object[] attribs = type.GetCustomAttributes(typeof(NullableAttribute), true);
if (attribs.Length == 0)
{
return(StaticConfig.AllowNullRecords);
}
return(((NullableAttribute)attribs[0]).CanBeNull);
}
// ingresses data, and also sets the temporary-length lease.
internal static DryadLinqQuery <T> IngressTemporaryDataDirectlyToDsc <T>(HpcLinqContext context, IEnumerable <T> source, string dscFileSetName, DryadLinqMetaData metaData, DscCompressionScheme outputScheme)
{
DryadLinqQuery <T> result = IngressDataDirectlyToDsc(context, source, dscFileSetName, metaData, outputScheme);
// try to set a temporary lease on the resulting fileset
try
{
DscFileSet fs = context.DscService.GetFileSet(dscFileSetName);
fs.SetLeaseEndTime(DateTime.Now.Add(StaticConfig.LeaseDurationForTempFiles));
}
catch (DscException)
{
// suppress
}
return(result);
}
/// <summary>
/// Create a new job executor object.
/// </summary>
public JobExecutor(HpcLinqContext context)
{
// use a new job submission object for each query
// this.errorMsg = "";
this.m_context = context;
this.currentStatus = JobStatus.NotSubmitted;
if (context.Runtime is HpcQueryRuntime)
{
YarnJobSubmission job = new YarnJobSubmission(context);
// job.LocalJM = false;
job.Initialize();
this.executionKind = ExecutionKind.JobScheduler;
this.jobSubmission = job;
}
else
{
throw new DryadLinqException(HpcLinqErrorCode.UnsupportedSchedulerType,
String.Format(SR.UnsupportedSchedulerType, context.Runtime));
}
internal static DryadLinqMetaData ForLocalDebug(HpcLinqContext context,
Type recordType,
string dscStreamName,
DscCompressionScheme compressionScheme)
{
DryadLinqMetaData metaData = new DryadLinqMetaData();
metaData.m_context = context;
metaData.m_dscStreamName = dscStreamName;
metaData.m_elemType = recordType;
metaData.m_compressionScheme = compressionScheme;
//metaData.m_version = context.ClientVersion;
//metaData.InitializeFlags();
//metaData.m_fp = 0UL;
//metaData.m_dataSetInfo = node.OutputDataSetInfo;
return(metaData);
}
// create DryadLinqMetaData from a query OutputNode
internal static DryadLinqMetaData FromOutputNode(HpcLinqContext context, DryadOutputNode node)
{
DryadLinqMetaData metaData = new DryadLinqMetaData();
if (!(DataPath.IsDsc(node.MetaDataUri) || DataPath.IsHdfs(node.MetaDataUri)))
{
throw new InvalidOperationException();
}
metaData.m_context = context;
metaData.m_dscStreamName = node.MetaDataUri;
metaData.m_elemType = node.OutputTypes[0];
metaData.m_compressionScheme = node.OutputCompressionScheme;
//metaData.m_version = context.ClientVersion;
//metaData.InitializeFlags();
//metaData.m_fp = 0UL;
//metaData.m_dataSetInfo = node.OutputDataSetInfo;
return(metaData);
}
internal TableEnumerator(HpcLinqContext context,
List <string[]> filePathList,
string associatedDscStreamName,
DscCompressionScheme scheme)
{
this.m_context = context;
this.m_current = default(T);
this.m_filePathList = filePathList;
this.m_associatedDscStreamName = associatedDscStreamName;
this.m_compressionScheme = scheme;
this.m_factory = (HpcLinqFactory <T>)HpcLinqCodeGen.GetFactory(context, typeof(T));
bool appendNewLinesToFiles = (typeof(T) == typeof(LineRecord));
NativeBlockStream nativeStream = new MultiBlockStream(m_filePathList, m_associatedDscStreamName,
FileAccess.Read, m_compressionScheme,
appendNewLinesToFiles);
this.m_reader = this.m_factory.MakeReader(nativeStream);
if (context.Configuration.AllowConcurrentUserDelegatesInSingleProcess)
{
this.m_reader.StartWorker();
}
}
public YarnJobSubmission(HpcLinqContext context)
{
m_context = context;
m_status = JobStatus.NotSubmitted;
m_wc = new WebClient();
}
internal static string MakeUniqueTemporaryHdfsFileSetUri(HpcLinqContext context)
{
string uri = DataPath.MakeHdfsStreamUri(context.Configuration.HdfsNameNode, MakeUniqueTemporaryDscFileSetName());
return(uri);
}
public HpcJobSubmission(HpcLinqContext context)
{
m_context = context;
}
//* streams plain enumerable data directly to DSC
internal static DryadLinqQuery <T> IngressDataDirectlyToDsc <T>(HpcLinqContext context,
IEnumerable <T> source,
string dscFileSetName,
DryadLinqMetaData metaData,
DscCompressionScheme outputScheme)
{
try
{
string dscFileSetUri = DataPath.MakeDscStreamUri(context.DscService.HostName, dscFileSetName);
if (source.Take(1).Count() == 0)
{
//there is no data.. we must create a FileSet with an empty file
//(the factory/stream approach opens files lazily and thus never opens a file if there is no data)
if (context.DscService.FileSetExists(dscFileSetName))
{
context.DscService.DeleteFileSet(dscFileSetName);
}
DscFileSet fileSet = context.DscService.CreateFileSet(dscFileSetName, outputScheme);
DscFile file = fileSet.AddNewFile(0);
string writePath = file.WritePath;
if (outputScheme == DscCompressionScheme.Gzip)
{
//even zero-byte file must go through the gzip-compressor (for headers etc).
using (Stream s = new FileStream(writePath, FileMode.Create))
{
var gzipStream = new GZipStream(s, CompressionMode.Compress);
gzipStream.Close();
}
}
else
{
StreamWriter sw = new StreamWriter(writePath, false);
sw.Close();
}
fileSet.Seal();
}
else
{
HpcLinqFactory <T> factory = (HpcLinqFactory <T>)HpcLinqCodeGen.GetFactory(context, typeof(T));
// new DscBlockStream(uri,Create,Write,compress) provides a DSC stream with one partition.
NativeBlockStream nativeStream = new DscBlockStream(dscFileSetUri, FileMode.Create, FileAccess.Write, outputScheme);
HpcRecordWriter <T> writer = factory.MakeWriter(nativeStream);
try
{
if (context.Configuration.AllowConcurrentUserDelegatesInSingleProcess)
{
foreach (T item in source)
{
writer.WriteRecordAsync(item);
}
}
else
{
foreach (T item in source)
{
writer.WriteRecordSync(item);
}
}
}
finally
{
writer.Close(); // closes the NativeBlockStream, which seals the dsc stream.
}
}
if (metaData != null)
{
DscFileSet fileSet = context.DscService.GetFileSet(dscFileSetName);
fileSet.SetMetadata(DryadLinqMetaData.RECORD_TYPE_NAME, Encoding.UTF8.GetBytes(metaData.ElemType.AssemblyQualifiedName));
}
return(DataProvider.GetPartitionedTable <T>(context, dscFileSetUri));
}
catch
{
// if we had a problem creating the empty fileset, try to delete it to avoid cruft being left in DSC.
try
{
context.DscService.DeleteFileSet(dscFileSetName);
}
catch
{
// suppress error during delete
}
throw; // rethrow the original exception.
}
}
internal static string MakeUniqueTemporaryDscFileSetUri(HpcLinqContext context)
{
string uri = DataPath.MakeDscStreamUri(context.DscService.HostName, MakeUniqueTemporaryDscFileSetName());
return(uri);
}
public DryadLinqQueryEnumerable(HpcLinqContext context, string fileSetName)
{
m_context = context;
m_fileSetName = fileSetName;
}
public DryadLinqLocalProvider(IQueryProvider linqToObjectsProvider, HpcLinqContext context)
: base(context)
{
this.m_linqToObjectsProvider = linqToObjectsProvider;
}
internal DryadLinqProviderBase(HpcLinqContext context)
{
m_context = context;
}
internal DryadLinqProvider(HpcLinqContext context)
: base(context)
{
}