public DscIOStream(string streamName, FileAccess access, DscCompressionScheme compressionScheme)
{
if (String.IsNullOrEmpty(streamName))
{
throw new ArgumentNullException("streamName");
}
Uri streamUri = new Uri(streamName);
this.m_dscClient = new DscService(streamUri.Host);
this.m_fileSetName = streamUri.LocalPath;
this.m_mode = access;
this.m_fstream = null;
this.m_atEOF = false;
this.m_compressionScheme = compressionScheme;
if (access == FileAccess.Read)
{
this.m_dscFileSet = this.m_dscClient.GetFileSet(streamName);
this.m_dscFileEnumerator = this.m_dscFileSet.GetFiles().GetEnumerator();
}
else if (access == FileAccess.Write)
{
this.m_dscFileSet = this.m_dscClient.CreateFileSet(streamName, compressionScheme);
}
else
{
throw new ArgumentException(SR.ReadWriteNotSupported, "access");
}
}
public DscBlockStream(DscIOStream dscStream, DscCompressionScheme scheme)
{
this.m_dscStream = dscStream;
this.m_compressionScheme = scheme;
this.m_isClosed = false;
this.m_compressStream = null;
}
public DscBlockStream(DscIOStream dscStream, DscCompressionScheme scheme)
{
this.m_dscStream = dscStream;
this.m_compressionScheme = scheme;
this.m_isClosed = false;
this.m_compressStream = null;
}
internal HpcLinqFileStream(FileStream fstream, DscCompressionScheme scheme)
{
this.m_fstream = fstream;
this.m_fhandle = fstream.SafeFileHandle;
this.m_compressionScheme = scheme;
this.m_isClosed = false;
this.m_compressStream = null;
}
/// <summary>
/// Converts an IEnumerable{T} to a LINQ-to-HPC IQueryable{T}.
/// </summary>
/// <typeparam name="T">The type of the records in the table.</typeparam>
/// <param name="data">The source data.</param>
/// <returns>An IQueryable{T} representing the data and associated with the HPC LINQ query provider.</returns>
/// <remarks>
/// The source data will be serialized to a DSC fileset using the LINQ-to-HPC serialization approach.
/// The resulting fileset will have an auto-generated name and a temporary lease.
/// </remarks>
public IQueryable <T> FromEnumerable <T>(IEnumerable <T> data)
{
string fileSetName = DataPath.MakeUniqueTemporaryDscFileSetName();
DscCompressionScheme compressionScheme = Configuration.IntermediateDataCompressionScheme;
DryadLinqMetaData metadata = DryadLinqMetaData.ForLocalDebug(this, typeof(T), fileSetName, compressionScheme);
return(DataProvider.IngressTemporaryDataDirectlyToDsc(this, data, fileSetName, metadata, compressionScheme));
}
internal HpcLinqFileStream(FileStream fstream, DscCompressionScheme scheme)
{
this.m_fstream = fstream;
this.m_fhandle = fstream.SafeFileHandle;
this.m_compressionScheme = scheme;
this.m_isClosed = false;
this.m_compressStream = null;
}
private void Initialize(string filePath, FileMode mode, FileAccess access, DscCompressionScheme scheme)
{
try
{
this.m_dscStream = new DscIOStream(filePath, access, mode, scheme);
}
catch (Exception e)
{
throw new DryadLinqException(HpcLinqErrorCode.FailedToCreateStream,
String.Format(SR.FailedToCreateStream, filePath), e);
}
this.m_isClosed = false;
this.m_compressionScheme = scheme;
this.m_compressStream = null;
}
private void Initialize(string filePath, FileMode mode, FileAccess access, DscCompressionScheme scheme)
{
try
{
this.m_fstream = new FileStream(filePath, mode, access);
}
catch (Exception e)
{
throw new DryadLinqException(HpcLinqErrorCode.CannotAccesFilePath,
String.Format(SR.CannotAccesFilePath, filePath), e);
}
this.m_fhandle = m_fstream.SafeFileHandle;
this.m_isClosed = false;
this.m_compressionScheme = scheme;
this.m_compressStream = null;
}
internal MultiBlockStream(List <string[]> srcList,
string associatedDscStreamName,
FileAccess access,
DscCompressionScheme scheme,
bool appendNewLinesToFiles)
{
this.m_srcList = srcList;
m_associatedDscStreamName = associatedDscStreamName;
if (srcList.Count == 0)
{
throw new DryadLinqException(HpcLinqErrorCode.MultiBlockEmptyPartitionList,
SR.MultiBlockEmptyPartitionList);
}
this.m_compressionScheme = scheme;
this.m_curIdx = 0;
this.m_curStream = this.GetStream(this.m_curIdx++, access);
this.m_appendNewLinesToFiles = appendNewLinesToFiles;
}
internal MultiBlockStream(List<string[]> srcList,
string associatedDscStreamName,
FileAccess access,
DscCompressionScheme scheme,
bool appendNewLinesToFiles)
{
this.m_srcList = srcList;
m_associatedDscStreamName = associatedDscStreamName;
if (srcList.Count == 0)
{
throw new DryadLinqException(HpcLinqErrorCode.MultiBlockEmptyPartitionList,
SR.MultiBlockEmptyPartitionList);
}
this.m_compressionScheme = scheme;
this.m_curIdx = 0;
this.m_curStream = this.GetStream(this.m_curIdx++, access);
this.m_appendNewLinesToFiles = appendNewLinesToFiles;
}
internal static DryadLinqMetaData ForLocalDebug(HpcLinqContext context,
Type recordType,
string dscStreamName,
DscCompressionScheme compressionScheme)
{
DryadLinqMetaData metaData = new DryadLinqMetaData();
metaData.m_context = context;
metaData.m_dscStreamName = dscStreamName;
metaData.m_elemType = recordType;
metaData.m_compressionScheme = compressionScheme;
//metaData.m_version = context.ClientVersion;
//metaData.InitializeFlags();
//metaData.m_fp = 0UL;
//metaData.m_dataSetInfo = node.OutputDataSetInfo;
return(metaData);
}
internal TableEnumerator(HpcLinqContext context,
List <string[]> filePathList,
string associatedDscStreamName,
DscCompressionScheme scheme)
{
this.m_context = context;
this.m_current = default(T);
this.m_filePathList = filePathList;
this.m_associatedDscStreamName = associatedDscStreamName;
this.m_compressionScheme = scheme;
this.m_factory = (HpcLinqFactory <T>)HpcLinqCodeGen.GetFactory(context, typeof(T));
bool appendNewLinesToFiles = (typeof(T) == typeof(LineRecord));
NativeBlockStream nativeStream = new MultiBlockStream(m_filePathList, m_associatedDscStreamName,
FileAccess.Read, m_compressionScheme,
appendNewLinesToFiles);
this.m_reader = this.m_factory.MakeReader(nativeStream);
if (context.Configuration.AllowConcurrentUserDelegatesInSingleProcess)
{
this.m_reader.StartWorker();
}
}
public DscBlockStream(string filePath, FileMode mode, FileAccess access, DscCompressionScheme scheme)
{
this.Initialize(filePath, mode, access, scheme);
}
public DscBlockStream(string filePath, FileAccess access, DscCompressionScheme scheme)
{
FileMode mode = (access == FileAccess.Read) ? FileMode.Open : FileMode.OpenOrCreate;
this.Initialize(filePath, mode, access, scheme);
}
public DscIOStream(string streamName, FileAccess access, FileMode createMode, DscCompressionScheme compressionScheme)
{
if (String.IsNullOrEmpty(streamName))
{
throw new ArgumentNullException("streamName");
}
Uri streamUri = new Uri(streamName);
this.m_dscClient = new DscService(streamUri.Host);
this.m_fileSetName = streamUri.LocalPath.TrimStart('/');
this.m_mode = access;
this.m_compressionScheme = compressionScheme;
bool streamExists = this.m_dscClient.FileSetExists(this.m_fileSetName);
if (access == FileAccess.Read)
{
switch (createMode)
{
case FileMode.Open:
case FileMode.OpenOrCreate:
if (!streamExists)
{
throw new FileNotFoundException(String.Format( SR.StreamDoesNotExist , streamName));
}
break;
case FileMode.Append:
case FileMode.Create:
case FileMode.CreateNew:
case FileMode.Truncate:
throw new NotSupportedException();
}
this.m_dscFileSet = this.m_dscClient.GetFileSet(streamName);
this.m_dscFileEnumerator = this.m_dscFileSet.GetFiles().GetEnumerator();
}
else if (access == FileAccess.Write)
{
switch (createMode)
{
case FileMode.Append:
if (!streamExists)
{
this.m_dscFileSet = this.m_dscClient.CreateFileSet(this.m_fileSetName, this.m_compressionScheme);
}
break;
case FileMode.Create:
if (streamExists)
{
this.m_dscClient.DeleteFileSet(this.m_fileSetName);
}
this.m_dscFileSet = this.m_dscClient.CreateFileSet(this.m_fileSetName, this.m_compressionScheme);
break;
case FileMode.CreateNew:
if (streamExists)
{
throw new IOException(String.Format(SR.StreamAlreadyExists, streamName));
}
break;
case FileMode.Truncate:
if (streamExists)
{
this.m_dscClient.DeleteFileSet(this.m_fileSetName);
}
this.m_dscFileSet = this.m_dscClient.CreateFileSet(this.m_fileSetName, this.m_compressionScheme);
break;
case FileMode.Open:
case FileMode.OpenOrCreate: // TODO: this should be dealt with correctly,
// although it's not obvious what open should do
throw new NotSupportedException();
}
}
else
{
throw new ArgumentException(SR.ReadWriteNotSupported, "access");
}
this.m_fstream = null;
this.m_atEOF = false;
}
private void Initialize(string filePath, FileMode mode, FileAccess access, DscCompressionScheme scheme)
{
try
{
this.m_dscStream = new DscIOStream(filePath, access, mode, scheme);
}
catch (Exception e)
{
throw new DryadLinqException(HpcLinqErrorCode.FailedToCreateStream,
String.Format(SR.FailedToCreateStream, filePath), e);
}
this.m_isClosed = false;
this.m_compressionScheme = scheme;
this.m_compressStream = null;
}
private void Initialize(string filePath, FileMode mode, FileAccess access, DscCompressionScheme scheme)
{
try
{
this.m_fstream = new FileStream(filePath, mode, access);
}
catch(Exception e)
{
throw new DryadLinqException(HpcLinqErrorCode.CannotAccesFilePath,
String.Format(SR.CannotAccesFilePath , filePath),e);
}
this.m_fhandle = m_fstream.SafeFileHandle;
this.m_isClosed = false;
this.m_compressionScheme = scheme;
this.m_compressStream = null;
}
internal DscFileSet CreateFileSet(string streamName, DscCompressionScheme compressionScheme)
{
throw new NotImplementedException();
}
internal DscFileSet CreateFileSet(string streamName, DscCompressionScheme compressionScheme)
{
throw new NotImplementedException();
}
//* streams plain enumerable data directly to DSC
internal static DryadLinqQuery <T> IngressDataDirectlyToDsc <T>(HpcLinqContext context,
IEnumerable <T> source,
string dscFileSetName,
DryadLinqMetaData metaData,
DscCompressionScheme outputScheme)
{
try
{
string dscFileSetUri = DataPath.MakeDscStreamUri(context.DscService.HostName, dscFileSetName);
if (source.Take(1).Count() == 0)
{
//there is no data.. we must create a FileSet with an empty file
//(the factory/stream approach opens files lazily and thus never opens a file if there is no data)
if (context.DscService.FileSetExists(dscFileSetName))
{
context.DscService.DeleteFileSet(dscFileSetName);
}
DscFileSet fileSet = context.DscService.CreateFileSet(dscFileSetName, outputScheme);
DscFile file = fileSet.AddNewFile(0);
string writePath = file.WritePath;
if (outputScheme == DscCompressionScheme.Gzip)
{
//even zero-byte file must go through the gzip-compressor (for headers etc).
using (Stream s = new FileStream(writePath, FileMode.Create))
{
var gzipStream = new GZipStream(s, CompressionMode.Compress);
gzipStream.Close();
}
}
else
{
StreamWriter sw = new StreamWriter(writePath, false);
sw.Close();
}
fileSet.Seal();
}
else
{
HpcLinqFactory <T> factory = (HpcLinqFactory <T>)HpcLinqCodeGen.GetFactory(context, typeof(T));
// new DscBlockStream(uri,Create,Write,compress) provides a DSC stream with one partition.
NativeBlockStream nativeStream = new DscBlockStream(dscFileSetUri, FileMode.Create, FileAccess.Write, outputScheme);
HpcRecordWriter <T> writer = factory.MakeWriter(nativeStream);
try
{
if (context.Configuration.AllowConcurrentUserDelegatesInSingleProcess)
{
foreach (T item in source)
{
writer.WriteRecordAsync(item);
}
}
else
{
foreach (T item in source)
{
writer.WriteRecordSync(item);
}
}
}
finally
{
writer.Close(); // closes the NativeBlockStream, which seals the dsc stream.
}
}
if (metaData != null)
{
DscFileSet fileSet = context.DscService.GetFileSet(dscFileSetName);
fileSet.SetMetadata(DryadLinqMetaData.RECORD_TYPE_NAME, Encoding.UTF8.GetBytes(metaData.ElemType.AssemblyQualifiedName));
}
return(DataProvider.GetPartitionedTable <T>(context, dscFileSetUri));
}
catch
{
// if we had a problem creating the empty fileset, try to delete it to avoid cruft being left in DSC.
try
{
context.DscService.DeleteFileSet(dscFileSetName);
}
catch
{
// suppress error during delete
}
throw; // rethrow the original exception.
}
}
// ingresses data, and also sets the temporary-length lease.
internal static DryadLinqQuery <T> IngressTemporaryDataDirectlyToDsc <T>(HpcLinqContext context, IEnumerable <T> source, string dscFileSetName, DryadLinqMetaData metaData, DscCompressionScheme outputScheme)
{
DryadLinqQuery <T> result = IngressDataDirectlyToDsc(context, source, dscFileSetName, metaData, outputScheme);
// try to set a temporary lease on the resulting fileset
try
{
DscFileSet fs = context.DscService.GetFileSet(dscFileSetName);
fs.SetLeaseEndTime(DateTime.Now.Add(StaticConfig.LeaseDurationForTempFiles));
}
catch (DscException)
{
// suppress
}
return(result);
}
public DscIOStream(string streamName, FileAccess access, FileMode createMode, DscCompressionScheme compressionScheme)
{
if (String.IsNullOrEmpty(streamName))
{
throw new ArgumentNullException("streamName");
}
Uri streamUri = new Uri(streamName);
this.m_dscClient = new DscService(streamUri.Host);
this.m_fileSetName = streamUri.LocalPath.TrimStart('/');
this.m_mode = access;
this.m_compressionScheme = compressionScheme;
bool streamExists = this.m_dscClient.FileSetExists(this.m_fileSetName);
if (access == FileAccess.Read)
{
switch (createMode)
{
case FileMode.Open:
case FileMode.OpenOrCreate:
if (!streamExists)
{
throw new FileNotFoundException(String.Format(SR.StreamDoesNotExist, streamName));
}
break;
case FileMode.Append:
case FileMode.Create:
case FileMode.CreateNew:
case FileMode.Truncate:
throw new NotSupportedException();
}
this.m_dscFileSet = this.m_dscClient.GetFileSet(streamName);
this.m_dscFileEnumerator = this.m_dscFileSet.GetFiles().GetEnumerator();
}
else if (access == FileAccess.Write)
{
switch (createMode)
{
case FileMode.Append:
if (!streamExists)
{
this.m_dscFileSet = this.m_dscClient.CreateFileSet(this.m_fileSetName, this.m_compressionScheme);
}
break;
case FileMode.Create:
if (streamExists)
{
this.m_dscClient.DeleteFileSet(this.m_fileSetName);
}
this.m_dscFileSet = this.m_dscClient.CreateFileSet(this.m_fileSetName, this.m_compressionScheme);
break;
case FileMode.CreateNew:
if (streamExists)
{
throw new IOException(String.Format(SR.StreamAlreadyExists, streamName));
}
break;
case FileMode.Truncate:
if (streamExists)
{
this.m_dscClient.DeleteFileSet(this.m_fileSetName);
}
this.m_dscFileSet = this.m_dscClient.CreateFileSet(this.m_fileSetName, this.m_compressionScheme);
break;
case FileMode.Open:
case FileMode.OpenOrCreate: // TODO: this should be dealt with correctly,
// although it's not obvious what open should do
throw new NotSupportedException();
}
}
else
{
throw new ArgumentException(SR.ReadWriteNotSupported, "access");
}
this.m_fstream = null;
this.m_atEOF = false;
}
public DscBlockStream(string filePath, FileAccess access, DscCompressionScheme scheme)
{
FileMode mode = (access == FileAccess.Read) ? FileMode.Open : FileMode.OpenOrCreate;
this.Initialize(filePath, mode, access, scheme);
}
internal static DryadLinqMetaData ForLocalDebug(HpcLinqContext context,
Type recordType,
string dscStreamName,
DscCompressionScheme compressionScheme)
{
DryadLinqMetaData metaData = new DryadLinqMetaData();
metaData.m_context = context;
metaData.m_dscStreamName = dscStreamName;
metaData.m_elemType = recordType;
metaData.m_compressionScheme = compressionScheme;
//metaData.m_version = context.ClientVersion;
//metaData.InitializeFlags();
//metaData.m_fp = 0UL;
//metaData.m_dataSetInfo = node.OutputDataSetInfo;
return metaData;
}
internal DryadOutputNode(HpcLinqContext context,
string outputUri,
bool isTempOutput,
DscCompressionScheme outputScheme,
Expression queryExpr,
DryadQueryNode child)
: base(QueryNodeType.OutputTable, child.QueryGen, queryExpr, child)
{
if (TypeSystem.IsTypeOrAnyGenericParamsAnonymous(child.OutputTypes[0]))
{
throw DryadLinqException.Create(HpcLinqErrorCode.OutputTypeCannotBeAnonymous,
SR.OutputTypeCannotBeAnonymous,
queryExpr);
}
this.m_context = context;
this.m_outputUri = outputUri;
this.m_outputType = child.OutputTypes[0];
this.m_outputDataSetInfo = child.OutputDataSetInfo;
this.m_partitionCount = child.OutputDataSetInfo.partitionInfo.Count;
this.m_dynamicManager = DynamicManager.Splitter;
this.m_outputCompressionScheme = outputScheme;
this.m_isTempOutput = isTempOutput;
}
public DscIOStream(string streamName, FileAccess access, DscCompressionScheme compressionScheme)
{
if (String.IsNullOrEmpty(streamName))
{
throw new ArgumentNullException("streamName");
}
Uri streamUri = new Uri(streamName);
this.m_dscClient = new DscService(streamUri.Host);
this.m_fileSetName = streamUri.LocalPath;
this.m_mode = access;
this.m_fstream = null;
this.m_atEOF = false;
this.m_compressionScheme = compressionScheme;
if (access == FileAccess.Read)
{
this.m_dscFileSet = this.m_dscClient.GetFileSet(streamName);
this.m_dscFileEnumerator = this.m_dscFileSet.GetFiles().GetEnumerator();
}
else if (access == FileAccess.Write)
{
this.m_dscFileSet = this.m_dscClient.CreateFileSet(streamName, compressionScheme);
}
else
{
throw new ArgumentException(SR.ReadWriteNotSupported, "access");
}
}
internal HpcLinqFileStream(string filePath, FileAccess access, DscCompressionScheme scheme)
{
FileMode mode = (access == FileAccess.Read) ? FileMode.Open : FileMode.OpenOrCreate;
Initialize(filePath, mode, access, scheme);
}
internal HpcLinqFileStream(string filePath, FileAccess access, DscCompressionScheme scheme)
{
FileMode mode = (access == FileAccess.Read) ? FileMode.Open : FileMode.OpenOrCreate;
Initialize(filePath, mode, access, scheme);
}
internal HpcLinqFileStream(string filePath, FileMode mode, FileAccess access, DscCompressionScheme scheme)
{
Initialize(filePath, mode, access, scheme);
}
public DscBlockStream(string filePath, FileMode mode, FileAccess access, DscCompressionScheme scheme)
{
this.Initialize(filePath, mode, access, scheme);
}
internal HpcLinqFileStream(string filePath, FileMode mode, FileAccess access, DscCompressionScheme scheme)
{
Initialize(filePath, mode, access, scheme);
}