/// <summary>
/// Open a DSC fileset as a LINQ-to-HPC IQueryable{T}.
/// </summary>
/// <typeparam name="T">The type of the records in the table.</typeparam>
/// <param name="dscFileSetName">The name of the DSC fileset. </param>
/// <returns>An IQueryable{T} representing the data and associated with the HPC LINQ query provider.</returns>
public IQueryable <T> FromDsc <T>(string fileSetName)
{
ThrowIfDisposed();
string fullPath = DataPath.MakeDscStreamUri(_dscService, fileSetName);
try {
DscFileSet fs = _dscService.GetFileSet(fileSetName);
if (!fs.IsSealed())
{
throw new DryadLinqException(HpcLinqErrorCode.FileSetMustBeSealed,
SR.FileSetMustBeSealed);
}
int fileCount = fs.GetFiles().Count();
if (fileCount < 1)
{
throw new DryadLinqException(HpcLinqErrorCode.FileSetMustHaveAtLeastOneFile,
SR.FileSetMustHaveAtLeastOneFile);
}
}
catch (DscException dscEx) {
throw new DryadLinqException(HpcLinqErrorCode.FileSetCouldNotBeOpened,
SR.FileSetCouldNotBeOpened, dscEx);
}
DryadLinqQuery <T> q = DataProvider.GetPartitionedTable <T>(this, fullPath);
q.CheckAndInitialize(); // force the data-info checks.
return(q);
}
//* streams plain enumerable data directly to DSC
internal static DryadLinqQuery <T> IngressDataDirectlyToDsc <T>(HpcLinqContext context,
IEnumerable <T> source,
string dscFileSetName,
DryadLinqMetaData metaData,
DscCompressionScheme outputScheme)
{
try
{
string dscFileSetUri = DataPath.MakeDscStreamUri(context.DscService.HostName, dscFileSetName);
if (source.Take(1).Count() == 0)
{
//there is no data.. we must create a FileSet with an empty file
//(the factory/stream approach opens files lazily and thus never opens a file if there is no data)
if (context.DscService.FileSetExists(dscFileSetName))
{
context.DscService.DeleteFileSet(dscFileSetName);
}
DscFileSet fileSet = context.DscService.CreateFileSet(dscFileSetName, outputScheme);
DscFile file = fileSet.AddNewFile(0);
string writePath = file.WritePath;
if (outputScheme == DscCompressionScheme.Gzip)
{
//even zero-byte file must go through the gzip-compressor (for headers etc).
using (Stream s = new FileStream(writePath, FileMode.Create))
{
var gzipStream = new GZipStream(s, CompressionMode.Compress);
gzipStream.Close();
}
}
else
{
StreamWriter sw = new StreamWriter(writePath, false);
sw.Close();
}
fileSet.Seal();
}
else
{
HpcLinqFactory <T> factory = (HpcLinqFactory <T>)HpcLinqCodeGen.GetFactory(context, typeof(T));
// new DscBlockStream(uri,Create,Write,compress) provides a DSC stream with one partition.
NativeBlockStream nativeStream = new DscBlockStream(dscFileSetUri, FileMode.Create, FileAccess.Write, outputScheme);
HpcRecordWriter <T> writer = factory.MakeWriter(nativeStream);
try
{
if (context.Configuration.AllowConcurrentUserDelegatesInSingleProcess)
{
foreach (T item in source)
{
writer.WriteRecordAsync(item);
}
}
else
{
foreach (T item in source)
{
writer.WriteRecordSync(item);
}
}
}
finally
{
writer.Close(); // closes the NativeBlockStream, which seals the dsc stream.
}
}
if (metaData != null)
{
DscFileSet fileSet = context.DscService.GetFileSet(dscFileSetName);
fileSet.SetMetadata(DryadLinqMetaData.RECORD_TYPE_NAME, Encoding.UTF8.GetBytes(metaData.ElemType.AssemblyQualifiedName));
}
return(DataProvider.GetPartitionedTable <T>(context, dscFileSetUri));
}
catch
{
// if we had a problem creating the empty fileset, try to delete it to avoid cruft being left in DSC.
try
{
context.DscService.DeleteFileSet(dscFileSetName);
}
catch
{
// suppress error during delete
}
throw; // rethrow the original exception.
}
}
internal static string MakeUniqueTemporaryDscFileSetUri(HpcLinqContext context)
{
string uri = DataPath.MakeDscStreamUri(context.DscService.HostName, MakeUniqueTemporaryDscFileSetName());
return(uri);
}
