/// <summary> /// Open a DSC fileset as a LINQ-to-HPC IQueryable{T}. /// </summary> /// <typeparam name="T">The type of the records in the table.</typeparam> /// <param name="dscFileSetName">The name of the DSC fileset. </param> /// <returns>An IQueryable{T} representing the data and associated with the HPC LINQ query provider.</returns> public IQueryable <T> FromDsc <T>(string fileSetName) { ThrowIfDisposed(); string fullPath = DataPath.MakeDscStreamUri(_dscService, fileSetName); try { DscFileSet fs = _dscService.GetFileSet(fileSetName); if (!fs.IsSealed()) { throw new DryadLinqException(HpcLinqErrorCode.FileSetMustBeSealed, SR.FileSetMustBeSealed); } int fileCount = fs.GetFiles().Count(); if (fileCount < 1) { throw new DryadLinqException(HpcLinqErrorCode.FileSetMustHaveAtLeastOneFile, SR.FileSetMustHaveAtLeastOneFile); } } catch (DscException dscEx) { throw new DryadLinqException(HpcLinqErrorCode.FileSetCouldNotBeOpened, SR.FileSetCouldNotBeOpened, dscEx); } DryadLinqQuery <T> q = DataProvider.GetPartitionedTable <T>(this, fullPath); q.CheckAndInitialize(); // force the data-info checks. return(q); }
//* streams plain enumerable data directly to DSC internal static DryadLinqQuery <T> IngressDataDirectlyToDsc <T>(HpcLinqContext context, IEnumerable <T> source, string dscFileSetName, DryadLinqMetaData metaData, DscCompressionScheme outputScheme) { try { string dscFileSetUri = DataPath.MakeDscStreamUri(context.DscService.HostName, dscFileSetName); if (source.Take(1).Count() == 0) { //there is no data.. we must create a FileSet with an empty file //(the factory/stream approach opens files lazily and thus never opens a file if there is no data) if (context.DscService.FileSetExists(dscFileSetName)) { context.DscService.DeleteFileSet(dscFileSetName); } DscFileSet fileSet = context.DscService.CreateFileSet(dscFileSetName, outputScheme); DscFile file = fileSet.AddNewFile(0); string writePath = file.WritePath; if (outputScheme == DscCompressionScheme.Gzip) { //even zero-byte file must go through the gzip-compressor (for headers etc). using (Stream s = new FileStream(writePath, FileMode.Create)) { var gzipStream = new GZipStream(s, CompressionMode.Compress); gzipStream.Close(); } } else { StreamWriter sw = new StreamWriter(writePath, false); sw.Close(); } fileSet.Seal(); } else { HpcLinqFactory <T> factory = (HpcLinqFactory <T>)HpcLinqCodeGen.GetFactory(context, typeof(T)); // new DscBlockStream(uri,Create,Write,compress) provides a DSC stream with one partition. NativeBlockStream nativeStream = new DscBlockStream(dscFileSetUri, FileMode.Create, FileAccess.Write, outputScheme); HpcRecordWriter <T> writer = factory.MakeWriter(nativeStream); try { if (context.Configuration.AllowConcurrentUserDelegatesInSingleProcess) { foreach (T item in source) { writer.WriteRecordAsync(item); } } else { foreach (T item in source) { writer.WriteRecordSync(item); } } } finally { writer.Close(); // closes the NativeBlockStream, which seals the dsc stream. } } if (metaData != null) { DscFileSet fileSet = context.DscService.GetFileSet(dscFileSetName); fileSet.SetMetadata(DryadLinqMetaData.RECORD_TYPE_NAME, Encoding.UTF8.GetBytes(metaData.ElemType.AssemblyQualifiedName)); } return(DataProvider.GetPartitionedTable <T>(context, dscFileSetUri)); } catch { // if we had a problem creating the empty fileset, try to delete it to avoid cruft being left in DSC. try { context.DscService.DeleteFileSet(dscFileSetName); } catch { // suppress error during delete } throw; // rethrow the original exception. } }
internal static string MakeUniqueTemporaryDscFileSetUri(HpcLinqContext context) { string uri = DataPath.MakeDscStreamUri(context.DscService.HostName, MakeUniqueTemporaryDscFileSetName()); return(uri); }