// Load a DryadLinqMetaData from an existing dsc stream. internal static DryadLinqMetaData FromDscStream(HpcLinqContext context, string dscStreamName) { DryadLinqMetaData metaData; try { DscFileSet fs = context.DscService.GetFileSet(dscStreamName); metaData = new DryadLinqMetaData(); metaData.m_context = context; metaData.m_dscStreamName = dscStreamName; //metaData.m_fp = 0L; //metaData.m_dataSetInfo = null; byte[] metaDataBytes; //record-type metaDataBytes = fs.GetMetadata(DryadLinqMetaData.RECORD_TYPE_NAME); if (metaDataBytes != null) { string recordTypeString = Encoding.UTF8.GetString(metaDataBytes); metaData.m_elemType = Type.GetType(recordTypeString); } //Compression-scheme metaData.m_compressionScheme = fs.CompressionScheme; } catch (Exception e) { throw new DryadLinqException(HpcLinqErrorCode.ErrorReadingMetadata, String.Format(SR.ErrorReadingMetadata), e); } return(metaData); }
/// <summary> /// Open a DSC fileset as a LINQ-to-HPC IQueryable{T}. /// </summary> /// <typeparam name="T">The type of the records in the table.</typeparam> /// <param name="dscFileSetName">The name of the DSC fileset. </param> /// <returns>An IQueryable{T} representing the data and associated with the HPC LINQ query provider.</returns> public IQueryable <T> FromDsc <T>(string fileSetName) { ThrowIfDisposed(); string fullPath = DataPath.MakeDscStreamUri(_dscService, fileSetName); try { DscFileSet fs = _dscService.GetFileSet(fileSetName); if (!fs.IsSealed()) { throw new DryadLinqException(HpcLinqErrorCode.FileSetMustBeSealed, SR.FileSetMustBeSealed); } int fileCount = fs.GetFiles().Count(); if (fileCount < 1) { throw new DryadLinqException(HpcLinqErrorCode.FileSetMustHaveAtLeastOneFile, SR.FileSetMustHaveAtLeastOneFile); } } catch (DscException dscEx) { throw new DryadLinqException(HpcLinqErrorCode.FileSetCouldNotBeOpened, SR.FileSetCouldNotBeOpened, dscEx); } DryadLinqQuery <T> q = DataProvider.GetPartitionedTable <T>(this, fullPath); q.CheckAndInitialize(); // force the data-info checks. return(q); }
// ingresses data, and also sets the temporary-length lease. internal static DryadLinqQuery <T> IngressTemporaryDataDirectlyToDsc <T>(HpcLinqContext context, IEnumerable <T> source, string dscFileSetName, DryadLinqMetaData metaData, DscCompressionScheme outputScheme) { DryadLinqQuery <T> result = IngressDataDirectlyToDsc(context, source, dscFileSetName, metaData, outputScheme); // try to set a temporary lease on the resulting fileset try { DscFileSet fs = context.DscService.GetFileSet(dscFileSetName); fs.SetLeaseEndTime(DateTime.Now.Add(StaticConfig.LeaseDurationForTempFiles)); } catch (DscException) { // suppress } return(result); }
public IEnumerator <T> GetEnumerator() { List <string[]> filePathList; // a list of dsc files, each of which is represented by an array holding the replica paths DscCompressionScheme compressionScheme; try { DscFileSet fileSet = m_context.DscService.GetFileSet(m_fileSetName); filePathList = fileSet.GetFiles().Select(file => file.ReadPaths).ToList(); DryadLinqMetaData metaData = DryadLinqMetaData.FromDscStream(m_context, m_fileSetName); compressionScheme = metaData.CompressionScheme; } catch (Exception e) { throw new DryadLinqException(HpcLinqErrorCode.FailedToGetReadPathsForStream, String.Format(SR.FailedToGetReadPathsForStream, this.m_fileSetName), e); } return(new TableEnumerator(m_context, filePathList, m_fileSetName, compressionScheme)); }
public DscIOStream(string streamName, FileAccess access, FileMode createMode, DscCompressionScheme compressionScheme) { if (String.IsNullOrEmpty(streamName)) { throw new ArgumentNullException("streamName"); } Uri streamUri = new Uri(streamName); this.m_dscClient = new DscService(streamUri.Host); this.m_fileSetName = streamUri.LocalPath.TrimStart('/'); this.m_mode = access; this.m_compressionScheme = compressionScheme; bool streamExists = this.m_dscClient.FileSetExists(this.m_fileSetName); if (access == FileAccess.Read) { switch (createMode) { case FileMode.Open: case FileMode.OpenOrCreate: if (!streamExists) { throw new FileNotFoundException(String.Format( SR.StreamDoesNotExist , streamName)); } break; case FileMode.Append: case FileMode.Create: case FileMode.CreateNew: case FileMode.Truncate: throw new NotSupportedException(); } this.m_dscFileSet = this.m_dscClient.GetFileSet(streamName); this.m_dscFileEnumerator = this.m_dscFileSet.GetFiles().GetEnumerator(); } else if (access == FileAccess.Write) { switch (createMode) { case FileMode.Append: if (!streamExists) { this.m_dscFileSet = this.m_dscClient.CreateFileSet(this.m_fileSetName, this.m_compressionScheme); } break; case FileMode.Create: if (streamExists) { this.m_dscClient.DeleteFileSet(this.m_fileSetName); } this.m_dscFileSet = this.m_dscClient.CreateFileSet(this.m_fileSetName, this.m_compressionScheme); break; case FileMode.CreateNew: if (streamExists) { throw new IOException(String.Format(SR.StreamAlreadyExists, streamName)); } break; case FileMode.Truncate: if (streamExists) { this.m_dscClient.DeleteFileSet(this.m_fileSetName); } this.m_dscFileSet = this.m_dscClient.CreateFileSet(this.m_fileSetName, this.m_compressionScheme); break; case FileMode.Open: case FileMode.OpenOrCreate: // TODO: this should be dealt with correctly, // although it's not obvious what open should do throw new NotSupportedException(); } } else { throw new ArgumentException(SR.ReadWriteNotSupported, "access"); } this.m_fstream = null; this.m_atEOF = false; }
public DscIOStream(string streamName, FileAccess access, DscCompressionScheme compressionScheme) { if (String.IsNullOrEmpty(streamName)) { throw new ArgumentNullException("streamName"); } Uri streamUri = new Uri(streamName); this.m_dscClient = new DscService(streamUri.Host); this.m_fileSetName = streamUri.LocalPath; this.m_mode = access; this.m_fstream = null; this.m_atEOF = false; this.m_compressionScheme = compressionScheme; if (access == FileAccess.Read) { this.m_dscFileSet = this.m_dscClient.GetFileSet(streamName); this.m_dscFileEnumerator = this.m_dscFileSet.GetFiles().GetEnumerator(); } else if (access == FileAccess.Write) { this.m_dscFileSet = this.m_dscClient.CreateFileSet(streamName, compressionScheme); } else { throw new ArgumentException(SR.ReadWriteNotSupported, "access"); } }
//* streams plain enumerable data directly to DSC internal static DryadLinqQuery <T> IngressDataDirectlyToDsc <T>(HpcLinqContext context, IEnumerable <T> source, string dscFileSetName, DryadLinqMetaData metaData, DscCompressionScheme outputScheme) { try { string dscFileSetUri = DataPath.MakeDscStreamUri(context.DscService.HostName, dscFileSetName); if (source.Take(1).Count() == 0) { //there is no data.. we must create a FileSet with an empty file //(the factory/stream approach opens files lazily and thus never opens a file if there is no data) if (context.DscService.FileSetExists(dscFileSetName)) { context.DscService.DeleteFileSet(dscFileSetName); } DscFileSet fileSet = context.DscService.CreateFileSet(dscFileSetName, outputScheme); DscFile file = fileSet.AddNewFile(0); string writePath = file.WritePath; if (outputScheme == DscCompressionScheme.Gzip) { //even zero-byte file must go through the gzip-compressor (for headers etc). using (Stream s = new FileStream(writePath, FileMode.Create)) { var gzipStream = new GZipStream(s, CompressionMode.Compress); gzipStream.Close(); } } else { StreamWriter sw = new StreamWriter(writePath, false); sw.Close(); } fileSet.Seal(); } else { HpcLinqFactory <T> factory = (HpcLinqFactory <T>)HpcLinqCodeGen.GetFactory(context, typeof(T)); // new DscBlockStream(uri,Create,Write,compress) provides a DSC stream with one partition. NativeBlockStream nativeStream = new DscBlockStream(dscFileSetUri, FileMode.Create, FileAccess.Write, outputScheme); HpcRecordWriter <T> writer = factory.MakeWriter(nativeStream); try { if (context.Configuration.AllowConcurrentUserDelegatesInSingleProcess) { foreach (T item in source) { writer.WriteRecordAsync(item); } } else { foreach (T item in source) { writer.WriteRecordSync(item); } } } finally { writer.Close(); // closes the NativeBlockStream, which seals the dsc stream. } } if (metaData != null) { DscFileSet fileSet = context.DscService.GetFileSet(dscFileSetName); fileSet.SetMetadata(DryadLinqMetaData.RECORD_TYPE_NAME, Encoding.UTF8.GetBytes(metaData.ElemType.AssemblyQualifiedName)); } return(DataProvider.GetPartitionedTable <T>(context, dscFileSetUri)); } catch { // if we had a problem creating the empty fileset, try to delete it to avoid cruft being left in DSC. try { context.DscService.DeleteFileSet(dscFileSetName); } catch { // suppress error during delete } throw; // rethrow the original exception. } }