/// <summary> /// Stores an IEnumerable{T} at a specified location. /// </summary> /// <typeparam name="T">The record type of the data.</typeparam> /// <param name="context">An instance of <see cref="DryadLinqContext"/></param> /// <param name="source">The data to store.</param> /// <param name="dataSetUri">The URI of the store location.</param> /// <param name="metaData">The metadata of the data.</param> /// <param name="outputScheme">The compression scheme.</param> /// <param name="isTemp">true if the data is only stored temporarily.</param> /// <returns>An instance of IQueryable{T} for the data.</returns> internal static DryadLinqQuery <T> StoreData <T>(DryadLinqContext context, IEnumerable <T> source, Uri dataSetUri, DryadLinqMetaData metaData, CompressionScheme outputScheme, bool isTemp = false) { string scheme = DataPath.GetScheme(dataSetUri); DataProvider dataProvider = DataProvider.GetDataProvider(scheme); dataSetUri = dataProvider.RewriteUri <T>(context, dataSetUri); dataProvider.Ingress(context, source, dataSetUri, metaData, outputScheme, isTemp); return(DataProvider.GetPartitionedTable <T>(context, dataSetUri)); }
/// <summary> /// Stores an IEnumerable{T} at a specified location. /// </summary> /// <typeparam name="T">The record type of the data.</typeparam> /// <param name="context">An instance of <see cref="DryadLinqContext"/></param> /// <param name="source">The data to store.</param> /// <param name="dataSetUri">The URI of the store location.</param> /// <param name="metaData">The metadata of the data.</param> /// <param name="outputScheme">The compression scheme.</param> /// <param name="isTemp">true if the data is only stored temporarily.</param> /// <param name="serializer">A stream-based serializer</param> /// <param name="deserializer">A stream-based deserializer</param> /// <returns>An instance of IQueryable{T} for the data.</returns> internal static DryadLinqQuery <T> StoreData <T>(DryadLinqContext context, IEnumerable <T> source, Uri dataSetUri, DryadLinqMetaData metaData, CompressionScheme outputScheme, bool isTemp, Expression <Action <IEnumerable <T>, Stream> > serializer, Expression <Func <Stream, IEnumerable <T> > > deserializer) { string scheme = DataPath.GetScheme(dataSetUri); DataProvider dataProvider = DataProvider.GetDataProvider(scheme); dataSetUri = dataProvider.RewriteUri <T>(context, dataSetUri); dataProvider.Ingress(context, source, dataSetUri, metaData, outputScheme, isTemp, serializer); DryadLinqQuery <T> res = DataProvider.GetPartitionedTable <T>(context, dataSetUri, deserializer); res.CheckAndInitialize(); // must initialize return(res); }
// create DryadLinqMetaData from a query OutputNode internal static DryadLinqMetaData FromOutputNode(HpcLinqContext context, DryadOutputNode node) { DryadLinqMetaData metaData = new DryadLinqMetaData(); if (!(DataPath.IsDsc(node.MetaDataUri) || DataPath.IsHdfs(node.MetaDataUri))) { throw new InvalidOperationException(); } metaData.m_context = context; metaData.m_dscStreamName = node.MetaDataUri; metaData.m_elemType = node.OutputTypes[0]; metaData.m_compressionScheme = node.OutputCompressionScheme; //metaData.m_version = context.ClientVersion; //metaData.InitializeFlags(); //metaData.m_fp = 0UL; //metaData.m_dataSetInfo = node.OutputDataSetInfo; return(metaData); }
// create DryadLinqMetaData from a query OutputNode internal static DryadLinqMetaData Get(DryadLinqContext context, DLinqOutputNode node) { DryadLinqMetaData metaData = new DryadLinqMetaData(); if (!DataPath.IsValidDataPath(node.OutputUri)) { throw new InvalidOperationException(); } metaData.m_context = context; metaData.m_dataSetUri = node.OutputUri; metaData.m_elemType = node.OutputTypes[0]; metaData.m_compressionScheme = node.OutputCompressionScheme; //metaData.m_version = context.ClientVersion(); //metaData.InitializeFlags(); //metaData.m_fp = 0UL; //metaData.m_dataSetInfo = node.OutputDataSetInfo; return(metaData); }
//* streams plain enumerable data directly to DSC internal static DryadLinqQuery <T> IngressDataDirectlyToDsc <T>(HpcLinqContext context, IEnumerable <T> source, string dscFileSetName, DryadLinqMetaData metaData, DscCompressionScheme outputScheme) { try { string dscFileSetUri = DataPath.MakeDscStreamUri(context.DscService.HostName, dscFileSetName); if (source.Take(1).Count() == 0) { //there is no data.. we must create a FileSet with an empty file //(the factory/stream approach opens files lazily and thus never opens a file if there is no data) if (context.DscService.FileSetExists(dscFileSetName)) { context.DscService.DeleteFileSet(dscFileSetName); } DscFileSet fileSet = context.DscService.CreateFileSet(dscFileSetName, outputScheme); DscFile file = fileSet.AddNewFile(0); string writePath = file.WritePath; if (outputScheme == DscCompressionScheme.Gzip) { //even zero-byte file must go through the gzip-compressor (for headers etc). using (Stream s = new FileStream(writePath, FileMode.Create)) { var gzipStream = new GZipStream(s, CompressionMode.Compress); gzipStream.Close(); } } else { StreamWriter sw = new StreamWriter(writePath, false); sw.Close(); } fileSet.Seal(); } else { HpcLinqFactory <T> factory = (HpcLinqFactory <T>)HpcLinqCodeGen.GetFactory(context, typeof(T)); // new DscBlockStream(uri,Create,Write,compress) provides a DSC stream with one partition. NativeBlockStream nativeStream = new DscBlockStream(dscFileSetUri, FileMode.Create, FileAccess.Write, outputScheme); HpcRecordWriter <T> writer = factory.MakeWriter(nativeStream); try { if (context.Configuration.AllowConcurrentUserDelegatesInSingleProcess) { foreach (T item in source) { writer.WriteRecordAsync(item); } } else { foreach (T item in source) { writer.WriteRecordSync(item); } } } finally { writer.Close(); // closes the NativeBlockStream, which seals the dsc stream. } } if (metaData != null) { DscFileSet fileSet = context.DscService.GetFileSet(dscFileSetName); fileSet.SetMetadata(DryadLinqMetaData.RECORD_TYPE_NAME, Encoding.UTF8.GetBytes(metaData.ElemType.AssemblyQualifiedName)); } return(DataProvider.GetPartitionedTable <T>(context, dscFileSetUri)); } catch { // if we had a problem creating the empty fileset, try to delete it to avoid cruft being left in DSC. try { context.DscService.DeleteFileSet(dscFileSetName); } catch { // suppress error during delete } throw; // rethrow the original exception. } }
internal static string MakeUniqueTemporaryHdfsFileSetUri(HpcLinqContext context) { string uri = DataPath.MakeHdfsStreamUri(context.Configuration.HdfsNameNode, MakeUniqueTemporaryDscFileSetName()); return(uri); }
internal static string MakeUniqueTemporaryDscFileSetUri(HpcLinqContext context) { string uri = DataPath.MakeDscStreamUri(context.DscService.HostName, MakeUniqueTemporaryDscFileSetName()); return(uri); }