internal static DryadLinqMetaData Get(DryadLinqContext context, Uri dataSetUri) { string scheme = DataPath.GetScheme(dataSetUri); DataProvider dataProvider = DataProvider.GetDataProvider(scheme); return(dataProvider.GetMetaData(context, dataSetUri)); }
/// <summary> /// Reads a specified dataset. /// </summary> /// <typeparam name="T">The record type of the dataset.</typeparam> /// <param name="context">An instance of <see cref="DryadLinqContext"/></param> /// <param name="dataSetUri">The URI of the dataset.</param> /// <returns>A sequence of records as IEnumerable{T}.</returns> public static IEnumerable <T> ReadData <T>(DryadLinqContext context, Uri dataSetUri) { string scheme = DataPath.GetScheme(dataSetUri); DataProvider dataProvider = DataProvider.GetDataProvider(scheme); dataSetUri = dataProvider.RewriteUri <T>(context, dataSetUri); return(new DryadLinqQueryEnumerable <T>(context, dataProvider, dataSetUri)); }
/// <summary> /// Get the dataset specified by a URI. /// </summary> /// <typeparam name="T">The record type of the dataset.</typeparam> /// <param name="context">An instance of <see cref="DryadLinqContext"/></param> /// <param name="dataSetUri">The URI of the dataset</param> /// <returns>A query object representing the dsc file set data.</returns> internal static DryadLinqQuery <T> GetPartitionedTable <T>(DryadLinqContext context, Uri dataSetUri) { string scheme = DataPath.GetScheme(dataSetUri); DataProvider dataProvider = DataProvider.GetDataProvider(scheme); DryadLinqProvider queryProvider = new DryadLinqProvider(context); dataSetUri = dataProvider.RewriteUri <T>(context, dataSetUri); return(new DryadLinqQuery <T>(null, queryProvider, dataProvider, dataSetUri)); }
/// <summary> /// Reads the dataset specified by a URI. /// </summary> /// <typeparam name="T">The record type of the dataset</typeparam> /// <param name="context">An instance of <see cref="DryadLinqContext"/></param> /// <param name="dataSetUri">The URI of the dataset</param> /// <param name="deserializer">A stream-based deserializer</param> /// <returns>A sequence of records as IEnumerable{T}</returns> internal static IEnumerable <T> ReadData <T>(DryadLinqContext context, Uri dataSetUri, Expression <Func <Stream, IEnumerable <T> > > deserializer) { string scheme = DataPath.GetScheme(dataSetUri); DataProvider dataProvider = DataProvider.GetDataProvider(scheme); dataSetUri = dataProvider.RewriteUri <T>(context, dataSetUri); return(new DryadLinqQueryEnumerable <T>(context, dataProvider, dataSetUri, deserializer)); }
/// <summary> /// Stores an IEnumerable{T} at a specified location. /// </summary> /// <typeparam name="T">The record type of the data.</typeparam> /// <param name="context">An instance of <see cref="DryadLinqContext"/></param> /// <param name="source">The data to store.</param> /// <param name="dataSetUri">The URI of the store location.</param> /// <param name="metaData">The metadata of the data.</param> /// <param name="outputScheme">The compression scheme.</param> /// <param name="isTemp">true if the data is only stored temporarily.</param> /// <returns>An instance of IQueryable{T} for the data.</returns> internal static DryadLinqQuery <T> StoreData <T>(DryadLinqContext context, IEnumerable <T> source, Uri dataSetUri, DryadLinqMetaData metaData, CompressionScheme outputScheme, bool isTemp = false) { string scheme = DataPath.GetScheme(dataSetUri); DataProvider dataProvider = DataProvider.GetDataProvider(scheme); dataSetUri = dataProvider.RewriteUri <T>(context, dataSetUri); dataProvider.Ingress(context, source, dataSetUri, metaData, outputScheme, isTemp); return(DataProvider.GetPartitionedTable <T>(context, dataSetUri)); }
/// <summary> /// Stores an IEnumerable{T} at a specified location. /// </summary> /// <typeparam name="T">The record type of the data.</typeparam> /// <param name="context">An instance of <see cref="DryadLinqContext"/></param> /// <param name="source">The data to store.</param> /// <param name="dataSetUri">The URI of the store location.</param> /// <param name="metaData">The metadata of the data.</param> /// <param name="outputScheme">The compression scheme.</param> /// <param name="isTemp">true if the data is only stored temporarily.</param> /// <param name="serializer">A stream-based serializer</param> /// <param name="deserializer">A stream-based deserializer</param> /// <returns>An instance of IQueryable{T} for the data.</returns> internal static DryadLinqQuery <T> StoreData <T>(DryadLinqContext context, IEnumerable <T> source, Uri dataSetUri, DryadLinqMetaData metaData, CompressionScheme outputScheme, bool isTemp, Expression <Action <IEnumerable <T>, Stream> > serializer, Expression <Func <Stream, IEnumerable <T> > > deserializer) { string scheme = DataPath.GetScheme(dataSetUri); DataProvider dataProvider = DataProvider.GetDataProvider(scheme); dataSetUri = dataProvider.RewriteUri <T>(context, dataSetUri); dataProvider.Ingress(context, source, dataSetUri, metaData, outputScheme, isTemp, serializer); DryadLinqQuery <T> res = DataProvider.GetPartitionedTable <T>(context, dataSetUri, deserializer); res.CheckAndInitialize(); // must initialize return(res); }