protected void CopyToInternal(DryadLinqQuery otherQuery) { otherQuery.m_queryProvider = this.m_queryProvider; otherQuery.m_dataProvider = this.m_dataProvider; otherQuery.m_isTemp = this.m_isTemp; otherQuery.m_queryExecutor = this.m_queryExecutor; }
/// <summary> /// Open a DSC fileset as a LINQ-to-HPC IQueryable{T}. /// </summary> /// <typeparam name="T">The type of the records in the table.</typeparam> /// <param name="dscFileSetName">The name of the DSC fileset. </param> /// <returns>An IQueryable{T} representing the data and associated with the HPC LINQ query provider.</returns> public IQueryable <T> FromDsc <T>(string fileSetName) { ThrowIfDisposed(); string fullPath = DataPath.MakeDscStreamUri(_dscService, fileSetName); try { DscFileSet fs = _dscService.GetFileSet(fileSetName); if (!fs.IsSealed()) { throw new DryadLinqException(HpcLinqErrorCode.FileSetMustBeSealed, SR.FileSetMustBeSealed); } int fileCount = fs.GetFiles().Count(); if (fileCount < 1) { throw new DryadLinqException(HpcLinqErrorCode.FileSetMustHaveAtLeastOneFile, SR.FileSetMustHaveAtLeastOneFile); } } catch (DscException dscEx) { throw new DryadLinqException(HpcLinqErrorCode.FileSetCouldNotBeOpened, SR.FileSetCouldNotBeOpened, dscEx); } DryadLinqQuery <T> q = DataProvider.GetPartitionedTable <T>(this, fullPath); q.CheckAndInitialize(); // force the data-info checks. return(q); }
/// <summary> /// Open a dataset as a DryadLinq specialized IQueryable{T}. /// </summary> /// <typeparam name="T">The type of the records in the table.</typeparam> /// <param name="dataSetUri">The name of the dataset. </param> /// <returns>An IQueryable{T} representing the data.</returns> public IQueryable <T> FromStore <T>(Uri dataSetUri) { ThrowIfDisposed(); DryadLinqQuery <T> q = DataProvider.GetPartitionedTable <T>(this, dataSetUri); q.CheckAndInitialize(); // force the data-info checks. return(q); }
protected void CloneBase(DryadLinqQuery otherQuery) { if (otherQuery.m_queryProvider == null) { otherQuery.m_queryProvider = this.m_queryProvider; } if (otherQuery.m_dataProvider == null) { otherQuery.m_dataProvider = this.m_dataProvider; } otherQuery.m_isTemp = this.m_isTemp; otherQuery.m_queryExecutor = this.m_queryExecutor; }
// ingresses data, and also sets the temporary-length lease. internal static DryadLinqQuery <T> IngressTemporaryDataDirectlyToDsc <T>(HpcLinqContext context, IEnumerable <T> source, string dscFileSetName, DryadLinqMetaData metaData, DscCompressionScheme outputScheme) { DryadLinqQuery <T> result = IngressDataDirectlyToDsc(context, source, dscFileSetName, metaData, outputScheme); // try to set a temporary lease on the resulting fileset try { DscFileSet fs = context.DscService.GetFileSet(dscFileSetName); fs.SetLeaseEndTime(DateTime.Now.Add(StaticConfig.LeaseDurationForTempFiles)); } catch (DscException) { // suppress } return(result); }
/// <summary> /// Stores an IEnumerable{T} at a specified location. /// </summary> /// <typeparam name="T">The record type of the data.</typeparam> /// <param name="context">An instance of <see cref="DryadLinqContext"/></param> /// <param name="source">The data to store.</param> /// <param name="dataSetUri">The URI of the store location.</param> /// <param name="metaData">The metadata of the data.</param> /// <param name="outputScheme">The compression scheme.</param> /// <param name="isTemp">true if the data is only stored temporarily.</param> /// <param name="serializer">A stream-based serializer</param> /// <param name="deserializer">A stream-based deserializer</param> /// <returns>An instance of IQueryable{T} for the data.</returns> internal static DryadLinqQuery <T> StoreData <T>(DryadLinqContext context, IEnumerable <T> source, Uri dataSetUri, DryadLinqMetaData metaData, CompressionScheme outputScheme, bool isTemp, Expression <Action <IEnumerable <T>, Stream> > serializer, Expression <Func <Stream, IEnumerable <T> > > deserializer) { string scheme = DataPath.GetScheme(dataSetUri); DataProvider dataProvider = DataProvider.GetDataProvider(scheme); dataSetUri = dataProvider.RewriteUri <T>(context, dataSetUri); dataProvider.Ingress(context, source, dataSetUri, metaData, outputScheme, isTemp, serializer); DryadLinqQuery <T> res = DataProvider.GetPartitionedTable <T>(context, dataSetUri, deserializer); res.CheckAndInitialize(); // must initialize return(res); }