public DataIndex(string defaultCompressionAlgorithmName) { // for write this.defaultCompressionAlgorithm = Compression.GetCompressionAlgorithmByName(defaultCompressionAlgorithmName ); listRegions = new AList <BCFile.BlockRegion>(); }
public MetaIndexEntry(string metaName, Compression.Algorithm compressionAlgorithm , BCFile.BlockRegion region) { this.metaName = metaName; this.compressionAlgorithm = compressionAlgorithm; this.region = region; }
/// <exception cref="System.IO.IOException"/> private BCFile.Reader.BlockReader CreateReader(Compression.Algorithm compressAlgo , BCFile.BlockRegion region) { BCFile.Reader.RBlockState rbs = new BCFile.Reader.RBlockState(compressAlgo, @in, region, conf); return(new BCFile.Reader.BlockReader(rbs)); }
internal MetaBlockRegister(Writer _enclosing, string name, Compression.Algorithm compressAlgo) { this._enclosing = _enclosing; this.name = name; this.compressAlgo = compressAlgo; }
/// <exception cref="System.IO.IOException"/> public DataIndex(BinaryReader reader) { // for data blocks, each entry specifies a block's offset, compressed size // and raw size // for read, deserialized from a file defaultCompressionAlgorithm = Compression.GetCompressionAlgorithmByName(Utils.ReadString (@in)); int n = Utils.ReadVInt(@in); listRegions = new AList <BCFile.BlockRegion>(n); for (int i = 0; i < n; i++) { BCFile.BlockRegion region = new BCFile.BlockRegion(@in); listRegions.AddItem(region); } }
/// <exception cref="System.IO.IOException"/> public MetaIndexEntry(BinaryReader reader) { string fullMetaName = Utils.ReadString(@in); if (fullMetaName.StartsWith(defaultPrefix)) { metaName = Runtime.Substring(fullMetaName, defaultPrefix.Length, fullMetaName .Length); } else { throw new IOException("Corrupted Meta region Index"); } compressionAlgorithm = Compression.GetCompressionAlgorithmByName(Utils.ReadString (@in)); region = new BCFile.BlockRegion(@in); }
/// <exception cref="System.IO.IOException"/> public RBlockState(Compression.Algorithm compressionAlgo, FSDataInputStream fsin, BCFile.BlockRegion region, Configuration conf) { // Index for meta blocks this.compressAlgo = compressionAlgo; this.region = region; this.decompressor = compressionAlgo.GetDecompressor(); try { this.@in = compressAlgo.CreateDecompressionStream(new BoundedRangeFileInputStream (fsin, this.region.GetOffset(), this.region.GetCompressedSize()), decompressor, TFile.GetFSInputBufferSize(conf)); } catch (IOException e) { compressAlgo.ReturnDecompressor(decompressor); throw; } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.IO.File.Tfile.MetaBlockAlreadyExists"/> private BCFile.Writer.BlockAppender PrepareMetaBlock(string name, Compression.Algorithm compressAlgo) { if (blkInProgress == true) { throw new InvalidOperationException("Cannot create Meta Block until previous block is closed." ); } if (metaIndex.GetMetaByName(name) != null) { throw new MetaBlockAlreadyExists("name=" + name); } BCFile.Writer.MetaBlockRegister mbr = new BCFile.Writer.MetaBlockRegister(this, name , compressAlgo); BCFile.Writer.WBlockState wbs = new BCFile.Writer.WBlockState(compressAlgo, @out, fsOutputBuffer, conf); BCFile.Writer.BlockAppender ba = new BCFile.Writer.BlockAppender(this, mbr, wbs); blkInProgress = true; metaBlkSeen = true; return(ba); }
/// <param name="compressionAlgo">The compression algorithm to be used to for compression. /// </param> /// <exception cref="System.IO.IOException"/> public WBlockState(Compression.Algorithm compressionAlgo, FSDataOutputStream fsOut , BytesWritable fsOutputBuffer, Configuration conf) { // !null only if using native // Hadoop compression this.compressAlgo = compressionAlgo; this.fsOut = fsOut; this.posStart = fsOut.GetPos(); fsOutputBuffer.Capacity = TFile.GetFSOutputBufferSize(conf); this.fsBufferedOutput = new SimpleBufferedOutputStream(this.fsOut, fsOutputBuffer .Bytes); this.compressor = compressAlgo.GetCompressor(); try { this.@out = compressionAlgo.CreateCompressionStream(fsBufferedOutput, compressor, 0); } catch (IOException e) { compressAlgo.ReturnCompressor(compressor); throw; } }