public DataIndex(string defaultCompressionAlgorithmName)
{
// for write
this.defaultCompressionAlgorithm = Compression.GetCompressionAlgorithmByName(defaultCompressionAlgorithmName
);
listRegions = new AList <BCFile.BlockRegion>();
}
public MetaIndexEntry(string metaName, Compression.Algorithm compressionAlgorithm
, BCFile.BlockRegion region)
{
this.metaName = metaName;
this.compressionAlgorithm = compressionAlgorithm;
this.region = region;
}
/// <exception cref="System.IO.IOException"/>
private BCFile.Reader.BlockReader CreateReader(Compression.Algorithm compressAlgo
, BCFile.BlockRegion region)
{
BCFile.Reader.RBlockState rbs = new BCFile.Reader.RBlockState(compressAlgo, @in,
region, conf);
return(new BCFile.Reader.BlockReader(rbs));
}
internal MetaBlockRegister(Writer _enclosing, string name, Compression.Algorithm
compressAlgo)
{
this._enclosing = _enclosing;
this.name = name;
this.compressAlgo = compressAlgo;
}
/// <exception cref="System.IO.IOException"/>
public DataIndex(BinaryReader reader)
{
// for data blocks, each entry specifies a block's offset, compressed size
// and raw size
// for read, deserialized from a file
defaultCompressionAlgorithm = Compression.GetCompressionAlgorithmByName(Utils.ReadString
(@in));
int n = Utils.ReadVInt(@in);
listRegions = new AList <BCFile.BlockRegion>(n);
for (int i = 0; i < n; i++)
{
BCFile.BlockRegion region = new BCFile.BlockRegion(@in);
listRegions.AddItem(region);
}
}
/// <exception cref="System.IO.IOException"/>
public MetaIndexEntry(BinaryReader reader)
{
string fullMetaName = Utils.ReadString(@in);
if (fullMetaName.StartsWith(defaultPrefix))
{
metaName = Runtime.Substring(fullMetaName, defaultPrefix.Length, fullMetaName
.Length);
}
else
{
throw new IOException("Corrupted Meta region Index");
}
compressionAlgorithm = Compression.GetCompressionAlgorithmByName(Utils.ReadString
(@in));
region = new BCFile.BlockRegion(@in);
}
/// <exception cref="System.IO.IOException"/>
public RBlockState(Compression.Algorithm compressionAlgo, FSDataInputStream fsin,
BCFile.BlockRegion region, Configuration conf)
{
// Index for meta blocks
this.compressAlgo = compressionAlgo;
this.region = region;
this.decompressor = compressionAlgo.GetDecompressor();
try
{
this.@in = compressAlgo.CreateDecompressionStream(new BoundedRangeFileInputStream
(fsin, this.region.GetOffset(), this.region.GetCompressedSize()), decompressor,
TFile.GetFSInputBufferSize(conf));
}
catch (IOException e)
{
compressAlgo.ReturnDecompressor(decompressor);
throw;
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.IO.File.Tfile.MetaBlockAlreadyExists"/>
private BCFile.Writer.BlockAppender PrepareMetaBlock(string name, Compression.Algorithm
compressAlgo)
{
if (blkInProgress == true)
{
throw new InvalidOperationException("Cannot create Meta Block until previous block is closed."
);
}
if (metaIndex.GetMetaByName(name) != null)
{
throw new MetaBlockAlreadyExists("name=" + name);
}
BCFile.Writer.MetaBlockRegister mbr = new BCFile.Writer.MetaBlockRegister(this, name
, compressAlgo);
BCFile.Writer.WBlockState wbs = new BCFile.Writer.WBlockState(compressAlgo, @out,
fsOutputBuffer, conf);
BCFile.Writer.BlockAppender ba = new BCFile.Writer.BlockAppender(this, mbr, wbs);
blkInProgress = true;
metaBlkSeen = true;
return(ba);
}
/// <param name="compressionAlgo">The compression algorithm to be used to for compression.
/// </param>
/// <exception cref="System.IO.IOException"/>
public WBlockState(Compression.Algorithm compressionAlgo, FSDataOutputStream fsOut
, BytesWritable fsOutputBuffer, Configuration conf)
{
// !null only if using native
// Hadoop compression
this.compressAlgo = compressionAlgo;
this.fsOut = fsOut;
this.posStart = fsOut.GetPos();
fsOutputBuffer.Capacity = TFile.GetFSOutputBufferSize(conf);
this.fsBufferedOutput = new SimpleBufferedOutputStream(this.fsOut, fsOutputBuffer
.Bytes);
this.compressor = compressAlgo.GetCompressor();
try
{
this.@out = compressionAlgo.CreateCompressionStream(fsBufferedOutput, compressor,
0);
}
catch (IOException e)
{
compressAlgo.ReturnCompressor(compressor);
throw;
}
}
