/// <exception cref="System.Exception"/>
public bool Call()
{
Decompressor dc = queue.Take();
CodecPool.ReturnDecompressor(dc);
return(dc != null);
}
public virtual void TestCodecPoolAndGzipDecompressor()
{
// BuiltInZlibInflater should not be used as the GzipCodec decompressor.
// Assert that this is the case.
// Don't use native libs for this test.
Configuration conf = new Configuration();
conf.SetBoolean("hadoop.native.lib", false);
NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request"
, ZlibFactory.IsNativeZlibLoaded(conf));
// This should give us a BuiltInZlibInflater.
Decompressor zlibDecompressor = ZlibFactory.GetZlibDecompressor(conf);
NUnit.Framework.Assert.IsNotNull("zlibDecompressor is null!", zlibDecompressor);
Assert.True("ZlibFactory returned unexpected inflator", zlibDecompressor
is BuiltInZlibInflater);
// its createOutputStream() just wraps the existing stream in a
// java.util.zip.GZIPOutputStream.
CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
CompressionCodec codec = ccf.GetCodec(new Path("foo.gz"));
Assert.True("Codec for .gz file is not GzipCodec", codec is GzipCodec
);
// make sure we don't get a null decompressor
Decompressor codecDecompressor = codec.CreateDecompressor();
if (null == codecDecompressor)
{
NUnit.Framework.Assert.Fail("Got null codecDecompressor");
}
// Asking the CodecPool for a decompressor for GzipCodec
// should not return null
Decompressor poolDecompressor = CodecPool.GetDecompressor(codec);
if (null == poolDecompressor)
{
NUnit.Framework.Assert.Fail("Got null poolDecompressor");
}
// return a couple decompressors
CodecPool.ReturnDecompressor(zlibDecompressor);
CodecPool.ReturnDecompressor(poolDecompressor);
Decompressor poolDecompressor2 = CodecPool.GetDecompressor(codec);
if (poolDecompressor.GetType() == typeof(BuiltInGzipDecompressor))
{
if (poolDecompressor == poolDecompressor2)
{
NUnit.Framework.Assert.Fail("Reused java gzip decompressor in pool");
}
}
else
{
if (poolDecompressor != poolDecompressor2)
{
NUnit.Framework.Assert.Fail("Did not reuse native gzip decompressor in pool");
}
}
}
/// <exception cref="System.IO.IOException"/>
public override void Close()
{
@in.Close();
if (trackedDecompressor != null)
{
CodecPool.ReturnDecompressor(trackedDecompressor);
trackedDecompressor = null;
}
}
public virtual void TestGzipLongOverflow()
{
Log.Info("testGzipLongOverflow");
// Don't use native libs for this test.
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false);
NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request"
, ZlibFactory.IsNativeZlibLoaded(conf));
// Ensure that the CodecPool has a BuiltInZlibInflater in it.
Decompressor zlibDecompressor = ZlibFactory.GetZlibDecompressor(conf);
NUnit.Framework.Assert.IsNotNull("zlibDecompressor is null!", zlibDecompressor);
Assert.True("ZlibFactory returned unexpected inflator", zlibDecompressor
is BuiltInZlibInflater);
CodecPool.ReturnDecompressor(zlibDecompressor);
// Now create a GZip text file.
string tmpDir = Runtime.GetProperty("test.build.data", "/tmp/");
Path f = new Path(new Path(tmpDir), "testGzipLongOverflow.bin.gz");
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream
(new FileOutputStream(f.ToString()))));
int Nbuf = 1024 * 4 + 1;
char[] buf = new char[1024 * 1024];
for (int i = 0; i < buf.Length; i++)
{
buf[i] = '\0';
}
for (int i_1 = 0; i_1 < Nbuf; i_1++)
{
bw.Write(buf);
}
bw.Close();
// Now read it back, using the CodecPool to establish the
// decompressor to use.
CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
CompressionCodec codec = ccf.GetCodec(f);
Decompressor decompressor = CodecPool.GetDecompressor(codec);
FileSystem fs = FileSystem.GetLocal(conf);
InputStream @is = fs.Open(f);
@is = codec.CreateInputStream(@is, decompressor);
BufferedReader br = new BufferedReader(new InputStreamReader(@is));
for (int j = 0; j < Nbuf; j++)
{
int n = br.Read(buf);
Assert.Equal("got wrong read length!", n, buf.Length);
for (int i_2 = 0; i_2 < buf.Length; i_2++)
{
Assert.Equal("got wrong byte!", buf[i_2], '\0');
}
}
br.Close();
}
public virtual void TestDecompressorPoolCounts()
{
// Get two decompressors and return them
Decompressor decomp1 = CodecPool.GetDecompressor(codec);
Decompressor decomp2 = CodecPool.GetDecompressor(codec);
Assert.Equal(LeaseCountErr, 2, CodecPool.GetLeasedDecompressorsCount
(codec));
CodecPool.ReturnDecompressor(decomp2);
Assert.Equal(LeaseCountErr, 1, CodecPool.GetLeasedDecompressorsCount
(codec));
CodecPool.ReturnDecompressor(decomp1);
Assert.Equal(LeaseCountErr, 0, CodecPool.GetLeasedDecompressorsCount
(codec));
CodecPool.ReturnDecompressor(decomp1);
Assert.Equal(LeaseCountErr, 0, CodecPool.GetLeasedCompressorsCount
(codec));
}
public virtual void TestDecompressorNotReturnSameInstance()
{
Decompressor decomp = CodecPool.GetDecompressor(codec);
CodecPool.ReturnDecompressor(decomp);
CodecPool.ReturnDecompressor(decomp);
ICollection <Decompressor> decompressors = new HashSet <Decompressor>();
for (int i = 0; i < 10; ++i)
{
decompressors.AddItem(CodecPool.GetDecompressor(codec));
}
Assert.Equal(10, decompressors.Count);
foreach (Decompressor decompressor in decompressors)
{
CodecPool.ReturnDecompressor(decompressor);
}
}
public virtual void TestGzipCodecRead()
{
// Create a gzipped file and try to read it back, using a decompressor
// from the CodecPool.
// Don't use native libs for this test.
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false);
NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request"
, ZlibFactory.IsNativeZlibLoaded(conf));
// Ensure that the CodecPool has a BuiltInZlibInflater in it.
Decompressor zlibDecompressor = ZlibFactory.GetZlibDecompressor(conf);
NUnit.Framework.Assert.IsNotNull("zlibDecompressor is null!", zlibDecompressor);
Assert.True("ZlibFactory returned unexpected inflator", zlibDecompressor
is BuiltInZlibInflater);
CodecPool.ReturnDecompressor(zlibDecompressor);
// Now create a GZip text file.
string tmpDir = Runtime.GetProperty("test.build.data", "/tmp/");
Path f = new Path(new Path(tmpDir), "testGzipCodecRead.txt.gz");
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream
(new FileOutputStream(f.ToString()))));
string msg = "This is the message in the file!";
bw.Write(msg);
bw.Close();
// Now read it back, using the CodecPool to establish the
// decompressor to use.
CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
CompressionCodec codec = ccf.GetCodec(f);
Decompressor decompressor = CodecPool.GetDecompressor(codec);
FileSystem fs = FileSystem.GetLocal(conf);
InputStream @is = fs.Open(f);
@is = codec.CreateInputStream(@is, decompressor);
BufferedReader br = new BufferedReader(new InputStreamReader(@is));
string line = br.ReadLine();
Assert.Equal("Didn't get the same message back!", msg, line);
br.Close();
}
/// <summary>Create an input stream with a codec taken from the global CodecPool.</summary>
/// <param name="codec">The codec to use to create the input stream.</param>
/// <param name="conf">The configuration to use if we need to create a new codec.</param>
/// <param name="in">The input stream to wrap.</param>
/// <returns>The new input stream</returns>
/// <exception cref="System.IO.IOException"/>
internal static CompressionInputStream CreateInputStreamWithCodecPool(CompressionCodec
codec, Configuration conf, InputStream @in)
{
Decompressor decompressor = CodecPool.GetDecompressor(codec);
CompressionInputStream stream = null;
try
{
stream = codec.CreateInputStream(@in, decompressor);
}
finally
{
if (stream == null)
{
CodecPool.ReturnDecompressor(decompressor);
}
else
{
stream.SetTrackedDecompressor(decompressor);
}
}
return(stream);
}
/// <exception cref="System.IO.IOException"/>
private void TestSplitableCodec(Type codecClass)
{
long Deflbytes = 2 * 1024 * 1024;
Configuration conf = new Configuration();
Random rand = new Random();
long seed = rand.NextLong();
Log.Info("seed: " + seed);
rand.SetSeed(seed);
SplittableCompressionCodec codec = ReflectionUtils.NewInstance(codecClass, conf);
FileSystem fs = FileSystem.GetLocal(conf);
FileStatus infile = fs.GetFileStatus(WriteSplitTestFile(fs, rand, codec, Deflbytes
));
if (infile.GetLen() > int.MaxValue)
{
NUnit.Framework.Assert.Fail("Unexpected compression: " + Deflbytes + " -> " + infile
.GetLen());
}
int flen = (int)infile.GetLen();
Text line = new Text();
Decompressor dcmp = CodecPool.GetDecompressor(codec);
try
{
for (int pos = 0; pos < infile.GetLen(); pos += rand.Next(flen / 8))
{
// read from random positions, verifying that there exist two sequential
// lines as written in writeSplitTestFile
SplitCompressionInputStream @in = codec.CreateInputStream(fs.Open(infile.GetPath(
)), dcmp, pos, flen, SplittableCompressionCodec.READ_MODE.Byblock);
if (@in.GetAdjustedStart() >= flen)
{
break;
}
Log.Info("SAMPLE " + @in.GetAdjustedStart() + "," + @in.GetAdjustedEnd());
LineReader lreader = new LineReader(@in);
lreader.ReadLine(line);
// ignore; likely partial
if (@in.GetPos() >= flen)
{
break;
}
lreader.ReadLine(line);
int seq1 = ReadLeadingInt(line);
lreader.ReadLine(line);
if (@in.GetPos() >= flen)
{
break;
}
int seq2 = ReadLeadingInt(line);
Assert.Equal("Mismatched lines", seq1 + 1, seq2);
}
}
finally
{
CodecPool.ReturnDecompressor(dcmp);
}
// remove on success
fs.Delete(infile.GetPath().GetParent(), true);
}
