/// <exception cref="System.IO.IOException"/>
public virtual void TestGzipCodecWrite(bool useNative)
{
// Create a gzipped file using a compressor from the CodecPool,
// and try to read it back via the regular GZIPInputStream.
// Use native libs per the parameter
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, useNative);
if (useNative)
{
if (!ZlibFactory.IsNativeZlibLoaded(conf))
{
Log.Warn("testGzipCodecWrite skipped: native libs not loaded");
return;
}
}
else
{
NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request"
, ZlibFactory.IsNativeZlibLoaded(conf));
}
// Ensure that the CodecPool has a BuiltInZlibDeflater in it.
Compressor zlibCompressor = ZlibFactory.GetZlibCompressor(conf);
NUnit.Framework.Assert.IsNotNull("zlibCompressor is null!", zlibCompressor);
Assert.True("ZlibFactory returned unexpected deflator", useNative
? zlibCompressor is ZlibCompressor : zlibCompressor is BuiltInZlibDeflater);
CodecPool.ReturnCompressor(zlibCompressor);
// Create a GZIP text file via the Compressor interface.
CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
CompressionCodec codec = ccf.GetCodec(new Path("foo.gz"));
Assert.True("Codec for .gz file is not GzipCodec", codec is GzipCodec
);
string msg = "This is the message we are going to compress.";
string tmpDir = Runtime.GetProperty("test.build.data", "/tmp/");
string fileName = new Path(new Path(tmpDir), "testGzipCodecWrite.txt.gz").ToString
();
BufferedWriter w = null;
Compressor gzipCompressor = CodecPool.GetCompressor(codec);
if (null != gzipCompressor)
{
// If it gives us back a Compressor, we should be able to use this
// to write files we can then read back with Java's gzip tools.
OutputStream os = new CompressorStream(new FileOutputStream(fileName), gzipCompressor
);
w = new BufferedWriter(new OutputStreamWriter(os));
w.Write(msg);
w.Close();
CodecPool.ReturnCompressor(gzipCompressor);
VerifyGzipFile(fileName, msg);
}
// Create a gzip text file via codec.getOutputStream().
w = new BufferedWriter(new OutputStreamWriter(codec.CreateOutputStream(new FileOutputStream
(fileName))));
w.Write(msg);
w.Close();
VerifyGzipFile(fileName, msg);
}
/// <exception cref="System.Exception"/>
public bool Call()
{
Decompressor c = CodecPool.GetDecompressor(this._enclosing.codec);
queue.Put(c);
return(c != null);
}
/// <exception cref="System.Exception"/>
public bool Call()
{
Decompressor dc = queue.Take();
CodecPool.ReturnDecompressor(dc);
return(dc != null);
}
/// <exception cref="System.Exception"/>
public bool Call()
{
Compressor c = queue.Take();
CodecPool.ReturnCompressor(c);
return(c != null);
}
public virtual void TestCodecPoolAndGzipDecompressor()
{
// BuiltInZlibInflater should not be used as the GzipCodec decompressor.
// Assert that this is the case.
// Don't use native libs for this test.
Configuration conf = new Configuration();
conf.SetBoolean("hadoop.native.lib", false);
NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request"
, ZlibFactory.IsNativeZlibLoaded(conf));
// This should give us a BuiltInZlibInflater.
Decompressor zlibDecompressor = ZlibFactory.GetZlibDecompressor(conf);
NUnit.Framework.Assert.IsNotNull("zlibDecompressor is null!", zlibDecompressor);
Assert.True("ZlibFactory returned unexpected inflator", zlibDecompressor
is BuiltInZlibInflater);
// its createOutputStream() just wraps the existing stream in a
// java.util.zip.GZIPOutputStream.
CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
CompressionCodec codec = ccf.GetCodec(new Path("foo.gz"));
Assert.True("Codec for .gz file is not GzipCodec", codec is GzipCodec
);
// make sure we don't get a null decompressor
Decompressor codecDecompressor = codec.CreateDecompressor();
if (null == codecDecompressor)
{
NUnit.Framework.Assert.Fail("Got null codecDecompressor");
}
// Asking the CodecPool for a decompressor for GzipCodec
// should not return null
Decompressor poolDecompressor = CodecPool.GetDecompressor(codec);
if (null == poolDecompressor)
{
NUnit.Framework.Assert.Fail("Got null poolDecompressor");
}
// return a couple decompressors
CodecPool.ReturnDecompressor(zlibDecompressor);
CodecPool.ReturnDecompressor(poolDecompressor);
Decompressor poolDecompressor2 = CodecPool.GetDecompressor(codec);
if (poolDecompressor.GetType() == typeof(BuiltInGzipDecompressor))
{
if (poolDecompressor == poolDecompressor2)
{
NUnit.Framework.Assert.Fail("Reused java gzip decompressor in pool");
}
}
else
{
if (poolDecompressor != poolDecompressor2)
{
NUnit.Framework.Assert.Fail("Did not reuse native gzip decompressor in pool");
}
}
}
/// <exception cref="System.IO.IOException"/>
public override void Close()
{
@in.Close();
if (trackedDecompressor != null)
{
CodecPool.ReturnDecompressor(trackedDecompressor);
trackedDecompressor = null;
}
}
/// <exception cref="System.IO.IOException"/>
public override void Close()
{
Finish();
@out.Close();
if (trackedCompressor != null)
{
CodecPool.ReturnCompressor(trackedCompressor);
trackedCompressor = null;
}
}
public virtual void TestGzipLongOverflow()
{
Log.Info("testGzipLongOverflow");
// Don't use native libs for this test.
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false);
NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request"
, ZlibFactory.IsNativeZlibLoaded(conf));
// Ensure that the CodecPool has a BuiltInZlibInflater in it.
Decompressor zlibDecompressor = ZlibFactory.GetZlibDecompressor(conf);
NUnit.Framework.Assert.IsNotNull("zlibDecompressor is null!", zlibDecompressor);
Assert.True("ZlibFactory returned unexpected inflator", zlibDecompressor
is BuiltInZlibInflater);
CodecPool.ReturnDecompressor(zlibDecompressor);
// Now create a GZip text file.
string tmpDir = Runtime.GetProperty("test.build.data", "/tmp/");
Path f = new Path(new Path(tmpDir), "testGzipLongOverflow.bin.gz");
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream
(new FileOutputStream(f.ToString()))));
int Nbuf = 1024 * 4 + 1;
char[] buf = new char[1024 * 1024];
for (int i = 0; i < buf.Length; i++)
{
buf[i] = '\0';
}
for (int i_1 = 0; i_1 < Nbuf; i_1++)
{
bw.Write(buf);
}
bw.Close();
// Now read it back, using the CodecPool to establish the
// decompressor to use.
CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
CompressionCodec codec = ccf.GetCodec(f);
Decompressor decompressor = CodecPool.GetDecompressor(codec);
FileSystem fs = FileSystem.GetLocal(conf);
InputStream @is = fs.Open(f);
@is = codec.CreateInputStream(@is, decompressor);
BufferedReader br = new BufferedReader(new InputStreamReader(@is));
for (int j = 0; j < Nbuf; j++)
{
int n = br.Read(buf);
Assert.Equal("got wrong read length!", n, buf.Length);
for (int i_2 = 0; i_2 < buf.Length; i_2++)
{
Assert.Equal("got wrong byte!", buf[i_2], '\0');
}
}
br.Close();
}
public virtual void TestDecompressorNotReturnSameInstance()
{
Decompressor decomp = CodecPool.GetDecompressor(codec);
CodecPool.ReturnDecompressor(decomp);
CodecPool.ReturnDecompressor(decomp);
ICollection <Decompressor> decompressors = new HashSet <Decompressor>();
for (int i = 0; i < 10; ++i)
{
decompressors.AddItem(CodecPool.GetDecompressor(codec));
}
Assert.Equal(10, decompressors.Count);
foreach (Decompressor decompressor in decompressors)
{
CodecPool.ReturnDecompressor(decompressor);
}
}
public virtual void TestCompressorNotReturnSameInstance()
{
Compressor comp = CodecPool.GetCompressor(codec);
CodecPool.ReturnCompressor(comp);
CodecPool.ReturnCompressor(comp);
ICollection <Compressor> compressors = new HashSet <Compressor>();
for (int i = 0; i < 10; ++i)
{
compressors.AddItem(CodecPool.GetCompressor(codec));
}
Assert.Equal(10, compressors.Count);
foreach (Compressor compressor in compressors)
{
CodecPool.ReturnCompressor(compressor);
}
}
public virtual void TestDecompressorPoolCounts()
{
// Get two decompressors and return them
Decompressor decomp1 = CodecPool.GetDecompressor(codec);
Decompressor decomp2 = CodecPool.GetDecompressor(codec);
Assert.Equal(LeaseCountErr, 2, CodecPool.GetLeasedDecompressorsCount
(codec));
CodecPool.ReturnDecompressor(decomp2);
Assert.Equal(LeaseCountErr, 1, CodecPool.GetLeasedDecompressorsCount
(codec));
CodecPool.ReturnDecompressor(decomp1);
Assert.Equal(LeaseCountErr, 0, CodecPool.GetLeasedDecompressorsCount
(codec));
CodecPool.ReturnDecompressor(decomp1);
Assert.Equal(LeaseCountErr, 0, CodecPool.GetLeasedCompressorsCount
(codec));
}
public virtual void TestCodecPoolGzipReuse()
{
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, true);
if (!ZlibFactory.IsNativeZlibLoaded(conf))
{
Log.Warn("testCodecPoolGzipReuse skipped: native libs not loaded");
return;
}
GzipCodec gzc = ReflectionUtils.NewInstance <GzipCodec>(conf);
DefaultCodec dfc = ReflectionUtils.NewInstance <DefaultCodec>(conf);
Compressor c1 = CodecPool.GetCompressor(gzc);
Compressor c2 = CodecPool.GetCompressor(dfc);
CodecPool.ReturnCompressor(c1);
CodecPool.ReturnCompressor(c2);
Assert.True("Got mismatched ZlibCompressor", c2 != CodecPool.GetCompressor
(gzc));
}
/// <exception cref="System.Exception"/>
public virtual void TestMultiThreadedCompressorPool()
{
int iterations = 4;
ExecutorService threadpool = Executors.NewFixedThreadPool(3);
LinkedBlockingDeque <Compressor> queue = new LinkedBlockingDeque <Compressor>(2 * iterations
);
Callable <bool> consumer = new _Callable_110(queue);
Callable <bool> producer = new _Callable_119(this, queue);
for (int i = 0; i < iterations; i++)
{
threadpool.Submit(consumer);
threadpool.Submit(producer);
}
// wait for completion
threadpool.Shutdown();
threadpool.AwaitTermination(1000, TimeUnit.Seconds);
Assert.Equal(LeaseCountErr, 0, CodecPool.GetLeasedCompressorsCount
(codec));
}
public virtual void TestGzipCodecRead()
{
// Create a gzipped file and try to read it back, using a decompressor
// from the CodecPool.
// Don't use native libs for this test.
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false);
NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request"
, ZlibFactory.IsNativeZlibLoaded(conf));
// Ensure that the CodecPool has a BuiltInZlibInflater in it.
Decompressor zlibDecompressor = ZlibFactory.GetZlibDecompressor(conf);
NUnit.Framework.Assert.IsNotNull("zlibDecompressor is null!", zlibDecompressor);
Assert.True("ZlibFactory returned unexpected inflator", zlibDecompressor
is BuiltInZlibInflater);
CodecPool.ReturnDecompressor(zlibDecompressor);
// Now create a GZip text file.
string tmpDir = Runtime.GetProperty("test.build.data", "/tmp/");
Path f = new Path(new Path(tmpDir), "testGzipCodecRead.txt.gz");
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream
(new FileOutputStream(f.ToString()))));
string msg = "This is the message in the file!";
bw.Write(msg);
bw.Close();
// Now read it back, using the CodecPool to establish the
// decompressor to use.
CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
CompressionCodec codec = ccf.GetCodec(f);
Decompressor decompressor = CodecPool.GetDecompressor(codec);
FileSystem fs = FileSystem.GetLocal(conf);
InputStream @is = fs.Open(f);
@is = codec.CreateInputStream(@is, decompressor);
BufferedReader br = new BufferedReader(new InputStreamReader(@is));
string line = br.ReadLine();
Assert.Equal("Didn't get the same message back!", msg, line);
br.Close();
}
/// <exception cref="System.IO.IOException"/>
private static void GzipReinitTest(Configuration conf, CompressionCodec codec)
{
// Add codec to cache
ZlibFactory.SetCompressionLevel(conf, ZlibCompressor.CompressionLevel.BestCompression
);
ZlibFactory.SetCompressionStrategy(conf, ZlibCompressor.CompressionStrategy.DefaultStrategy
);
Compressor c1 = CodecPool.GetCompressor(codec);
CodecPool.ReturnCompressor(c1);
// reset compressor's compression level to perform no compression
ZlibFactory.SetCompressionLevel(conf, ZlibCompressor.CompressionLevel.NoCompression
);
Compressor c2 = CodecPool.GetCompressor(codec, conf);
// ensure same compressor placed earlier
Assert.True("Got mismatched ZlibCompressor", c1 == c2);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
CompressionOutputStream cos = null;
// write trivially compressable data
byte[] b = new byte[1 << 15];
Arrays.Fill(b, unchecked ((byte)43));
try
{
cos = codec.CreateOutputStream(bos, c2);
cos.Write(b);
}
finally
{
if (cos != null)
{
cos.Close();
}
CodecPool.ReturnCompressor(c2);
}
byte[] outbytes = bos.ToByteArray();
// verify data were not compressed
Assert.True("Compressed bytes contrary to configuration", outbytes
.Length >= b.Length);
}
/// <summary>Write infLen bytes (deflated) to file in test dir using codec.</summary>
/// <remarks>
/// Write infLen bytes (deflated) to file in test dir using codec.
/// Records are of the form
/// <i><b64 rand><i+i><b64 rand>
/// </remarks>
/// <exception cref="System.IO.IOException"/>
private static Path WriteSplitTestFile(FileSystem fs, Random rand, CompressionCodec
codec, long infLen)
{
int RecSize = 1024;
Path wd = new Path(new Path(Runtime.GetProperty("test.build.data", "/tmp")).MakeQualified
(fs), codec.GetType().Name);
Path file = new Path(wd, "test" + codec.GetDefaultExtension());
byte[] b = new byte[RecSize];
Base64 b64 = new Base64(0, null);
DataOutputStream fout = null;
Compressor cmp = CodecPool.GetCompressor(codec);
try
{
fout = new DataOutputStream(codec.CreateOutputStream(fs.Create(file, true), cmp));
DataOutputBuffer dob = new DataOutputBuffer(RecSize * 4 / 3 + 4);
int seq = 0;
while (infLen > 0)
{
rand.NextBytes(b);
byte[] b64enc = b64.Encode(b);
// ensures rand printable, no LF
dob.Reset();
dob.WriteInt(seq);
System.Array.Copy(dob.GetData(), 0, b64enc, 0, dob.GetLength());
fout.Write(b64enc);
fout.Write('\n');
++seq;
infLen -= b64enc.Length;
}
Log.Info("Wrote " + seq + " records to " + file);
}
finally
{
IOUtils.Cleanup(Log, fout);
CodecPool.ReturnCompressor(cmp);
}
return(file);
}
/// <summary>Create an input stream with a codec taken from the global CodecPool.</summary>
/// <param name="codec">The codec to use to create the input stream.</param>
/// <param name="conf">The configuration to use if we need to create a new codec.</param>
/// <param name="in">The input stream to wrap.</param>
/// <returns>The new input stream</returns>
/// <exception cref="System.IO.IOException"/>
internal static CompressionInputStream CreateInputStreamWithCodecPool(CompressionCodec
codec, Configuration conf, InputStream @in)
{
Decompressor decompressor = CodecPool.GetDecompressor(codec);
CompressionInputStream stream = null;
try
{
stream = codec.CreateInputStream(@in, decompressor);
}
finally
{
if (stream == null)
{
CodecPool.ReturnDecompressor(decompressor);
}
else
{
stream.SetTrackedDecompressor(decompressor);
}
}
return(stream);
}
/// <summary>Create an output stream with a codec taken from the global CodecPool.</summary>
/// <param name="codec">The codec to use to create the output stream.</param>
/// <param name="conf">The configuration to use if we need to create a new codec.</param>
/// <param name="out">The output stream to wrap.</param>
/// <returns>The new output stream</returns>
/// <exception cref="System.IO.IOException"/>
internal static CompressionOutputStream CreateOutputStreamWithCodecPool(CompressionCodec
codec, Configuration conf, OutputStream @out)
{
Compressor compressor = CodecPool.GetCompressor(codec, conf);
CompressionOutputStream stream = null;
try
{
stream = codec.CreateOutputStream(@out, compressor);
}
finally
{
if (stream == null)
{
CodecPool.ReturnCompressor(compressor);
}
else
{
stream.SetTrackedCompressor(compressor);
}
}
return(stream);
}
/// <exception cref="System.IO.IOException"/>
private void TestSplitableCodec(Type codecClass)
{
long Deflbytes = 2 * 1024 * 1024;
Configuration conf = new Configuration();
Random rand = new Random();
long seed = rand.NextLong();
Log.Info("seed: " + seed);
rand.SetSeed(seed);
SplittableCompressionCodec codec = ReflectionUtils.NewInstance(codecClass, conf);
FileSystem fs = FileSystem.GetLocal(conf);
FileStatus infile = fs.GetFileStatus(WriteSplitTestFile(fs, rand, codec, Deflbytes
));
if (infile.GetLen() > int.MaxValue)
{
NUnit.Framework.Assert.Fail("Unexpected compression: " + Deflbytes + " -> " + infile
.GetLen());
}
int flen = (int)infile.GetLen();
Text line = new Text();
Decompressor dcmp = CodecPool.GetDecompressor(codec);
try
{
for (int pos = 0; pos < infile.GetLen(); pos += rand.Next(flen / 8))
{
// read from random positions, verifying that there exist two sequential
// lines as written in writeSplitTestFile
SplitCompressionInputStream @in = codec.CreateInputStream(fs.Open(infile.GetPath(
)), dcmp, pos, flen, SplittableCompressionCodec.READ_MODE.Byblock);
if (@in.GetAdjustedStart() >= flen)
{
break;
}
Log.Info("SAMPLE " + @in.GetAdjustedStart() + "," + @in.GetAdjustedEnd());
LineReader lreader = new LineReader(@in);
lreader.ReadLine(line);
// ignore; likely partial
if (@in.GetPos() >= flen)
{
break;
}
lreader.ReadLine(line);
int seq1 = ReadLeadingInt(line);
lreader.ReadLine(line);
if (@in.GetPos() >= flen)
{
break;
}
int seq2 = ReadLeadingInt(line);
Assert.Equal("Mismatched lines", seq1 + 1, seq2);
}
}
finally
{
CodecPool.ReturnDecompressor(dcmp);
}
// remove on success
fs.Delete(infile.GetPath().GetParent(), true);
}