/// <exception cref="System.IO.IOException"/>
public virtual void TestGzipCodecWrite(bool useNative)
{
// Create a gzipped file using a compressor from the CodecPool,
// and try to read it back via the regular GZIPInputStream.
// Use native libs per the parameter
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, useNative);
if (useNative)
{
if (!ZlibFactory.IsNativeZlibLoaded(conf))
{
Log.Warn("testGzipCodecWrite skipped: native libs not loaded");
return;
}
}
else
{
NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request"
, ZlibFactory.IsNativeZlibLoaded(conf));
}
// Ensure that the CodecPool has a BuiltInZlibDeflater in it.
Compressor zlibCompressor = ZlibFactory.GetZlibCompressor(conf);
NUnit.Framework.Assert.IsNotNull("zlibCompressor is null!", zlibCompressor);
Assert.True("ZlibFactory returned unexpected deflator", useNative
? zlibCompressor is ZlibCompressor : zlibCompressor is BuiltInZlibDeflater);
CodecPool.ReturnCompressor(zlibCompressor);
// Create a GZIP text file via the Compressor interface.
CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
CompressionCodec codec = ccf.GetCodec(new Path("foo.gz"));
Assert.True("Codec for .gz file is not GzipCodec", codec is GzipCodec
);
string msg = "This is the message we are going to compress.";
string tmpDir = Runtime.GetProperty("test.build.data", "/tmp/");
string fileName = new Path(new Path(tmpDir), "testGzipCodecWrite.txt.gz").ToString
();
BufferedWriter w = null;
Compressor gzipCompressor = CodecPool.GetCompressor(codec);
if (null != gzipCompressor)
{
// If it gives us back a Compressor, we should be able to use this
// to write files we can then read back with Java's gzip tools.
OutputStream os = new CompressorStream(new FileOutputStream(fileName), gzipCompressor
);
w = new BufferedWriter(new OutputStreamWriter(os));
w.Write(msg);
w.Close();
CodecPool.ReturnCompressor(gzipCompressor);
VerifyGzipFile(fileName, msg);
}
// Create a gzip text file via codec.getOutputStream().
w = new BufferedWriter(new OutputStreamWriter(codec.CreateOutputStream(new FileOutputStream
(fileName))));
w.Write(msg);
w.Close();
VerifyGzipFile(fileName, msg);
}
/// <summary>Returns a factory for a given set of codecs</summary>
/// <param name="classes">the codec classes to include</param>
/// <returns>a new factory</returns>
private static CompressionCodecFactory SetClasses(Type[] classes)
{
Configuration conf = new Configuration();
CompressionCodecFactory.SetCodecClasses(conf, Arrays.AsList(classes));
return(new CompressionCodecFactory(conf));
}
public virtual void TestCodecPoolAndGzipDecompressor()
{
// BuiltInZlibInflater should not be used as the GzipCodec decompressor.
// Assert that this is the case.
// Don't use native libs for this test.
Configuration conf = new Configuration();
conf.SetBoolean("hadoop.native.lib", false);
NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request"
, ZlibFactory.IsNativeZlibLoaded(conf));
// This should give us a BuiltInZlibInflater.
Decompressor zlibDecompressor = ZlibFactory.GetZlibDecompressor(conf);
NUnit.Framework.Assert.IsNotNull("zlibDecompressor is null!", zlibDecompressor);
Assert.True("ZlibFactory returned unexpected inflator", zlibDecompressor
is BuiltInZlibInflater);
// its createOutputStream() just wraps the existing stream in a
// java.util.zip.GZIPOutputStream.
CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
CompressionCodec codec = ccf.GetCodec(new Path("foo.gz"));
Assert.True("Codec for .gz file is not GzipCodec", codec is GzipCodec
);
// make sure we don't get a null decompressor
Decompressor codecDecompressor = codec.CreateDecompressor();
if (null == codecDecompressor)
{
NUnit.Framework.Assert.Fail("Got null codecDecompressor");
}
// Asking the CodecPool for a decompressor for GzipCodec
// should not return null
Decompressor poolDecompressor = CodecPool.GetDecompressor(codec);
if (null == poolDecompressor)
{
NUnit.Framework.Assert.Fail("Got null poolDecompressor");
}
// return a couple decompressors
CodecPool.ReturnDecompressor(zlibDecompressor);
CodecPool.ReturnDecompressor(poolDecompressor);
Decompressor poolDecompressor2 = CodecPool.GetDecompressor(codec);
if (poolDecompressor.GetType() == typeof(BuiltInGzipDecompressor))
{
if (poolDecompressor == poolDecompressor2)
{
NUnit.Framework.Assert.Fail("Reused java gzip decompressor in pool");
}
}
else
{
if (poolDecompressor != poolDecompressor2)
{
NUnit.Framework.Assert.Fail("Did not reuse native gzip decompressor in pool");
}
}
}
public virtual void TestGzipLongOverflow()
{
Log.Info("testGzipLongOverflow");
// Don't use native libs for this test.
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false);
NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request"
, ZlibFactory.IsNativeZlibLoaded(conf));
// Ensure that the CodecPool has a BuiltInZlibInflater in it.
Decompressor zlibDecompressor = ZlibFactory.GetZlibDecompressor(conf);
NUnit.Framework.Assert.IsNotNull("zlibDecompressor is null!", zlibDecompressor);
Assert.True("ZlibFactory returned unexpected inflator", zlibDecompressor
is BuiltInZlibInflater);
CodecPool.ReturnDecompressor(zlibDecompressor);
// Now create a GZip text file.
string tmpDir = Runtime.GetProperty("test.build.data", "/tmp/");
Path f = new Path(new Path(tmpDir), "testGzipLongOverflow.bin.gz");
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream
(new FileOutputStream(f.ToString()))));
int Nbuf = 1024 * 4 + 1;
char[] buf = new char[1024 * 1024];
for (int i = 0; i < buf.Length; i++)
{
buf[i] = '\0';
}
for (int i_1 = 0; i_1 < Nbuf; i_1++)
{
bw.Write(buf);
}
bw.Close();
// Now read it back, using the CodecPool to establish the
// decompressor to use.
CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
CompressionCodec codec = ccf.GetCodec(f);
Decompressor decompressor = CodecPool.GetDecompressor(codec);
FileSystem fs = FileSystem.GetLocal(conf);
InputStream @is = fs.Open(f);
@is = codec.CreateInputStream(@is, decompressor);
BufferedReader br = new BufferedReader(new InputStreamReader(@is));
for (int j = 0; j < Nbuf; j++)
{
int n = br.Read(buf);
Assert.Equal("got wrong read length!", n, buf.Length);
for (int i_2 = 0; i_2 < buf.Length; i_2++)
{
Assert.Equal("got wrong byte!", buf[i_2], '\0');
}
}
br.Close();
}
public virtual void TestGzipCodecRead()
{
// Create a gzipped file and try to read it back, using a decompressor
// from the CodecPool.
// Don't use native libs for this test.
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false);
NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request"
, ZlibFactory.IsNativeZlibLoaded(conf));
// Ensure that the CodecPool has a BuiltInZlibInflater in it.
Decompressor zlibDecompressor = ZlibFactory.GetZlibDecompressor(conf);
NUnit.Framework.Assert.IsNotNull("zlibDecompressor is null!", zlibDecompressor);
Assert.True("ZlibFactory returned unexpected inflator", zlibDecompressor
is BuiltInZlibInflater);
CodecPool.ReturnDecompressor(zlibDecompressor);
// Now create a GZip text file.
string tmpDir = Runtime.GetProperty("test.build.data", "/tmp/");
Path f = new Path(new Path(tmpDir), "testGzipCodecRead.txt.gz");
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream
(new FileOutputStream(f.ToString()))));
string msg = "This is the message in the file!";
bw.Write(msg);
bw.Close();
// Now read it back, using the CodecPool to establish the
// decompressor to use.
CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
CompressionCodec codec = ccf.GetCodec(f);
Decompressor decompressor = CodecPool.GetDecompressor(codec);
FileSystem fs = FileSystem.GetLocal(conf);
InputStream @is = fs.Open(f);
@is = codec.CreateInputStream(@is, decompressor);
BufferedReader br = new BufferedReader(new InputStreamReader(@is));
string line = br.ReadLine();
Assert.Equal("Didn't get the same message back!", msg, line);
br.Close();
}
public static void TestFinding()
{
CompressionCodecFactory factory = new CompressionCodecFactory(new Configuration()
);
CompressionCodec codec = factory.GetCodec(new Path("/tmp/foo.bar"));
Assert.Equal("default factory foo codec", null, codec);
codec = factory.GetCodecByClassName(typeof(TestCodecFactory.BarCodec).GetCanonicalName
());
Assert.Equal("default factory foo codec", null, codec);
codec = factory.GetCodec(new Path("/tmp/foo.gz"));
CheckCodec("default factory for .gz", typeof(GzipCodec), codec);
codec = factory.GetCodecByClassName(typeof(GzipCodec).GetCanonicalName());
CheckCodec("default factory for gzip codec", typeof(GzipCodec), codec);
codec = factory.GetCodecByName("gzip");
CheckCodec("default factory for gzip codec", typeof(GzipCodec), codec);
codec = factory.GetCodecByName("GZIP");
CheckCodec("default factory for gzip codec", typeof(GzipCodec), codec);
codec = factory.GetCodecByName("GZIPCodec");
CheckCodec("default factory for gzip codec", typeof(GzipCodec), codec);
codec = factory.GetCodecByName("gzipcodec");
CheckCodec("default factory for gzip codec", typeof(GzipCodec), codec);
Type klass = factory.GetCodecClassByName("gzipcodec");
Assert.Equal(typeof(GzipCodec), klass);
codec = factory.GetCodec(new Path("/tmp/foo.bz2"));
CheckCodec("default factory for .bz2", typeof(BZip2Codec), codec);
codec = factory.GetCodecByClassName(typeof(BZip2Codec).GetCanonicalName());
CheckCodec("default factory for bzip2 codec", typeof(BZip2Codec), codec);
codec = factory.GetCodecByName("bzip2");
CheckCodec("default factory for bzip2 codec", typeof(BZip2Codec), codec);
codec = factory.GetCodecByName("bzip2codec");
CheckCodec("default factory for bzip2 codec", typeof(BZip2Codec), codec);
codec = factory.GetCodecByName("BZIP2");
CheckCodec("default factory for bzip2 codec", typeof(BZip2Codec), codec);
codec = factory.GetCodecByName("BZIP2CODEC");
CheckCodec("default factory for bzip2 codec", typeof(BZip2Codec), codec);
codec = factory.GetCodecByClassName(typeof(DeflateCodec).GetCanonicalName());
CheckCodec("default factory for deflate codec", typeof(DeflateCodec), codec);
codec = factory.GetCodecByName("deflate");
CheckCodec("default factory for deflate codec", typeof(DeflateCodec), codec);
codec = factory.GetCodecByName("deflatecodec");
CheckCodec("default factory for deflate codec", typeof(DeflateCodec), codec);
codec = factory.GetCodecByName("DEFLATE");
CheckCodec("default factory for deflate codec", typeof(DeflateCodec), codec);
codec = factory.GetCodecByName("DEFLATECODEC");
CheckCodec("default factory for deflate codec", typeof(DeflateCodec), codec);
factory = SetClasses(new Type[0]);
// gz, bz2, snappy, lz4 are picked up by service loader, but bar isn't
codec = factory.GetCodec(new Path("/tmp/foo.bar"));
Assert.Equal("empty factory bar codec", null, codec);
codec = factory.GetCodecByClassName(typeof(TestCodecFactory.BarCodec).GetCanonicalName
());
Assert.Equal("empty factory bar codec", null, codec);
codec = factory.GetCodec(new Path("/tmp/foo.gz"));
CheckCodec("empty factory gz codec", typeof(GzipCodec), codec);
codec = factory.GetCodecByClassName(typeof(GzipCodec).GetCanonicalName());
CheckCodec("empty factory gz codec", typeof(GzipCodec), codec);
codec = factory.GetCodec(new Path("/tmp/foo.bz2"));
CheckCodec("empty factory for .bz2", typeof(BZip2Codec), codec);
codec = factory.GetCodecByClassName(typeof(BZip2Codec).GetCanonicalName());
CheckCodec("empty factory for bzip2 codec", typeof(BZip2Codec), codec);
codec = factory.GetCodec(new Path("/tmp/foo.snappy"));
CheckCodec("empty factory snappy codec", typeof(SnappyCodec), codec);
codec = factory.GetCodecByClassName(typeof(SnappyCodec).GetCanonicalName());
CheckCodec("empty factory snappy codec", typeof(SnappyCodec), codec);
codec = factory.GetCodec(new Path("/tmp/foo.lz4"));
CheckCodec("empty factory lz4 codec", typeof(Lz4Codec), codec);
codec = factory.GetCodecByClassName(typeof(Lz4Codec).GetCanonicalName());
CheckCodec("empty factory lz4 codec", typeof(Lz4Codec), codec);
factory = SetClasses(new Type[] { typeof(TestCodecFactory.BarCodec), typeof(TestCodecFactory.FooCodec
), typeof(TestCodecFactory.FooBarCodec) });
codec = factory.GetCodec(new Path("/tmp/.foo.bar.gz"));
CheckCodec("full factory gz codec", typeof(GzipCodec), codec);
codec = factory.GetCodecByClassName(typeof(GzipCodec).GetCanonicalName());
CheckCodec("full codec gz codec", typeof(GzipCodec), codec);
codec = factory.GetCodec(new Path("/tmp/foo.bz2"));
CheckCodec("full factory for .bz2", typeof(BZip2Codec), codec);
codec = factory.GetCodecByClassName(typeof(BZip2Codec).GetCanonicalName());
CheckCodec("full codec bzip2 codec", typeof(BZip2Codec), codec);
codec = factory.GetCodec(new Path("/tmp/foo.bar"));
CheckCodec("full factory bar codec", typeof(TestCodecFactory.BarCodec), codec);
codec = factory.GetCodecByClassName(typeof(TestCodecFactory.BarCodec).GetCanonicalName
());
CheckCodec("full factory bar codec", typeof(TestCodecFactory.BarCodec), codec);
codec = factory.GetCodecByName("bar");
CheckCodec("full factory bar codec", typeof(TestCodecFactory.BarCodec), codec);
codec = factory.GetCodecByName("BAR");
CheckCodec("full factory bar codec", typeof(TestCodecFactory.BarCodec), codec);
codec = factory.GetCodec(new Path("/tmp/foo/baz.foo.bar"));
CheckCodec("full factory foo bar codec", typeof(TestCodecFactory.FooBarCodec), codec
);
codec = factory.GetCodecByClassName(typeof(TestCodecFactory.FooBarCodec).GetCanonicalName
());
CheckCodec("full factory foo bar codec", typeof(TestCodecFactory.FooBarCodec), codec
);
codec = factory.GetCodecByName("foobar");
CheckCodec("full factory foo bar codec", typeof(TestCodecFactory.FooBarCodec), codec
);
codec = factory.GetCodecByName("FOOBAR");
CheckCodec("full factory foo bar codec", typeof(TestCodecFactory.FooBarCodec), codec
);
codec = factory.GetCodec(new Path("/tmp/foo.foo"));
CheckCodec("full factory foo codec", typeof(TestCodecFactory.FooCodec), codec);
codec = factory.GetCodecByClassName(typeof(TestCodecFactory.FooCodec).GetCanonicalName
());
CheckCodec("full factory foo codec", typeof(TestCodecFactory.FooCodec), codec);
codec = factory.GetCodecByName("foo");
CheckCodec("full factory foo codec", typeof(TestCodecFactory.FooCodec), codec);
codec = factory.GetCodecByName("FOO");
CheckCodec("full factory foo codec", typeof(TestCodecFactory.FooCodec), codec);
factory = SetClasses(new Type[] { typeof(TestCodecFactory.NewGzipCodec) });
codec = factory.GetCodec(new Path("/tmp/foo.gz"));
CheckCodec("overridden factory for .gz", typeof(TestCodecFactory.NewGzipCodec), codec
);
codec = factory.GetCodecByClassName(typeof(TestCodecFactory.NewGzipCodec).GetCanonicalName
());
CheckCodec("overridden factory for gzip codec", typeof(TestCodecFactory.NewGzipCodec
), codec);
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeys.IoCompressionCodecsKey, " org.apache.hadoop.io.compress.GzipCodec , "
+ " org.apache.hadoop.io.compress.DefaultCodec , " + " org.apache.hadoop.io.compress.BZip2Codec "
);
try
{
CompressionCodecFactory.GetCodecClasses(conf);
}
catch (ArgumentException)
{
Fail("IllegalArgumentException is unexpected");
}
}
