/// <exception cref="System.IO.IOException"/>
public virtual void TestGzipCodecWrite(bool useNative)
{
// Create a gzipped file using a compressor from the CodecPool,
// and try to read it back via the regular GZIPInputStream.
// Use native libs per the parameter
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, useNative);
if (useNative)
{
if (!ZlibFactory.IsNativeZlibLoaded(conf))
{
Log.Warn("testGzipCodecWrite skipped: native libs not loaded");
return;
}
}
else
{
NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request"
, ZlibFactory.IsNativeZlibLoaded(conf));
}
// Ensure that the CodecPool has a BuiltInZlibDeflater in it.
Compressor zlibCompressor = ZlibFactory.GetZlibCompressor(conf);
NUnit.Framework.Assert.IsNotNull("zlibCompressor is null!", zlibCompressor);
Assert.True("ZlibFactory returned unexpected deflator", useNative
? zlibCompressor is ZlibCompressor : zlibCompressor is BuiltInZlibDeflater);
CodecPool.ReturnCompressor(zlibCompressor);
// Create a GZIP text file via the Compressor interface.
CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
CompressionCodec codec = ccf.GetCodec(new Path("foo.gz"));
Assert.True("Codec for .gz file is not GzipCodec", codec is GzipCodec
);
string msg = "This is the message we are going to compress.";
string tmpDir = Runtime.GetProperty("test.build.data", "/tmp/");
string fileName = new Path(new Path(tmpDir), "testGzipCodecWrite.txt.gz").ToString
();
BufferedWriter w = null;
Compressor gzipCompressor = CodecPool.GetCompressor(codec);
if (null != gzipCompressor)
{
// If it gives us back a Compressor, we should be able to use this
// to write files we can then read back with Java's gzip tools.
OutputStream os = new CompressorStream(new FileOutputStream(fileName), gzipCompressor
);
w = new BufferedWriter(new OutputStreamWriter(os));
w.Write(msg);
w.Close();
CodecPool.ReturnCompressor(gzipCompressor);
VerifyGzipFile(fileName, msg);
}
// Create a gzip text file via codec.getOutputStream().
w = new BufferedWriter(new OutputStreamWriter(codec.CreateOutputStream(new FileOutputStream
(fileName))));
w.Write(msg);
w.Close();
VerifyGzipFile(fileName, msg);
}
public virtual void TestNativeCodeLoaded()
{
if (RequireTestJni() == false)
{
Log.Info("TestNativeCodeLoader: libhadoop.so testing is not required.");
return;
}
if (!NativeCodeLoader.IsNativeCodeLoaded())
{
NUnit.Framework.Assert.Fail("TestNativeCodeLoader: libhadoop.so testing was required, but "
+ "libhadoop.so was not loaded.");
}
NUnit.Framework.Assert.IsFalse(NativeCodeLoader.GetLibraryName().IsEmpty());
// library names are depended on platform and build envs
// so just check names are available
NUnit.Framework.Assert.IsFalse(ZlibFactory.GetLibraryName().IsEmpty());
if (NativeCodeLoader.BuildSupportsSnappy())
{
NUnit.Framework.Assert.IsFalse(SnappyCodec.GetLibraryName().IsEmpty());
}
if (NativeCodeLoader.BuildSupportsOpenssl())
{
NUnit.Framework.Assert.IsFalse(OpensslCipher.GetLibraryName().IsEmpty());
}
NUnit.Framework.Assert.IsFalse(Lz4Codec.GetLibraryName().IsEmpty());
Log.Info("TestNativeCodeLoader: libhadoop.so is loaded.");
}
/// <exception cref="System.IO.IOException"/>
public override CompressionOutputStream CreateOutputStream(OutputStream @out)
{
if (!ZlibFactory.IsNativeZlibLoaded(conf))
{
return(new GzipCodec.GzipOutputStream(@out));
}
return(CompressionCodec.Util.CreateOutputStreamWithCodecPool(this, conf, @out));
}
public virtual void TestCodecPoolAndGzipDecompressor()
{
// BuiltInZlibInflater should not be used as the GzipCodec decompressor.
// Assert that this is the case.
// Don't use native libs for this test.
Configuration conf = new Configuration();
conf.SetBoolean("hadoop.native.lib", false);
NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request"
, ZlibFactory.IsNativeZlibLoaded(conf));
// This should give us a BuiltInZlibInflater.
Decompressor zlibDecompressor = ZlibFactory.GetZlibDecompressor(conf);
NUnit.Framework.Assert.IsNotNull("zlibDecompressor is null!", zlibDecompressor);
Assert.True("ZlibFactory returned unexpected inflator", zlibDecompressor
is BuiltInZlibInflater);
// its createOutputStream() just wraps the existing stream in a
// java.util.zip.GZIPOutputStream.
CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
CompressionCodec codec = ccf.GetCodec(new Path("foo.gz"));
Assert.True("Codec for .gz file is not GzipCodec", codec is GzipCodec
);
// make sure we don't get a null decompressor
Decompressor codecDecompressor = codec.CreateDecompressor();
if (null == codecDecompressor)
{
NUnit.Framework.Assert.Fail("Got null codecDecompressor");
}
// Asking the CodecPool for a decompressor for GzipCodec
// should not return null
Decompressor poolDecompressor = CodecPool.GetDecompressor(codec);
if (null == poolDecompressor)
{
NUnit.Framework.Assert.Fail("Got null poolDecompressor");
}
// return a couple decompressors
CodecPool.ReturnDecompressor(zlibDecompressor);
CodecPool.ReturnDecompressor(poolDecompressor);
Decompressor poolDecompressor2 = CodecPool.GetDecompressor(codec);
if (poolDecompressor.GetType() == typeof(BuiltInGzipDecompressor))
{
if (poolDecompressor == poolDecompressor2)
{
NUnit.Framework.Assert.Fail("Reused java gzip decompressor in pool");
}
}
else
{
if (poolDecompressor != poolDecompressor2)
{
NUnit.Framework.Assert.Fail("Did not reuse native gzip decompressor in pool");
}
}
}
public virtual void TestGzipLongOverflow()
{
Log.Info("testGzipLongOverflow");
// Don't use native libs for this test.
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false);
NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request"
, ZlibFactory.IsNativeZlibLoaded(conf));
// Ensure that the CodecPool has a BuiltInZlibInflater in it.
Decompressor zlibDecompressor = ZlibFactory.GetZlibDecompressor(conf);
NUnit.Framework.Assert.IsNotNull("zlibDecompressor is null!", zlibDecompressor);
Assert.True("ZlibFactory returned unexpected inflator", zlibDecompressor
is BuiltInZlibInflater);
CodecPool.ReturnDecompressor(zlibDecompressor);
// Now create a GZip text file.
string tmpDir = Runtime.GetProperty("test.build.data", "/tmp/");
Path f = new Path(new Path(tmpDir), "testGzipLongOverflow.bin.gz");
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream
(new FileOutputStream(f.ToString()))));
int Nbuf = 1024 * 4 + 1;
char[] buf = new char[1024 * 1024];
for (int i = 0; i < buf.Length; i++)
{
buf[i] = '\0';
}
for (int i_1 = 0; i_1 < Nbuf; i_1++)
{
bw.Write(buf);
}
bw.Close();
// Now read it back, using the CodecPool to establish the
// decompressor to use.
CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
CompressionCodec codec = ccf.GetCodec(f);
Decompressor decompressor = CodecPool.GetDecompressor(codec);
FileSystem fs = FileSystem.GetLocal(conf);
InputStream @is = fs.Open(f);
@is = codec.CreateInputStream(@is, decompressor);
BufferedReader br = new BufferedReader(new InputStreamReader(@is));
for (int j = 0; j < Nbuf; j++)
{
int n = br.Read(buf);
Assert.Equal("got wrong read length!", n, buf.Length);
for (int i_2 = 0; i_2 < buf.Length; i_2++)
{
Assert.Equal("got wrong byte!", buf[i_2], '\0');
}
}
br.Close();
}
public virtual void TestGzipCodecWithParam()
{
Configuration conf = new Configuration(this.conf);
ZlibFactory.SetCompressionLevel(conf, ZlibCompressor.CompressionLevel.BestCompression
);
ZlibFactory.SetCompressionStrategy(conf, ZlibCompressor.CompressionStrategy.HuffmanOnly
);
CodecTest(conf, seed, 0, "org.apache.hadoop.io.compress.GzipCodec");
CodecTest(conf, seed, count, "org.apache.hadoop.io.compress.GzipCodec");
}
public virtual void TestNativeGzipConcat()
{
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, true);
if (!ZlibFactory.IsNativeZlibLoaded(conf))
{
Log.Warn("skipped: native libs not loaded");
return;
}
GzipConcatTest(conf, typeof(GzipCodec.GzipZlibDecompressor));
}
/// <exception cref="System.IO.IOException"/>
private static void CodecTestWithNOCompression(Configuration conf, string codecClass
)
{
// Create a compressor with NO_COMPRESSION and make sure that
// output is not compressed by comparing the size with the
// original input
CompressionCodec codec = null;
ZlibFactory.SetCompressionLevel(conf, ZlibCompressor.CompressionLevel.NoCompression
);
try
{
codec = (CompressionCodec)ReflectionUtils.NewInstance(conf.GetClassByName(codecClass
), conf);
}
catch (TypeLoadException)
{
throw new IOException("Illegal codec!");
}
Compressor c = codec.CreateCompressor();
// ensure same compressor placed earlier
ByteArrayOutputStream bos = new ByteArrayOutputStream();
CompressionOutputStream cos = null;
// write trivially compressable data
byte[] b = new byte[1 << 15];
Arrays.Fill(b, unchecked ((byte)43));
try
{
cos = codec.CreateOutputStream(bos, c);
cos.Write(b);
}
finally
{
if (cos != null)
{
cos.Close();
}
}
byte[] outbytes = bos.ToByteArray();
// verify data were not compressed
Assert.True("Compressed bytes contrary to configuration(NO_COMPRESSION)"
, outbytes.Length >= b.Length);
}
public virtual void TestCodecPoolCompressorReinit()
{
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, true);
if (ZlibFactory.IsNativeZlibLoaded(conf))
{
GzipCodec gzc = ReflectionUtils.NewInstance <GzipCodec>(conf);
GzipReinitTest(conf, gzc);
}
else
{
Log.Warn("testCodecPoolCompressorReinit skipped: native libs not loaded");
}
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false);
DefaultCodec dfc = ReflectionUtils.NewInstance <DefaultCodec>(conf);
GzipReinitTest(conf, dfc);
}
public virtual void TestCodecInitWithCompressionLevel()
{
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, true);
if (ZlibFactory.IsNativeZlibLoaded(conf))
{
Log.Info("testCodecInitWithCompressionLevel with native");
CodecTestWithNOCompression(conf, "org.apache.hadoop.io.compress.GzipCodec");
CodecTestWithNOCompression(conf, "org.apache.hadoop.io.compress.DefaultCodec");
}
else
{
Log.Warn("testCodecInitWithCompressionLevel for native skipped" + ": native libs not loaded"
);
}
conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false);
CodecTestWithNOCompression(conf, "org.apache.hadoop.io.compress.DefaultCodec");
}
public virtual void TestCodecPoolGzipReuse()
{
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, true);
if (!ZlibFactory.IsNativeZlibLoaded(conf))
{
Log.Warn("testCodecPoolGzipReuse skipped: native libs not loaded");
return;
}
GzipCodec gzc = ReflectionUtils.NewInstance <GzipCodec>(conf);
DefaultCodec dfc = ReflectionUtils.NewInstance <DefaultCodec>(conf);
Compressor c1 = CodecPool.GetCompressor(gzc);
Compressor c2 = CodecPool.GetCompressor(dfc);
CodecPool.ReturnCompressor(c1);
CodecPool.ReturnCompressor(c2);
Assert.True("Got mismatched ZlibCompressor", c2 != CodecPool.GetCompressor
(gzc));
}
/// <exception cref="System.IO.IOException"/>
private static void GzipReinitTest(Configuration conf, CompressionCodec codec)
{
// Add codec to cache
ZlibFactory.SetCompressionLevel(conf, ZlibCompressor.CompressionLevel.BestCompression
);
ZlibFactory.SetCompressionStrategy(conf, ZlibCompressor.CompressionStrategy.DefaultStrategy
);
Compressor c1 = CodecPool.GetCompressor(codec);
CodecPool.ReturnCompressor(c1);
// reset compressor's compression level to perform no compression
ZlibFactory.SetCompressionLevel(conf, ZlibCompressor.CompressionLevel.NoCompression
);
Compressor c2 = CodecPool.GetCompressor(codec, conf);
// ensure same compressor placed earlier
Assert.True("Got mismatched ZlibCompressor", c1 == c2);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
CompressionOutputStream cos = null;
// write trivially compressable data
byte[] b = new byte[1 << 15];
Arrays.Fill(b, unchecked ((byte)43));
try
{
cos = codec.CreateOutputStream(bos, c2);
cos.Write(b);
}
finally
{
if (cos != null)
{
cos.Close();
}
CodecPool.ReturnCompressor(c2);
}
byte[] outbytes = bos.ToByteArray();
// verify data were not compressed
Assert.True("Compressed bytes contrary to configuration", outbytes
.Length >= b.Length);
}
public virtual void TestGzipCodecRead()
{
// Create a gzipped file and try to read it back, using a decompressor
// from the CodecPool.
// Don't use native libs for this test.
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false);
NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request"
, ZlibFactory.IsNativeZlibLoaded(conf));
// Ensure that the CodecPool has a BuiltInZlibInflater in it.
Decompressor zlibDecompressor = ZlibFactory.GetZlibDecompressor(conf);
NUnit.Framework.Assert.IsNotNull("zlibDecompressor is null!", zlibDecompressor);
Assert.True("ZlibFactory returned unexpected inflator", zlibDecompressor
is BuiltInZlibInflater);
CodecPool.ReturnDecompressor(zlibDecompressor);
// Now create a GZip text file.
string tmpDir = Runtime.GetProperty("test.build.data", "/tmp/");
Path f = new Path(new Path(tmpDir), "testGzipCodecRead.txt.gz");
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream
(new FileOutputStream(f.ToString()))));
string msg = "This is the message in the file!";
bw.Write(msg);
bw.Close();
// Now read it back, using the CodecPool to establish the
// decompressor to use.
CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
CompressionCodec codec = ccf.GetCodec(f);
Decompressor decompressor = CodecPool.GetDecompressor(codec);
FileSystem fs = FileSystem.GetLocal(conf);
InputStream @is = fs.Open(f);
@is = codec.CreateInputStream(@is, decompressor);
BufferedReader br = new BufferedReader(new InputStreamReader(@is));
string line = br.ReadLine();
Assert.Equal("Didn't get the same message back!", msg, line);
br.Close();
}
/// <summary>Method for compressor availability check</summary>
private static bool IsAvailable <T, E>(CompressDecompressTester.TesterPair <T, E> pair
)
where T : Compressor
where E : Decompressor
{
Compressor compressor = pair.compressor;
if (compressor.GetType().IsAssignableFrom(typeof(Lz4Compressor)) && (NativeCodeLoader
.IsNativeCodeLoaded()))
{
return(true);
}
else
{
if (compressor.GetType().IsAssignableFrom(typeof(BuiltInZlibDeflater)) && NativeCodeLoader
.IsNativeCodeLoaded())
{
return(true);
}
else
{
if (compressor.GetType().IsAssignableFrom(typeof(ZlibCompressor)))
{
return(ZlibFactory.IsNativeZlibLoaded(new Configuration()));
}
else
{
if (compressor.GetType().IsAssignableFrom(typeof(SnappyCompressor)) && IsNativeSnappyLoadable
())
{
return(true);
}
}
}
}
return(false);
}
public virtual Compressor CreateCompressor()
{
return(ZlibFactory.GetZlibCompressor(conf));
}
public TestTFileByteArrays()
{
usingNative = ZlibFactory.IsNativeZlibLoaded(conf);
}
public override Compressor CreateCompressor()
{
return((ZlibFactory.IsNativeZlibLoaded(conf)) ? new GzipCodec.GzipZlibCompressor(
conf) : null);
}
public override Type GetCompressorType()
{
return(ZlibFactory.IsNativeZlibLoaded(conf) ? typeof(GzipCodec.GzipZlibCompressor
) : null);
}
public override Decompressor CreateDecompressor()
{
return((ZlibFactory.IsNativeZlibLoaded(conf)) ? new GzipCodec.GzipZlibDecompressor
() : new BuiltInGzipDecompressor());
}
public override Type GetDecompressorType()
{
return(ZlibFactory.IsNativeZlibLoaded(conf) ? typeof(GzipCodec.GzipZlibDecompressor
) : typeof(BuiltInGzipDecompressor));
}
public override DirectDecompressor CreateDirectDecompressor()
{
return(ZlibFactory.IsNativeZlibLoaded(conf) ? new ZlibDecompressor.ZlibDirectDecompressor
(ZlibDecompressor.CompressionHeader.AutodetectGzipZlib, 0) : null);
}
public GzipZlibCompressor(Configuration conf)
: base(ZlibFactory.GetCompressionLevel(conf), ZlibFactory.GetCompressionStrategy(
conf), ZlibCompressor.CompressionHeader.GzipFormat, 64 * 1024)
{
}
public virtual Type GetDecompressorType()
{
return(ZlibFactory.GetZlibDecompressorType(conf));
}
/// <summary><inheritDoc/></summary>
public virtual DirectDecompressor CreateDirectDecompressor()
{
return(ZlibFactory.GetZlibDirectDecompressor(conf));
}
/// <summary>A tool to test native library availability,</summary>
public static void Main(string[] args)
{
string usage = "NativeLibraryChecker [-a|-h]\n" + " -a use -a to check all libraries are available\n"
+ " by default just check hadoop library (and\n" + " winutils.exe on Windows OS) is available\n"
+ " exit with error code 1 if check failed\n" + " -h print this message\n";
if (args.Length > 1 || (args.Length == 1 && !(args[0].Equals("-a") || args[0].Equals
("-h"))))
{
System.Console.Error.WriteLine(usage);
ExitUtil.Terminate(1);
}
bool checkAll = false;
if (args.Length == 1)
{
if (args[0].Equals("-h"))
{
System.Console.Out.WriteLine(usage);
return;
}
checkAll = true;
}
Configuration conf = new Configuration();
bool nativeHadoopLoaded = NativeCodeLoader.IsNativeCodeLoaded();
bool zlibLoaded = false;
bool snappyLoaded = false;
// lz4 is linked within libhadoop
bool lz4Loaded = nativeHadoopLoaded;
bool bzip2Loaded = Bzip2Factory.IsNativeBzip2Loaded(conf);
bool openSslLoaded = false;
bool winutilsExists = false;
string openSslDetail = string.Empty;
string hadoopLibraryName = string.Empty;
string zlibLibraryName = string.Empty;
string snappyLibraryName = string.Empty;
string lz4LibraryName = string.Empty;
string bzip2LibraryName = string.Empty;
string winutilsPath = null;
if (nativeHadoopLoaded)
{
hadoopLibraryName = NativeCodeLoader.GetLibraryName();
zlibLoaded = ZlibFactory.IsNativeZlibLoaded(conf);
if (zlibLoaded)
{
zlibLibraryName = ZlibFactory.GetLibraryName();
}
snappyLoaded = NativeCodeLoader.BuildSupportsSnappy() && SnappyCodec.IsNativeCodeLoaded
();
if (snappyLoaded && NativeCodeLoader.BuildSupportsSnappy())
{
snappyLibraryName = SnappyCodec.GetLibraryName();
}
if (OpensslCipher.GetLoadingFailureReason() != null)
{
openSslDetail = OpensslCipher.GetLoadingFailureReason();
openSslLoaded = false;
}
else
{
openSslDetail = OpensslCipher.GetLibraryName();
openSslLoaded = true;
}
if (lz4Loaded)
{
lz4LibraryName = Lz4Codec.GetLibraryName();
}
if (bzip2Loaded)
{
bzip2LibraryName = Bzip2Factory.GetLibraryName(conf);
}
}
// winutils.exe is required on Windows
winutilsPath = Shell.GetWinUtilsPath();
if (winutilsPath != null)
{
winutilsExists = true;
}
else
{
winutilsPath = string.Empty;
}
System.Console.Out.WriteLine("Native library checking:");
System.Console.Out.Printf("hadoop: %b %s%n", nativeHadoopLoaded, hadoopLibraryName
);
System.Console.Out.Printf("zlib: %b %s%n", zlibLoaded, zlibLibraryName);
System.Console.Out.Printf("snappy: %b %s%n", snappyLoaded, snappyLibraryName);
System.Console.Out.Printf("lz4: %b %s%n", lz4Loaded, lz4LibraryName);
System.Console.Out.Printf("bzip2: %b %s%n", bzip2Loaded, bzip2LibraryName);
System.Console.Out.Printf("openssl: %b %s%n", openSslLoaded, openSslDetail);
if (Shell.Windows)
{
System.Console.Out.Printf("winutils: %b %s%n", winutilsExists, winutilsPath);
}
if ((!nativeHadoopLoaded) || (Shell.Windows && (!winutilsExists)) || (checkAll &&
!(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded)))
{
// return 1 to indicated check failed
ExitUtil.Terminate(1);
}
}
