/// <summary>
/// Create a
/// <see cref="CompressionOutputStream"/>
/// that will write to the given
/// <see cref="System.IO.OutputStream"/>
/// with the given
/// <see cref="Compressor"/>
/// .
/// </summary>
/// <param name="out">the location for the final output stream</param>
/// <param name="compressor">compressor to use</param>
/// <returns>
/// a stream the user can write uncompressed data to, to have it
/// compressed
/// </returns>
/// <exception cref="System.IO.IOException"/>
public virtual CompressionOutputStream CreateOutputStream(OutputStream @out, Compressor
compressor)
{
return(Bzip2Factory.IsNativeBzip2Loaded(conf) ? new CompressorStream(@out, compressor
, conf.GetInt("io.file.buffer.size", 4 * 1024)) : new BZip2Codec.BZip2CompressionOutputStream
(@out));
}
/// <summary>
/// Create a
/// <see cref="CompressionInputStream"/>
/// that will read from the given
/// <see cref="System.IO.InputStream"/>
/// with the given
/// <see cref="Decompressor"/>
/// , and return a
/// stream for uncompressed data.
/// </summary>
/// <param name="in">the stream to read compressed bytes from</param>
/// <param name="decompressor">decompressor to use</param>
/// <returns>a stream to read uncompressed bytes from</returns>
/// <exception cref="System.IO.IOException"/>
public virtual CompressionInputStream CreateInputStream(InputStream @in, Decompressor
decompressor)
{
return(Bzip2Factory.IsNativeBzip2Loaded(conf) ? new DecompressorStream(@in, decompressor
, conf.GetInt("io.file.buffer.size", 4 * 1024)) : new BZip2Codec.BZip2CompressionInputStream
(@in));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.TypeLoadException"/>
/// <exception cref="InstantiationException"/>
/// <exception cref="System.MemberAccessException"/>
public virtual void TestSequenceFileBZip2NativeCodec()
{
Configuration conf = new Configuration();
conf.Set("io.compression.codec.bzip2.library", "system-native");
if (NativeCodeLoader.IsNativeCodeLoaded())
{
if (Bzip2Factory.IsNativeBzip2Loaded(conf))
{
SequenceFileCodecTest(conf, 0, "org.apache.hadoop.io.compress.BZip2Codec", 100);
SequenceFileCodecTest(conf, 100, "org.apache.hadoop.io.compress.BZip2Codec", 100);
SequenceFileCodecTest(conf, 200000, "org.apache.hadoop.io.compress.BZip2Codec", 1000000
);
}
else
{
Log.Warn("Native hadoop library available but native bzip2 is not");
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestBZip2NativeCodec()
{
Configuration conf = new Configuration();
conf.Set("io.compression.codec.bzip2.library", "system-native");
if (NativeCodeLoader.IsNativeCodeLoaded())
{
if (Bzip2Factory.IsNativeBzip2Loaded(conf))
{
CodecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec");
CodecTest(conf, seed, count, "org.apache.hadoop.io.compress.BZip2Codec");
conf.Set("io.compression.codec.bzip2.library", "java-builtin");
CodecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec");
CodecTest(conf, seed, count, "org.apache.hadoop.io.compress.BZip2Codec");
}
else
{
Log.Warn("Native hadoop library available but native bzip2 is not");
}
}
}
/// <summary>A tool to test native library availability,</summary>
public static void Main(string[] args)
{
string usage = "NativeLibraryChecker [-a|-h]\n" + " -a use -a to check all libraries are available\n"
+ " by default just check hadoop library (and\n" + " winutils.exe on Windows OS) is available\n"
+ " exit with error code 1 if check failed\n" + " -h print this message\n";
if (args.Length > 1 || (args.Length == 1 && !(args[0].Equals("-a") || args[0].Equals
("-h"))))
{
System.Console.Error.WriteLine(usage);
ExitUtil.Terminate(1);
}
bool checkAll = false;
if (args.Length == 1)
{
if (args[0].Equals("-h"))
{
System.Console.Out.WriteLine(usage);
return;
}
checkAll = true;
}
Configuration conf = new Configuration();
bool nativeHadoopLoaded = NativeCodeLoader.IsNativeCodeLoaded();
bool zlibLoaded = false;
bool snappyLoaded = false;
// lz4 is linked within libhadoop
bool lz4Loaded = nativeHadoopLoaded;
bool bzip2Loaded = Bzip2Factory.IsNativeBzip2Loaded(conf);
bool openSslLoaded = false;
bool winutilsExists = false;
string openSslDetail = string.Empty;
string hadoopLibraryName = string.Empty;
string zlibLibraryName = string.Empty;
string snappyLibraryName = string.Empty;
string lz4LibraryName = string.Empty;
string bzip2LibraryName = string.Empty;
string winutilsPath = null;
if (nativeHadoopLoaded)
{
hadoopLibraryName = NativeCodeLoader.GetLibraryName();
zlibLoaded = ZlibFactory.IsNativeZlibLoaded(conf);
if (zlibLoaded)
{
zlibLibraryName = ZlibFactory.GetLibraryName();
}
snappyLoaded = NativeCodeLoader.BuildSupportsSnappy() && SnappyCodec.IsNativeCodeLoaded
();
if (snappyLoaded && NativeCodeLoader.BuildSupportsSnappy())
{
snappyLibraryName = SnappyCodec.GetLibraryName();
}
if (OpensslCipher.GetLoadingFailureReason() != null)
{
openSslDetail = OpensslCipher.GetLoadingFailureReason();
openSslLoaded = false;
}
else
{
openSslDetail = OpensslCipher.GetLibraryName();
openSslLoaded = true;
}
if (lz4Loaded)
{
lz4LibraryName = Lz4Codec.GetLibraryName();
}
if (bzip2Loaded)
{
bzip2LibraryName = Bzip2Factory.GetLibraryName(conf);
}
}
// winutils.exe is required on Windows
winutilsPath = Shell.GetWinUtilsPath();
if (winutilsPath != null)
{
winutilsExists = true;
}
else
{
winutilsPath = string.Empty;
}
System.Console.Out.WriteLine("Native library checking:");
System.Console.Out.Printf("hadoop: %b %s%n", nativeHadoopLoaded, hadoopLibraryName
);
System.Console.Out.Printf("zlib: %b %s%n", zlibLoaded, zlibLibraryName);
System.Console.Out.Printf("snappy: %b %s%n", snappyLoaded, snappyLibraryName);
System.Console.Out.Printf("lz4: %b %s%n", lz4Loaded, lz4LibraryName);
System.Console.Out.Printf("bzip2: %b %s%n", bzip2Loaded, bzip2LibraryName);
System.Console.Out.Printf("openssl: %b %s%n", openSslLoaded, openSslDetail);
if (Shell.Windows)
{
System.Console.Out.Printf("winutils: %b %s%n", winutilsExists, winutilsPath);
}
if ((!nativeHadoopLoaded) || (Shell.Windows && (!winutilsExists)) || (checkAll &&
!(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded)))
{
// return 1 to indicated check failed
ExitUtil.Terminate(1);
}
}
/// <summary>
/// Create a new
/// <see cref="Decompressor"/>
/// for use by this
/// <see cref="CompressionCodec"/>
/// .
/// </summary>
/// <returns>a new decompressor for use by this codec</returns>
public virtual Decompressor CreateDecompressor()
{
return(Bzip2Factory.GetBzip2Decompressor(conf));
}
/// <summary>
/// Get the type of
/// <see cref="Decompressor"/>
/// needed by this
/// <see cref="CompressionCodec"/>
/// .
/// </summary>
/// <returns>the type of decompressor needed by this codec.</returns>
public virtual Type GetDecompressorType()
{
return(Bzip2Factory.GetBzip2DecompressorType(conf));
}
/// <summary>
/// Create a new
/// <see cref="Compressor"/>
/// for use by this
/// <see cref="CompressionCodec"/>
/// .
/// </summary>
/// <returns>a new compressor for use by this codec</returns>
public virtual Compressor CreateCompressor()
{
return(Bzip2Factory.GetBzip2Compressor(conf));
}