/// <summary> /// Create a /// <see cref="CompressionOutputStream"/> /// that will write to the given /// <see cref="System.IO.OutputStream"/> /// with the given /// <see cref="Compressor"/> /// . /// </summary> /// <param name="out">the location for the final output stream</param> /// <param name="compressor">compressor to use</param> /// <returns> /// a stream the user can write uncompressed data to, to have it /// compressed /// </returns> /// <exception cref="System.IO.IOException"/> public virtual CompressionOutputStream CreateOutputStream(OutputStream @out, Compressor compressor) { return(Bzip2Factory.IsNativeBzip2Loaded(conf) ? new CompressorStream(@out, compressor , conf.GetInt("io.file.buffer.size", 4 * 1024)) : new BZip2Codec.BZip2CompressionOutputStream (@out)); }
/// <summary> /// Create a /// <see cref="CompressionInputStream"/> /// that will read from the given /// <see cref="System.IO.InputStream"/> /// with the given /// <see cref="Decompressor"/> /// , and return a /// stream for uncompressed data. /// </summary> /// <param name="in">the stream to read compressed bytes from</param> /// <param name="decompressor">decompressor to use</param> /// <returns>a stream to read uncompressed bytes from</returns> /// <exception cref="System.IO.IOException"/> public virtual CompressionInputStream CreateInputStream(InputStream @in, Decompressor decompressor) { return(Bzip2Factory.IsNativeBzip2Loaded(conf) ? new DecompressorStream(@in, decompressor , conf.GetInt("io.file.buffer.size", 4 * 1024)) : new BZip2Codec.BZip2CompressionInputStream (@in)); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.TypeLoadException"/> /// <exception cref="InstantiationException"/> /// <exception cref="System.MemberAccessException"/> public virtual void TestSequenceFileBZip2NativeCodec() { Configuration conf = new Configuration(); conf.Set("io.compression.codec.bzip2.library", "system-native"); if (NativeCodeLoader.IsNativeCodeLoaded()) { if (Bzip2Factory.IsNativeBzip2Loaded(conf)) { SequenceFileCodecTest(conf, 0, "org.apache.hadoop.io.compress.BZip2Codec", 100); SequenceFileCodecTest(conf, 100, "org.apache.hadoop.io.compress.BZip2Codec", 100); SequenceFileCodecTest(conf, 200000, "org.apache.hadoop.io.compress.BZip2Codec", 1000000 ); } else { Log.Warn("Native hadoop library available but native bzip2 is not"); } } }
/// <exception cref="System.IO.IOException"/> public virtual void TestBZip2NativeCodec() { Configuration conf = new Configuration(); conf.Set("io.compression.codec.bzip2.library", "system-native"); if (NativeCodeLoader.IsNativeCodeLoaded()) { if (Bzip2Factory.IsNativeBzip2Loaded(conf)) { CodecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec"); CodecTest(conf, seed, count, "org.apache.hadoop.io.compress.BZip2Codec"); conf.Set("io.compression.codec.bzip2.library", "java-builtin"); CodecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec"); CodecTest(conf, seed, count, "org.apache.hadoop.io.compress.BZip2Codec"); } else { Log.Warn("Native hadoop library available but native bzip2 is not"); } } }
/// <summary>A tool to test native library availability,</summary> public static void Main(string[] args) { string usage = "NativeLibraryChecker [-a|-h]\n" + " -a use -a to check all libraries are available\n" + " by default just check hadoop library (and\n" + " winutils.exe on Windows OS) is available\n" + " exit with error code 1 if check failed\n" + " -h print this message\n"; if (args.Length > 1 || (args.Length == 1 && !(args[0].Equals("-a") || args[0].Equals ("-h")))) { System.Console.Error.WriteLine(usage); ExitUtil.Terminate(1); } bool checkAll = false; if (args.Length == 1) { if (args[0].Equals("-h")) { System.Console.Out.WriteLine(usage); return; } checkAll = true; } Configuration conf = new Configuration(); bool nativeHadoopLoaded = NativeCodeLoader.IsNativeCodeLoaded(); bool zlibLoaded = false; bool snappyLoaded = false; // lz4 is linked within libhadoop bool lz4Loaded = nativeHadoopLoaded; bool bzip2Loaded = Bzip2Factory.IsNativeBzip2Loaded(conf); bool openSslLoaded = false; bool winutilsExists = false; string openSslDetail = string.Empty; string hadoopLibraryName = string.Empty; string zlibLibraryName = string.Empty; string snappyLibraryName = string.Empty; string lz4LibraryName = string.Empty; string bzip2LibraryName = string.Empty; string winutilsPath = null; if (nativeHadoopLoaded) { hadoopLibraryName = NativeCodeLoader.GetLibraryName(); zlibLoaded = ZlibFactory.IsNativeZlibLoaded(conf); if (zlibLoaded) { zlibLibraryName = ZlibFactory.GetLibraryName(); } snappyLoaded = NativeCodeLoader.BuildSupportsSnappy() && SnappyCodec.IsNativeCodeLoaded (); if (snappyLoaded && NativeCodeLoader.BuildSupportsSnappy()) { snappyLibraryName = SnappyCodec.GetLibraryName(); } if (OpensslCipher.GetLoadingFailureReason() != null) { openSslDetail = OpensslCipher.GetLoadingFailureReason(); openSslLoaded = false; } else { openSslDetail = OpensslCipher.GetLibraryName(); openSslLoaded = true; } if (lz4Loaded) { lz4LibraryName = Lz4Codec.GetLibraryName(); } if (bzip2Loaded) { bzip2LibraryName = Bzip2Factory.GetLibraryName(conf); } } // winutils.exe is required on Windows winutilsPath = Shell.GetWinUtilsPath(); if (winutilsPath != null) { winutilsExists = true; } else { winutilsPath = string.Empty; } System.Console.Out.WriteLine("Native library checking:"); System.Console.Out.Printf("hadoop: %b %s%n", nativeHadoopLoaded, hadoopLibraryName ); System.Console.Out.Printf("zlib: %b %s%n", zlibLoaded, zlibLibraryName); System.Console.Out.Printf("snappy: %b %s%n", snappyLoaded, snappyLibraryName); System.Console.Out.Printf("lz4: %b %s%n", lz4Loaded, lz4LibraryName); System.Console.Out.Printf("bzip2: %b %s%n", bzip2Loaded, bzip2LibraryName); System.Console.Out.Printf("openssl: %b %s%n", openSslLoaded, openSslDetail); if (Shell.Windows) { System.Console.Out.Printf("winutils: %b %s%n", winutilsExists, winutilsPath); } if ((!nativeHadoopLoaded) || (Shell.Windows && (!winutilsExists)) || (checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) { // return 1 to indicated check failed ExitUtil.Terminate(1); } }