public virtual void TestCodecPoolAndGzipDecompressor() { // BuiltInZlibInflater should not be used as the GzipCodec decompressor. // Assert that this is the case. // Don't use native libs for this test. Configuration conf = new Configuration(); conf.SetBoolean("hadoop.native.lib", false); NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request" , ZlibFactory.IsNativeZlibLoaded(conf)); // This should give us a BuiltInZlibInflater. Decompressor zlibDecompressor = ZlibFactory.GetZlibDecompressor(conf); NUnit.Framework.Assert.IsNotNull("zlibDecompressor is null!", zlibDecompressor); Assert.True("ZlibFactory returned unexpected inflator", zlibDecompressor is BuiltInZlibInflater); // its createOutputStream() just wraps the existing stream in a // java.util.zip.GZIPOutputStream. CompressionCodecFactory ccf = new CompressionCodecFactory(conf); CompressionCodec codec = ccf.GetCodec(new Path("foo.gz")); Assert.True("Codec for .gz file is not GzipCodec", codec is GzipCodec ); // make sure we don't get a null decompressor Decompressor codecDecompressor = codec.CreateDecompressor(); if (null == codecDecompressor) { NUnit.Framework.Assert.Fail("Got null codecDecompressor"); } // Asking the CodecPool for a decompressor for GzipCodec // should not return null Decompressor poolDecompressor = CodecPool.GetDecompressor(codec); if (null == poolDecompressor) { NUnit.Framework.Assert.Fail("Got null poolDecompressor"); } // return a couple decompressors CodecPool.ReturnDecompressor(zlibDecompressor); CodecPool.ReturnDecompressor(poolDecompressor); Decompressor poolDecompressor2 = CodecPool.GetDecompressor(codec); if (poolDecompressor.GetType() == typeof(BuiltInGzipDecompressor)) { if (poolDecompressor == poolDecompressor2) { NUnit.Framework.Assert.Fail("Reused java gzip decompressor in pool"); } } else { if (poolDecompressor != poolDecompressor2) { NUnit.Framework.Assert.Fail("Did not reuse native gzip decompressor in pool"); } } }
public virtual void TestGzipLongOverflow() { Log.Info("testGzipLongOverflow"); // Don't use native libs for this test. Configuration conf = new Configuration(); conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false); NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request" , ZlibFactory.IsNativeZlibLoaded(conf)); // Ensure that the CodecPool has a BuiltInZlibInflater in it. Decompressor zlibDecompressor = ZlibFactory.GetZlibDecompressor(conf); NUnit.Framework.Assert.IsNotNull("zlibDecompressor is null!", zlibDecompressor); Assert.True("ZlibFactory returned unexpected inflator", zlibDecompressor is BuiltInZlibInflater); CodecPool.ReturnDecompressor(zlibDecompressor); // Now create a GZip text file. string tmpDir = Runtime.GetProperty("test.build.data", "/tmp/"); Path f = new Path(new Path(tmpDir), "testGzipLongOverflow.bin.gz"); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream (new FileOutputStream(f.ToString())))); int Nbuf = 1024 * 4 + 1; char[] buf = new char[1024 * 1024]; for (int i = 0; i < buf.Length; i++) { buf[i] = '\0'; } for (int i_1 = 0; i_1 < Nbuf; i_1++) { bw.Write(buf); } bw.Close(); // Now read it back, using the CodecPool to establish the // decompressor to use. CompressionCodecFactory ccf = new CompressionCodecFactory(conf); CompressionCodec codec = ccf.GetCodec(f); Decompressor decompressor = CodecPool.GetDecompressor(codec); FileSystem fs = FileSystem.GetLocal(conf); InputStream @is = fs.Open(f); @is = codec.CreateInputStream(@is, decompressor); BufferedReader br = new BufferedReader(new InputStreamReader(@is)); for (int j = 0; j < Nbuf; j++) { int n = br.Read(buf); Assert.Equal("got wrong read length!", n, buf.Length); for (int i_2 = 0; i_2 < buf.Length; i_2++) { Assert.Equal("got wrong byte!", buf[i_2], '\0'); } } br.Close(); }
public virtual void TestGzipCodecRead() { // Create a gzipped file and try to read it back, using a decompressor // from the CodecPool. // Don't use native libs for this test. Configuration conf = new Configuration(); conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false); NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request" , ZlibFactory.IsNativeZlibLoaded(conf)); // Ensure that the CodecPool has a BuiltInZlibInflater in it. Decompressor zlibDecompressor = ZlibFactory.GetZlibDecompressor(conf); NUnit.Framework.Assert.IsNotNull("zlibDecompressor is null!", zlibDecompressor); Assert.True("ZlibFactory returned unexpected inflator", zlibDecompressor is BuiltInZlibInflater); CodecPool.ReturnDecompressor(zlibDecompressor); // Now create a GZip text file. string tmpDir = Runtime.GetProperty("test.build.data", "/tmp/"); Path f = new Path(new Path(tmpDir), "testGzipCodecRead.txt.gz"); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream (new FileOutputStream(f.ToString())))); string msg = "This is the message in the file!"; bw.Write(msg); bw.Close(); // Now read it back, using the CodecPool to establish the // decompressor to use. CompressionCodecFactory ccf = new CompressionCodecFactory(conf); CompressionCodec codec = ccf.GetCodec(f); Decompressor decompressor = CodecPool.GetDecompressor(codec); FileSystem fs = FileSystem.GetLocal(conf); InputStream @is = fs.Open(f); @is = codec.CreateInputStream(@is, decompressor); BufferedReader br = new BufferedReader(new InputStreamReader(@is)); string line = br.ReadLine(); Assert.Equal("Didn't get the same message back!", msg, line); br.Close(); }
public virtual Decompressor CreateDecompressor() { return(ZlibFactory.GetZlibDecompressor(conf)); }