/// <exception cref="System.Exception"/> public bool Call() { Decompressor dc = queue.Take(); CodecPool.ReturnDecompressor(dc); return(dc != null); }
public virtual void TestCodecPoolAndGzipDecompressor() { // BuiltInZlibInflater should not be used as the GzipCodec decompressor. // Assert that this is the case. // Don't use native libs for this test. Configuration conf = new Configuration(); conf.SetBoolean("hadoop.native.lib", false); NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request" , ZlibFactory.IsNativeZlibLoaded(conf)); // This should give us a BuiltInZlibInflater. Decompressor zlibDecompressor = ZlibFactory.GetZlibDecompressor(conf); NUnit.Framework.Assert.IsNotNull("zlibDecompressor is null!", zlibDecompressor); Assert.True("ZlibFactory returned unexpected inflator", zlibDecompressor is BuiltInZlibInflater); // its createOutputStream() just wraps the existing stream in a // java.util.zip.GZIPOutputStream. CompressionCodecFactory ccf = new CompressionCodecFactory(conf); CompressionCodec codec = ccf.GetCodec(new Path("foo.gz")); Assert.True("Codec for .gz file is not GzipCodec", codec is GzipCodec ); // make sure we don't get a null decompressor Decompressor codecDecompressor = codec.CreateDecompressor(); if (null == codecDecompressor) { NUnit.Framework.Assert.Fail("Got null codecDecompressor"); } // Asking the CodecPool for a decompressor for GzipCodec // should not return null Decompressor poolDecompressor = CodecPool.GetDecompressor(codec); if (null == poolDecompressor) { NUnit.Framework.Assert.Fail("Got null poolDecompressor"); } // return a couple decompressors CodecPool.ReturnDecompressor(zlibDecompressor); CodecPool.ReturnDecompressor(poolDecompressor); Decompressor poolDecompressor2 = CodecPool.GetDecompressor(codec); if (poolDecompressor.GetType() == typeof(BuiltInGzipDecompressor)) { if (poolDecompressor == poolDecompressor2) { NUnit.Framework.Assert.Fail("Reused java gzip decompressor in pool"); } } else { if (poolDecompressor != poolDecompressor2) { NUnit.Framework.Assert.Fail("Did not reuse native gzip decompressor in pool"); } } }
/// <exception cref="System.IO.IOException"/> public override void Close() { @in.Close(); if (trackedDecompressor != null) { CodecPool.ReturnDecompressor(trackedDecompressor); trackedDecompressor = null; } }
public virtual void TestGzipLongOverflow() { Log.Info("testGzipLongOverflow"); // Don't use native libs for this test. Configuration conf = new Configuration(); conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false); NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request" , ZlibFactory.IsNativeZlibLoaded(conf)); // Ensure that the CodecPool has a BuiltInZlibInflater in it. Decompressor zlibDecompressor = ZlibFactory.GetZlibDecompressor(conf); NUnit.Framework.Assert.IsNotNull("zlibDecompressor is null!", zlibDecompressor); Assert.True("ZlibFactory returned unexpected inflator", zlibDecompressor is BuiltInZlibInflater); CodecPool.ReturnDecompressor(zlibDecompressor); // Now create a GZip text file. string tmpDir = Runtime.GetProperty("test.build.data", "/tmp/"); Path f = new Path(new Path(tmpDir), "testGzipLongOverflow.bin.gz"); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream (new FileOutputStream(f.ToString())))); int Nbuf = 1024 * 4 + 1; char[] buf = new char[1024 * 1024]; for (int i = 0; i < buf.Length; i++) { buf[i] = '\0'; } for (int i_1 = 0; i_1 < Nbuf; i_1++) { bw.Write(buf); } bw.Close(); // Now read it back, using the CodecPool to establish the // decompressor to use. CompressionCodecFactory ccf = new CompressionCodecFactory(conf); CompressionCodec codec = ccf.GetCodec(f); Decompressor decompressor = CodecPool.GetDecompressor(codec); FileSystem fs = FileSystem.GetLocal(conf); InputStream @is = fs.Open(f); @is = codec.CreateInputStream(@is, decompressor); BufferedReader br = new BufferedReader(new InputStreamReader(@is)); for (int j = 0; j < Nbuf; j++) { int n = br.Read(buf); Assert.Equal("got wrong read length!", n, buf.Length); for (int i_2 = 0; i_2 < buf.Length; i_2++) { Assert.Equal("got wrong byte!", buf[i_2], '\0'); } } br.Close(); }
public virtual void TestDecompressorPoolCounts() { // Get two decompressors and return them Decompressor decomp1 = CodecPool.GetDecompressor(codec); Decompressor decomp2 = CodecPool.GetDecompressor(codec); Assert.Equal(LeaseCountErr, 2, CodecPool.GetLeasedDecompressorsCount (codec)); CodecPool.ReturnDecompressor(decomp2); Assert.Equal(LeaseCountErr, 1, CodecPool.GetLeasedDecompressorsCount (codec)); CodecPool.ReturnDecompressor(decomp1); Assert.Equal(LeaseCountErr, 0, CodecPool.GetLeasedDecompressorsCount (codec)); CodecPool.ReturnDecompressor(decomp1); Assert.Equal(LeaseCountErr, 0, CodecPool.GetLeasedCompressorsCount (codec)); }
public virtual void TestDecompressorNotReturnSameInstance() { Decompressor decomp = CodecPool.GetDecompressor(codec); CodecPool.ReturnDecompressor(decomp); CodecPool.ReturnDecompressor(decomp); ICollection <Decompressor> decompressors = new HashSet <Decompressor>(); for (int i = 0; i < 10; ++i) { decompressors.AddItem(CodecPool.GetDecompressor(codec)); } Assert.Equal(10, decompressors.Count); foreach (Decompressor decompressor in decompressors) { CodecPool.ReturnDecompressor(decompressor); } }
public virtual void TestGzipCodecRead() { // Create a gzipped file and try to read it back, using a decompressor // from the CodecPool. // Don't use native libs for this test. Configuration conf = new Configuration(); conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false); NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request" , ZlibFactory.IsNativeZlibLoaded(conf)); // Ensure that the CodecPool has a BuiltInZlibInflater in it. Decompressor zlibDecompressor = ZlibFactory.GetZlibDecompressor(conf); NUnit.Framework.Assert.IsNotNull("zlibDecompressor is null!", zlibDecompressor); Assert.True("ZlibFactory returned unexpected inflator", zlibDecompressor is BuiltInZlibInflater); CodecPool.ReturnDecompressor(zlibDecompressor); // Now create a GZip text file. string tmpDir = Runtime.GetProperty("test.build.data", "/tmp/"); Path f = new Path(new Path(tmpDir), "testGzipCodecRead.txt.gz"); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream (new FileOutputStream(f.ToString())))); string msg = "This is the message in the file!"; bw.Write(msg); bw.Close(); // Now read it back, using the CodecPool to establish the // decompressor to use. CompressionCodecFactory ccf = new CompressionCodecFactory(conf); CompressionCodec codec = ccf.GetCodec(f); Decompressor decompressor = CodecPool.GetDecompressor(codec); FileSystem fs = FileSystem.GetLocal(conf); InputStream @is = fs.Open(f); @is = codec.CreateInputStream(@is, decompressor); BufferedReader br = new BufferedReader(new InputStreamReader(@is)); string line = br.ReadLine(); Assert.Equal("Didn't get the same message back!", msg, line); br.Close(); }
/// <summary>Create an input stream with a codec taken from the global CodecPool.</summary> /// <param name="codec">The codec to use to create the input stream.</param> /// <param name="conf">The configuration to use if we need to create a new codec.</param> /// <param name="in">The input stream to wrap.</param> /// <returns>The new input stream</returns> /// <exception cref="System.IO.IOException"/> internal static CompressionInputStream CreateInputStreamWithCodecPool(CompressionCodec codec, Configuration conf, InputStream @in) { Decompressor decompressor = CodecPool.GetDecompressor(codec); CompressionInputStream stream = null; try { stream = codec.CreateInputStream(@in, decompressor); } finally { if (stream == null) { CodecPool.ReturnDecompressor(decompressor); } else { stream.SetTrackedDecompressor(decompressor); } } return(stream); }
/// <exception cref="System.IO.IOException"/> private void TestSplitableCodec(Type codecClass) { long Deflbytes = 2 * 1024 * 1024; Configuration conf = new Configuration(); Random rand = new Random(); long seed = rand.NextLong(); Log.Info("seed: " + seed); rand.SetSeed(seed); SplittableCompressionCodec codec = ReflectionUtils.NewInstance(codecClass, conf); FileSystem fs = FileSystem.GetLocal(conf); FileStatus infile = fs.GetFileStatus(WriteSplitTestFile(fs, rand, codec, Deflbytes )); if (infile.GetLen() > int.MaxValue) { NUnit.Framework.Assert.Fail("Unexpected compression: " + Deflbytes + " -> " + infile .GetLen()); } int flen = (int)infile.GetLen(); Text line = new Text(); Decompressor dcmp = CodecPool.GetDecompressor(codec); try { for (int pos = 0; pos < infile.GetLen(); pos += rand.Next(flen / 8)) { // read from random positions, verifying that there exist two sequential // lines as written in writeSplitTestFile SplitCompressionInputStream @in = codec.CreateInputStream(fs.Open(infile.GetPath( )), dcmp, pos, flen, SplittableCompressionCodec.READ_MODE.Byblock); if (@in.GetAdjustedStart() >= flen) { break; } Log.Info("SAMPLE " + @in.GetAdjustedStart() + "," + @in.GetAdjustedEnd()); LineReader lreader = new LineReader(@in); lreader.ReadLine(line); // ignore; likely partial if (@in.GetPos() >= flen) { break; } lreader.ReadLine(line); int seq1 = ReadLeadingInt(line); lreader.ReadLine(line); if (@in.GetPos() >= flen) { break; } int seq2 = ReadLeadingInt(line); Assert.Equal("Mismatched lines", seq1 + 1, seq2); } } finally { CodecPool.ReturnDecompressor(dcmp); } // remove on success fs.Delete(infile.GetPath().GetParent(), true); }