/// <exception cref="System.Exception"/> public bool Call() { Compressor c = queue.Take(); CodecPool.ReturnCompressor(c); return(c != null); }
/// <exception cref="System.IO.IOException"/> public virtual void TestGzipCodecWrite(bool useNative) { // Create a gzipped file using a compressor from the CodecPool, // and try to read it back via the regular GZIPInputStream. // Use native libs per the parameter Configuration conf = new Configuration(); conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, useNative); if (useNative) { if (!ZlibFactory.IsNativeZlibLoaded(conf)) { Log.Warn("testGzipCodecWrite skipped: native libs not loaded"); return; } } else { NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request" , ZlibFactory.IsNativeZlibLoaded(conf)); } // Ensure that the CodecPool has a BuiltInZlibDeflater in it. Compressor zlibCompressor = ZlibFactory.GetZlibCompressor(conf); NUnit.Framework.Assert.IsNotNull("zlibCompressor is null!", zlibCompressor); Assert.True("ZlibFactory returned unexpected deflator", useNative ? zlibCompressor is ZlibCompressor : zlibCompressor is BuiltInZlibDeflater); CodecPool.ReturnCompressor(zlibCompressor); // Create a GZIP text file via the Compressor interface. CompressionCodecFactory ccf = new CompressionCodecFactory(conf); CompressionCodec codec = ccf.GetCodec(new Path("foo.gz")); Assert.True("Codec for .gz file is not GzipCodec", codec is GzipCodec ); string msg = "This is the message we are going to compress."; string tmpDir = Runtime.GetProperty("test.build.data", "/tmp/"); string fileName = new Path(new Path(tmpDir), "testGzipCodecWrite.txt.gz").ToString (); BufferedWriter w = null; Compressor gzipCompressor = CodecPool.GetCompressor(codec); if (null != gzipCompressor) { // If it gives us back a Compressor, we should be able to use this // to write files we can then read back with Java's gzip tools. OutputStream os = new CompressorStream(new FileOutputStream(fileName), gzipCompressor ); w = new BufferedWriter(new OutputStreamWriter(os)); w.Write(msg); w.Close(); CodecPool.ReturnCompressor(gzipCompressor); VerifyGzipFile(fileName, msg); } // Create a gzip text file via codec.getOutputStream(). w = new BufferedWriter(new OutputStreamWriter(codec.CreateOutputStream(new FileOutputStream (fileName)))); w.Write(msg); w.Close(); VerifyGzipFile(fileName, msg); }
/// <exception cref="System.IO.IOException"/> public override void Close() { Finish(); @out.Close(); if (trackedCompressor != null) { CodecPool.ReturnCompressor(trackedCompressor); trackedCompressor = null; } }
public virtual void TestCompressorNotReturnSameInstance() { Compressor comp = CodecPool.GetCompressor(codec); CodecPool.ReturnCompressor(comp); CodecPool.ReturnCompressor(comp); ICollection <Compressor> compressors = new HashSet <Compressor>(); for (int i = 0; i < 10; ++i) { compressors.AddItem(CodecPool.GetCompressor(codec)); } Assert.Equal(10, compressors.Count); foreach (Compressor compressor in compressors) { CodecPool.ReturnCompressor(compressor); } }
public virtual void TestCompressorPoolCounts() { // Get two compressors and return them Compressor comp1 = CodecPool.GetCompressor(codec); Compressor comp2 = CodecPool.GetCompressor(codec); Assert.Equal(LeaseCountErr, 2, CodecPool.GetLeasedCompressorsCount (codec)); CodecPool.ReturnCompressor(comp2); Assert.Equal(LeaseCountErr, 1, CodecPool.GetLeasedCompressorsCount (codec)); CodecPool.ReturnCompressor(comp1); Assert.Equal(LeaseCountErr, 0, CodecPool.GetLeasedCompressorsCount (codec)); CodecPool.ReturnCompressor(comp1); Assert.Equal(LeaseCountErr, 0, CodecPool.GetLeasedCompressorsCount (codec)); }
public virtual void TestCodecPoolGzipReuse() { Configuration conf = new Configuration(); conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, true); if (!ZlibFactory.IsNativeZlibLoaded(conf)) { Log.Warn("testCodecPoolGzipReuse skipped: native libs not loaded"); return; } GzipCodec gzc = ReflectionUtils.NewInstance <GzipCodec>(conf); DefaultCodec dfc = ReflectionUtils.NewInstance <DefaultCodec>(conf); Compressor c1 = CodecPool.GetCompressor(gzc); Compressor c2 = CodecPool.GetCompressor(dfc); CodecPool.ReturnCompressor(c1); CodecPool.ReturnCompressor(c2); Assert.True("Got mismatched ZlibCompressor", c2 != CodecPool.GetCompressor (gzc)); }
/// <exception cref="System.IO.IOException"/> private static void GzipReinitTest(Configuration conf, CompressionCodec codec) { // Add codec to cache ZlibFactory.SetCompressionLevel(conf, ZlibCompressor.CompressionLevel.BestCompression ); ZlibFactory.SetCompressionStrategy(conf, ZlibCompressor.CompressionStrategy.DefaultStrategy ); Compressor c1 = CodecPool.GetCompressor(codec); CodecPool.ReturnCompressor(c1); // reset compressor's compression level to perform no compression ZlibFactory.SetCompressionLevel(conf, ZlibCompressor.CompressionLevel.NoCompression ); Compressor c2 = CodecPool.GetCompressor(codec, conf); // ensure same compressor placed earlier Assert.True("Got mismatched ZlibCompressor", c1 == c2); ByteArrayOutputStream bos = new ByteArrayOutputStream(); CompressionOutputStream cos = null; // write trivially compressable data byte[] b = new byte[1 << 15]; Arrays.Fill(b, unchecked ((byte)43)); try { cos = codec.CreateOutputStream(bos, c2); cos.Write(b); } finally { if (cos != null) { cos.Close(); } CodecPool.ReturnCompressor(c2); } byte[] outbytes = bos.ToByteArray(); // verify data were not compressed Assert.True("Compressed bytes contrary to configuration", outbytes .Length >= b.Length); }
/// <summary>Write infLen bytes (deflated) to file in test dir using codec.</summary> /// <remarks> /// Write infLen bytes (deflated) to file in test dir using codec. /// Records are of the form /// <i><b64 rand><i+i><b64 rand> /// </remarks> /// <exception cref="System.IO.IOException"/> private static Path WriteSplitTestFile(FileSystem fs, Random rand, CompressionCodec codec, long infLen) { int RecSize = 1024; Path wd = new Path(new Path(Runtime.GetProperty("test.build.data", "/tmp")).MakeQualified (fs), codec.GetType().Name); Path file = new Path(wd, "test" + codec.GetDefaultExtension()); byte[] b = new byte[RecSize]; Base64 b64 = new Base64(0, null); DataOutputStream fout = null; Compressor cmp = CodecPool.GetCompressor(codec); try { fout = new DataOutputStream(codec.CreateOutputStream(fs.Create(file, true), cmp)); DataOutputBuffer dob = new DataOutputBuffer(RecSize * 4 / 3 + 4); int seq = 0; while (infLen > 0) { rand.NextBytes(b); byte[] b64enc = b64.Encode(b); // ensures rand printable, no LF dob.Reset(); dob.WriteInt(seq); System.Array.Copy(dob.GetData(), 0, b64enc, 0, dob.GetLength()); fout.Write(b64enc); fout.Write('\n'); ++seq; infLen -= b64enc.Length; } Log.Info("Wrote " + seq + " records to " + file); } finally { IOUtils.Cleanup(Log, fout); CodecPool.ReturnCompressor(cmp); } return(file); }
/// <summary>Create an output stream with a codec taken from the global CodecPool.</summary> /// <param name="codec">The codec to use to create the output stream.</param> /// <param name="conf">The configuration to use if we need to create a new codec.</param> /// <param name="out">The output stream to wrap.</param> /// <returns>The new output stream</returns> /// <exception cref="System.IO.IOException"/> internal static CompressionOutputStream CreateOutputStreamWithCodecPool(CompressionCodec codec, Configuration conf, OutputStream @out) { Compressor compressor = CodecPool.GetCompressor(codec, conf); CompressionOutputStream stream = null; try { stream = codec.CreateOutputStream(@out, compressor); } finally { if (stream == null) { CodecPool.ReturnCompressor(compressor); } else { stream.SetTrackedCompressor(compressor); } } return(stream); }