Exemplo n.º 1
0
            /// <exception cref="System.Exception"/>
            public bool Call()
            {
                Compressor c = queue.Take();

                CodecPool.ReturnCompressor(c);
                return(c != null);
            }
Exemplo n.º 2
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestGzipCodecWrite(bool useNative)
        {
            // Create a gzipped file using a compressor from the CodecPool,
            // and try to read it back via the regular GZIPInputStream.
            // Use native libs per the parameter
            Configuration conf = new Configuration();

            conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, useNative);
            if (useNative)
            {
                if (!ZlibFactory.IsNativeZlibLoaded(conf))
                {
                    Log.Warn("testGzipCodecWrite skipped: native libs not loaded");
                    return;
                }
            }
            else
            {
                NUnit.Framework.Assert.IsFalse("ZlibFactory is using native libs against request"
                                               , ZlibFactory.IsNativeZlibLoaded(conf));
            }
            // Ensure that the CodecPool has a BuiltInZlibDeflater in it.
            Compressor zlibCompressor = ZlibFactory.GetZlibCompressor(conf);

            NUnit.Framework.Assert.IsNotNull("zlibCompressor is null!", zlibCompressor);
            Assert.True("ZlibFactory returned unexpected deflator", useNative
                                 ? zlibCompressor is ZlibCompressor : zlibCompressor is BuiltInZlibDeflater);
            CodecPool.ReturnCompressor(zlibCompressor);
            // Create a GZIP text file via the Compressor interface.
            CompressionCodecFactory ccf   = new CompressionCodecFactory(conf);
            CompressionCodec        codec = ccf.GetCodec(new Path("foo.gz"));

            Assert.True("Codec for .gz file is not GzipCodec", codec is GzipCodec
                        );
            string msg      = "This is the message we are going to compress.";
            string tmpDir   = Runtime.GetProperty("test.build.data", "/tmp/");
            string fileName = new Path(new Path(tmpDir), "testGzipCodecWrite.txt.gz").ToString
                                  ();
            BufferedWriter w = null;
            Compressor     gzipCompressor = CodecPool.GetCompressor(codec);

            if (null != gzipCompressor)
            {
                // If it gives us back a Compressor, we should be able to use this
                // to write files we can then read back with Java's gzip tools.
                OutputStream os = new CompressorStream(new FileOutputStream(fileName), gzipCompressor
                                                       );
                w = new BufferedWriter(new OutputStreamWriter(os));
                w.Write(msg);
                w.Close();
                CodecPool.ReturnCompressor(gzipCompressor);
                VerifyGzipFile(fileName, msg);
            }
            // Create a gzip text file via codec.getOutputStream().
            w = new BufferedWriter(new OutputStreamWriter(codec.CreateOutputStream(new FileOutputStream
                                                                                       (fileName))));
            w.Write(msg);
            w.Close();
            VerifyGzipFile(fileName, msg);
        }
 /// <exception cref="System.IO.IOException"/>
 public override void Close()
 {
     Finish();
     @out.Close();
     if (trackedCompressor != null)
     {
         CodecPool.ReturnCompressor(trackedCompressor);
         trackedCompressor = null;
     }
 }
Exemplo n.º 4
0
        public virtual void TestCompressorNotReturnSameInstance()
        {
            Compressor comp = CodecPool.GetCompressor(codec);

            CodecPool.ReturnCompressor(comp);
            CodecPool.ReturnCompressor(comp);
            ICollection <Compressor> compressors = new HashSet <Compressor>();

            for (int i = 0; i < 10; ++i)
            {
                compressors.AddItem(CodecPool.GetCompressor(codec));
            }
            Assert.Equal(10, compressors.Count);
            foreach (Compressor compressor in compressors)
            {
                CodecPool.ReturnCompressor(compressor);
            }
        }
Exemplo n.º 5
0
        public virtual void TestCompressorPoolCounts()
        {
            // Get two compressors and return them
            Compressor comp1 = CodecPool.GetCompressor(codec);
            Compressor comp2 = CodecPool.GetCompressor(codec);

            Assert.Equal(LeaseCountErr, 2, CodecPool.GetLeasedCompressorsCount
                             (codec));
            CodecPool.ReturnCompressor(comp2);
            Assert.Equal(LeaseCountErr, 1, CodecPool.GetLeasedCompressorsCount
                             (codec));
            CodecPool.ReturnCompressor(comp1);
            Assert.Equal(LeaseCountErr, 0, CodecPool.GetLeasedCompressorsCount
                             (codec));
            CodecPool.ReturnCompressor(comp1);
            Assert.Equal(LeaseCountErr, 0, CodecPool.GetLeasedCompressorsCount
                             (codec));
        }
Exemplo n.º 6
0
        public virtual void TestCodecPoolGzipReuse()
        {
            Configuration conf = new Configuration();

            conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, true);
            if (!ZlibFactory.IsNativeZlibLoaded(conf))
            {
                Log.Warn("testCodecPoolGzipReuse skipped: native libs not loaded");
                return;
            }
            GzipCodec    gzc = ReflectionUtils.NewInstance <GzipCodec>(conf);
            DefaultCodec dfc = ReflectionUtils.NewInstance <DefaultCodec>(conf);
            Compressor   c1  = CodecPool.GetCompressor(gzc);
            Compressor   c2  = CodecPool.GetCompressor(dfc);

            CodecPool.ReturnCompressor(c1);
            CodecPool.ReturnCompressor(c2);
            Assert.True("Got mismatched ZlibCompressor", c2 != CodecPool.GetCompressor
                            (gzc));
        }
Exemplo n.º 7
0
        /// <exception cref="System.IO.IOException"/>
        private static void GzipReinitTest(Configuration conf, CompressionCodec codec)
        {
            // Add codec to cache
            ZlibFactory.SetCompressionLevel(conf, ZlibCompressor.CompressionLevel.BestCompression
                                            );
            ZlibFactory.SetCompressionStrategy(conf, ZlibCompressor.CompressionStrategy.DefaultStrategy
                                               );
            Compressor c1 = CodecPool.GetCompressor(codec);

            CodecPool.ReturnCompressor(c1);
            // reset compressor's compression level to perform no compression
            ZlibFactory.SetCompressionLevel(conf, ZlibCompressor.CompressionLevel.NoCompression
                                            );
            Compressor c2 = CodecPool.GetCompressor(codec, conf);

            // ensure same compressor placed earlier
            Assert.True("Got mismatched ZlibCompressor", c1 == c2);
            ByteArrayOutputStream   bos = new ByteArrayOutputStream();
            CompressionOutputStream cos = null;

            // write trivially compressable data
            byte[] b = new byte[1 << 15];
            Arrays.Fill(b, unchecked ((byte)43));
            try
            {
                cos = codec.CreateOutputStream(bos, c2);
                cos.Write(b);
            }
            finally
            {
                if (cos != null)
                {
                    cos.Close();
                }
                CodecPool.ReturnCompressor(c2);
            }
            byte[] outbytes = bos.ToByteArray();
            // verify data were not compressed
            Assert.True("Compressed bytes contrary to configuration", outbytes
                        .Length >= b.Length);
        }
Exemplo n.º 8
0
        /// <summary>Write infLen bytes (deflated) to file in test dir using codec.</summary>
        /// <remarks>
        /// Write infLen bytes (deflated) to file in test dir using codec.
        /// Records are of the form
        /// &lt;i&gt;&lt;b64 rand&gt;&lt;i+i&gt;&lt;b64 rand&gt;
        /// </remarks>
        /// <exception cref="System.IO.IOException"/>
        private static Path WriteSplitTestFile(FileSystem fs, Random rand, CompressionCodec
                                               codec, long infLen)
        {
            int  RecSize = 1024;
            Path wd      = new Path(new Path(Runtime.GetProperty("test.build.data", "/tmp")).MakeQualified
                                        (fs), codec.GetType().Name);
            Path file = new Path(wd, "test" + codec.GetDefaultExtension());

            byte[]           b    = new byte[RecSize];
            Base64           b64  = new Base64(0, null);
            DataOutputStream fout = null;
            Compressor       cmp  = CodecPool.GetCompressor(codec);

            try
            {
                fout = new DataOutputStream(codec.CreateOutputStream(fs.Create(file, true), cmp));
                DataOutputBuffer dob = new DataOutputBuffer(RecSize * 4 / 3 + 4);
                int seq = 0;
                while (infLen > 0)
                {
                    rand.NextBytes(b);
                    byte[] b64enc = b64.Encode(b);
                    // ensures rand printable, no LF
                    dob.Reset();
                    dob.WriteInt(seq);
                    System.Array.Copy(dob.GetData(), 0, b64enc, 0, dob.GetLength());
                    fout.Write(b64enc);
                    fout.Write('\n');
                    ++seq;
                    infLen -= b64enc.Length;
                }
                Log.Info("Wrote " + seq + " records to " + file);
            }
            finally
            {
                IOUtils.Cleanup(Log, fout);
                CodecPool.ReturnCompressor(cmp);
            }
            return(file);
        }
Exemplo n.º 9
0
            /// <summary>Create an output stream with a codec taken from the global CodecPool.</summary>
            /// <param name="codec">The codec to use to create the output stream.</param>
            /// <param name="conf">The configuration to use if we need to create a new codec.</param>
            /// <param name="out">The output stream to wrap.</param>
            /// <returns>The new output stream</returns>
            /// <exception cref="System.IO.IOException"/>
            internal static CompressionOutputStream CreateOutputStreamWithCodecPool(CompressionCodec
                                                                                    codec, Configuration conf, OutputStream @out)
            {
                Compressor compressor          = CodecPool.GetCompressor(codec, conf);
                CompressionOutputStream stream = null;

                try
                {
                    stream = codec.CreateOutputStream(@out, compressor);
                }
                finally
                {
                    if (stream == null)
                    {
                        CodecPool.ReturnCompressor(compressor);
                    }
                    else
                    {
                        stream.SetTrackedCompressor(compressor);
                    }
                }
                return(stream);
            }