コード例 #1
0
        /// <exception cref="System.IO.IOException"/>
        private static void CodecTestWithNOCompression(Configuration conf, string codecClass
                                                       )
        {
            // Create a compressor with NO_COMPRESSION and make sure that
            // output is not compressed by comparing the size with the
            // original input
            CompressionCodec codec = null;

            ZlibFactory.SetCompressionLevel(conf, ZlibCompressor.CompressionLevel.NoCompression
                                            );
            try
            {
                codec = (CompressionCodec)ReflectionUtils.NewInstance(conf.GetClassByName(codecClass
                                                                                          ), conf);
            }
            catch (TypeLoadException)
            {
                throw new IOException("Illegal codec!");
            }
            Compressor c = codec.CreateCompressor();
            // ensure same compressor placed earlier
            ByteArrayOutputStream   bos = new ByteArrayOutputStream();
            CompressionOutputStream cos = null;

            // write trivially compressable data
            byte[] b = new byte[1 << 15];
            Arrays.Fill(b, unchecked ((byte)43));
            try
            {
                cos = codec.CreateOutputStream(bos, c);
                cos.Write(b);
            }
            finally
            {
                if (cos != null)
                {
                    cos.Close();
                }
            }
            byte[] outbytes = bos.ToByteArray();
            // verify data were not compressed
            Assert.True("Compressed bytes contrary to configuration(NO_COMPRESSION)"
                        , outbytes.Length >= b.Length);
        }
コード例 #2
0
        /// <exception cref="System.IO.IOException"/>
        private static void GzipReinitTest(Configuration conf, CompressionCodec codec)
        {
            // Add codec to cache
            ZlibFactory.SetCompressionLevel(conf, ZlibCompressor.CompressionLevel.BestCompression
                                            );
            ZlibFactory.SetCompressionStrategy(conf, ZlibCompressor.CompressionStrategy.DefaultStrategy
                                               );
            Compressor c1 = CodecPool.GetCompressor(codec);

            CodecPool.ReturnCompressor(c1);
            // reset compressor's compression level to perform no compression
            ZlibFactory.SetCompressionLevel(conf, ZlibCompressor.CompressionLevel.NoCompression
                                            );
            Compressor c2 = CodecPool.GetCompressor(codec, conf);

            // ensure same compressor placed earlier
            Assert.True("Got mismatched ZlibCompressor", c1 == c2);
            ByteArrayOutputStream   bos = new ByteArrayOutputStream();
            CompressionOutputStream cos = null;

            // write trivially compressable data
            byte[] b = new byte[1 << 15];
            Arrays.Fill(b, unchecked ((byte)43));
            try
            {
                cos = codec.CreateOutputStream(bos, c2);
                cos.Write(b);
            }
            finally
            {
                if (cos != null)
                {
                    cos.Close();
                }
                CodecPool.ReturnCompressor(c2);
            }
            byte[] outbytes = bos.ToByteArray();
            // verify data were not compressed
            Assert.True("Compressed bytes contrary to configuration", outbytes
                        .Length >= b.Length);
        }
コード例 #3
0
        /// <summary>A little test program.</summary>
        /// <param name="args"/>
        /// <exception cref="System.Exception"/>
        public static void Main(string[] args)
        {
            Configuration conf = new Configuration();

            Org.Apache.Hadoop.IO.Compress.CompressionCodecFactory factory = new Org.Apache.Hadoop.IO.Compress.CompressionCodecFactory
                                                                                (conf);
            bool encode = false;

            for (int i = 0; i < args.Length; ++i)
            {
                if ("-in".Equals(args[i]))
                {
                    encode = true;
                }
                else
                {
                    if ("-out".Equals(args[i]))
                    {
                        encode = false;
                    }
                    else
                    {
                        CompressionCodec codec = factory.GetCodec(new Path(args[i]));
                        if (codec == null)
                        {
                            System.Console.Out.WriteLine("Codec for " + args[i] + " not found.");
                        }
                        else
                        {
                            if (encode)
                            {
                                CompressionOutputStream @out = null;
                                InputStream             @in  = null;
                                try
                                {
                                    @out = codec.CreateOutputStream(new FileOutputStream(args[i]));
                                    byte[] buffer     = new byte[100];
                                    string inFilename = RemoveSuffix(args[i], codec.GetDefaultExtension());
                                    @in = new FileInputStream(inFilename);
                                    int len = @in.Read(buffer);
                                    while (len > 0)
                                    {
                                        @out.Write(buffer, 0, len);
                                        len = @in.Read(buffer);
                                    }
                                }
                                finally
                                {
                                    if (@out != null)
                                    {
                                        @out.Close();
                                    }
                                    if (@in != null)
                                    {
                                        @in.Close();
                                    }
                                }
                            }
                            else
                            {
                                CompressionInputStream @in = null;
                                try
                                {
                                    @in = codec.CreateInputStream(new FileInputStream(args[i]));
                                    byte[] buffer = new byte[100];
                                    int    len    = @in.Read(buffer);
                                    while (len > 0)
                                    {
                                        System.Console.Out.Write(buffer, 0, len);
                                        len = @in.Read(buffer);
                                    }
                                }
                                finally
                                {
                                    if (@in != null)
                                    {
                                        @in.Close();
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }