/// <exception cref="System.IO.IOException"/> private static void CodecTestWithNOCompression(Configuration conf, string codecClass ) { // Create a compressor with NO_COMPRESSION and make sure that // output is not compressed by comparing the size with the // original input CompressionCodec codec = null; ZlibFactory.SetCompressionLevel(conf, ZlibCompressor.CompressionLevel.NoCompression ); try { codec = (CompressionCodec)ReflectionUtils.NewInstance(conf.GetClassByName(codecClass ), conf); } catch (TypeLoadException) { throw new IOException("Illegal codec!"); } Compressor c = codec.CreateCompressor(); // ensure same compressor placed earlier ByteArrayOutputStream bos = new ByteArrayOutputStream(); CompressionOutputStream cos = null; // write trivially compressable data byte[] b = new byte[1 << 15]; Arrays.Fill(b, unchecked ((byte)43)); try { cos = codec.CreateOutputStream(bos, c); cos.Write(b); } finally { if (cos != null) { cos.Close(); } } byte[] outbytes = bos.ToByteArray(); // verify data were not compressed Assert.True("Compressed bytes contrary to configuration(NO_COMPRESSION)" , outbytes.Length >= b.Length); }
/// <exception cref="System.IO.IOException"/> private static void GzipReinitTest(Configuration conf, CompressionCodec codec) { // Add codec to cache ZlibFactory.SetCompressionLevel(conf, ZlibCompressor.CompressionLevel.BestCompression ); ZlibFactory.SetCompressionStrategy(conf, ZlibCompressor.CompressionStrategy.DefaultStrategy ); Compressor c1 = CodecPool.GetCompressor(codec); CodecPool.ReturnCompressor(c1); // reset compressor's compression level to perform no compression ZlibFactory.SetCompressionLevel(conf, ZlibCompressor.CompressionLevel.NoCompression ); Compressor c2 = CodecPool.GetCompressor(codec, conf); // ensure same compressor placed earlier Assert.True("Got mismatched ZlibCompressor", c1 == c2); ByteArrayOutputStream bos = new ByteArrayOutputStream(); CompressionOutputStream cos = null; // write trivially compressable data byte[] b = new byte[1 << 15]; Arrays.Fill(b, unchecked ((byte)43)); try { cos = codec.CreateOutputStream(bos, c2); cos.Write(b); } finally { if (cos != null) { cos.Close(); } CodecPool.ReturnCompressor(c2); } byte[] outbytes = bos.ToByteArray(); // verify data were not compressed Assert.True("Compressed bytes contrary to configuration", outbytes .Length >= b.Length); }
/// <summary>Create an output stream with a codec taken from the global CodecPool.</summary> /// <param name="codec">The codec to use to create the output stream.</param> /// <param name="conf">The configuration to use if we need to create a new codec.</param> /// <param name="out">The output stream to wrap.</param> /// <returns>The new output stream</returns> /// <exception cref="System.IO.IOException"/> internal static CompressionOutputStream CreateOutputStreamWithCodecPool(CompressionCodec codec, Configuration conf, OutputStream @out) { Compressor compressor = CodecPool.GetCompressor(codec, conf); CompressionOutputStream stream = null; try { stream = codec.CreateOutputStream(@out, compressor); } finally { if (stream == null) { CodecPool.ReturnCompressor(compressor); } else { stream.SetTrackedCompressor(compressor); } } return(stream); }
/// <exception cref="System.IO.IOException"/> private static void CodecTest(Configuration conf, int seed, int count, string codecClass ) { // Create the codec CompressionCodec codec = null; try { codec = (CompressionCodec)ReflectionUtils.NewInstance(conf.GetClassByName(codecClass ), conf); } catch (TypeLoadException) { throw new IOException("Illegal codec!"); } Log.Info("Created a Codec object of type: " + codecClass); // Generate data DataOutputBuffer data = new DataOutputBuffer(); RandomDatum.Generator generator = new RandomDatum.Generator(seed); for (int i = 0; i < count; ++i) { generator.Next(); RandomDatum key = generator.GetKey(); RandomDatum value = generator.GetValue(); key.Write(data); value.Write(data); } Log.Info("Generated " + count + " records"); // Compress data DataOutputBuffer compressedDataBuffer = new DataOutputBuffer(); CompressionOutputStream deflateFilter = codec.CreateOutputStream(compressedDataBuffer ); DataOutputStream deflateOut = new DataOutputStream(new BufferedOutputStream(deflateFilter )); deflateOut.Write(data.GetData(), 0, data.GetLength()); deflateOut.Flush(); deflateFilter.Finish(); Log.Info("Finished compressing data"); // De-compress data DataInputBuffer deCompressedDataBuffer = new DataInputBuffer(); deCompressedDataBuffer.Reset(compressedDataBuffer.GetData(), 0, compressedDataBuffer .GetLength()); CompressionInputStream inflateFilter = codec.CreateInputStream(deCompressedDataBuffer ); DataInputStream inflateIn = new DataInputStream(new BufferedInputStream(inflateFilter )); // Check DataInputBuffer originalData = new DataInputBuffer(); originalData.Reset(data.GetData(), 0, data.GetLength()); DataInputStream originalIn = new DataInputStream(new BufferedInputStream(originalData )); for (int i_1 = 0; i_1 < count; ++i_1) { RandomDatum k1 = new RandomDatum(); RandomDatum v1 = new RandomDatum(); k1.ReadFields(originalIn); v1.ReadFields(originalIn); RandomDatum k2 = new RandomDatum(); RandomDatum v2 = new RandomDatum(); k2.ReadFields(inflateIn); v2.ReadFields(inflateIn); Assert.True("original and compressed-then-decompressed-output not equal" , k1.Equals(k2) && v1.Equals(v2)); // original and compressed-then-decompressed-output have the same hashCode IDictionary <RandomDatum, string> m = new Dictionary <RandomDatum, string>(); m[k1] = k1.ToString(); m[v1] = v1.ToString(); string result = m[k2]; Assert.Equal("k1 and k2 hashcode not equal", result, k1.ToString ()); result = m[v2]; Assert.Equal("v1 and v2 hashcode not equal", result, v1.ToString ()); } // De-compress data byte-at-a-time originalData.Reset(data.GetData(), 0, data.GetLength()); deCompressedDataBuffer.Reset(compressedDataBuffer.GetData(), 0, compressedDataBuffer .GetLength()); inflateFilter = codec.CreateInputStream(deCompressedDataBuffer); // Check originalIn = new DataInputStream(new BufferedInputStream(originalData)); int expected; do { expected = originalIn.Read(); Assert.Equal("Inflated stream read by byte does not match", expected , inflateFilter.Read()); }while (expected != -1); Log.Info("SUCCESS! Completed checking " + count + " records"); }
/// <summary>A little test program.</summary> /// <param name="args"/> /// <exception cref="System.Exception"/> public static void Main(string[] args) { Configuration conf = new Configuration(); Org.Apache.Hadoop.IO.Compress.CompressionCodecFactory factory = new Org.Apache.Hadoop.IO.Compress.CompressionCodecFactory (conf); bool encode = false; for (int i = 0; i < args.Length; ++i) { if ("-in".Equals(args[i])) { encode = true; } else { if ("-out".Equals(args[i])) { encode = false; } else { CompressionCodec codec = factory.GetCodec(new Path(args[i])); if (codec == null) { System.Console.Out.WriteLine("Codec for " + args[i] + " not found."); } else { if (encode) { CompressionOutputStream @out = null; InputStream @in = null; try { @out = codec.CreateOutputStream(new FileOutputStream(args[i])); byte[] buffer = new byte[100]; string inFilename = RemoveSuffix(args[i], codec.GetDefaultExtension()); @in = new FileInputStream(inFilename); int len = @in.Read(buffer); while (len > 0) { @out.Write(buffer, 0, len); len = @in.Read(buffer); } } finally { if (@out != null) { @out.Close(); } if (@in != null) { @in.Close(); } } } else { CompressionInputStream @in = null; try { @in = codec.CreateInputStream(new FileInputStream(args[i])); byte[] buffer = new byte[100]; int len = @in.Read(buffer); while (len > 0) { System.Console.Out.Write(buffer, 0, len); len = @in.Read(buffer); } } finally { if (@in != null) { @in.Close(); } } } } } } } }