public virtual void TestGzipCompatibility() { Random r = new Random(); long seed = r.NextLong(); r.SetSeed(seed); Log.Info("seed: " + seed); DataOutputBuffer dflbuf = new DataOutputBuffer(); GZIPOutputStream gzout = new GZIPOutputStream(dflbuf); byte[] b = new byte[r.Next(128 * 1024 + 1)]; r.NextBytes(b); gzout.Write(b); gzout.Close(); DataInputBuffer gzbuf = new DataInputBuffer(); gzbuf.Reset(dflbuf.GetData(), dflbuf.GetLength()); Configuration conf = new Configuration(); conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false); CompressionCodec codec = ReflectionUtils.NewInstance <GzipCodec>(conf); Decompressor decom = codec.CreateDecompressor(); NUnit.Framework.Assert.IsNotNull(decom); Assert.Equal(typeof(BuiltInGzipDecompressor), decom.GetType()); InputStream gzin = codec.CreateInputStream(gzbuf, decom); dflbuf.Reset(); IOUtils.CopyBytes(gzin, dflbuf, 4096); byte[] dflchk = Arrays.CopyOf(dflbuf.GetData(), dflbuf.GetLength()); Assert.AssertArrayEquals(b, dflchk); }
/// <exception cref="System.IO.IOException"/> public static int WriteCompressedByteArray(BinaryWriter writer, byte[] bytes) { if (bytes != null) { ByteArrayOutputStream bos = new ByteArrayOutputStream(); GZIPOutputStream gzout = new GZIPOutputStream(bos); try { gzout.Write(bytes, 0, bytes.Length); gzout.Close(); gzout = null; } finally { IOUtils.CloseStream(gzout); } byte[] buffer = bos.ToByteArray(); int len = buffer.Length; @out.WriteInt(len); @out.Write(buffer, 0, len); /* debug only! Once we have confidence, can lose this. */ return((bytes.Length != 0) ? (100 * buffer.Length) / bytes.Length : 0); } else { @out.WriteInt(-1); return(-1); } }
/// <summary>Similar to the unix gzip command, only it does not delete the file after compressing it.</summary> /// <param name="uncompressedFileName">The file to gzip</param> /// <param name="compressedFileName">The file name for the compressed file</param> /// <exception cref="System.IO.IOException"/> public static void GzipFile(File uncompressedFileName, File compressedFileName) { using (GZIPOutputStream @out = new GZIPOutputStream(new FileOutputStream(compressedFileName))) { using (FileInputStream @in = new FileInputStream(uncompressedFileName)) { byte[] buf = new byte[1024]; for (int len; (len = @in.Read(buf)) > 0;) { @out.Write(buf, 0, len); } } } }
/// <exception cref="System.IO.IOException"/> internal virtual void GzipConcatTest(Configuration conf, Type decomClass) { Random r = new Random(); long seed = r.NextLong(); r.SetSeed(seed); Log.Info(decomClass + " seed: " + seed); int Concat = r.Next(4) + 3; int Buflen = 128 * 1024; DataOutputBuffer dflbuf = new DataOutputBuffer(); DataOutputBuffer chkbuf = new DataOutputBuffer(); byte[] b = new byte[Buflen]; for (int i = 0; i < Concat; ++i) { GZIPOutputStream gzout = new GZIPOutputStream(dflbuf); r.NextBytes(b); int len = r.Next(Buflen); int off = r.Next(Buflen - len); chkbuf.Write(b, off, len); gzout.Write(b, off, len); gzout.Close(); } byte[] chk = Arrays.CopyOf(chkbuf.GetData(), chkbuf.GetLength()); CompressionCodec codec = ReflectionUtils.NewInstance <GzipCodec>(conf); Decompressor decom = codec.CreateDecompressor(); NUnit.Framework.Assert.IsNotNull(decom); Assert.Equal(decomClass, decom.GetType()); DataInputBuffer gzbuf = new DataInputBuffer(); gzbuf.Reset(dflbuf.GetData(), dflbuf.GetLength()); InputStream gzin = codec.CreateInputStream(gzbuf, decom); dflbuf.Reset(); IOUtils.CopyBytes(gzin, dflbuf, 4096); byte[] dflchk = Arrays.CopyOf(dflbuf.GetData(), dflbuf.GetLength()); Assert.AssertArrayEquals(chk, dflchk); }
/// <summary> /// gzip Compress /// </summary> /// <param name="originalBytes">Data to be compressed</param> /// <returns>Compressed data</returns> private static sbyte[] Compress(sbyte[] originalBytes) { if (originalBytes == null || originalBytes.Length == 0) { return(null); } try { //using (var result = new MemoryStream()) //{ // var lengthBytes = BitConverter.GetBytes(originalBytes.Length); // result.Write(lengthBytes, 0, 4); // using (var compressionStream = new GZipStream(result, // CompressionMode.Compress)) // { // compressionStream.Write((byte[])(Array)originalBytes, 0, originalBytes.Length); // compressionStream.Flush(); // } // return (sbyte[])(Array)result.ToArray(); //} using (MemoryStream memoryStream = new MemoryStream()) { GZIPOutputStream outputStream = new GZIPOutputStream(memoryStream); outputStream.Write((byte[])(Array)originalBytes); outputStream.Finish(); return((sbyte[])(Array)memoryStream.ToArray()); } } catch (System.Exception e) { System.Console.WriteLine(e.Message); return(null); } }