static CompressionResult RunCompression(string fileName, int iterations) { int size = 0; long[] ticks = new long[iterations]; byte[] uncompressed = File.ReadAllBytes(fileName); var target = new SnappyCompressor(); var s = new Stopwatch(); for (int i = 0; i < iterations; i++) { var result = new byte[target.MaxCompressedLength(uncompressed.Length)]; s.Start(); size = target.Compress(uncompressed, 0, uncompressed.Length, result); s.Stop(); ticks[i] = s.ElapsedTicks; s.Reset(); } return(new CompressionResult { Direction = CompressionDirection.Compress, FileName = FileMap[Path.GetFileName(fileName)], CompressedSize = size, FileBytes = uncompressed.Length, ElapsedTime = new TimeSpan(ticks.Sum()), StandardDeviation = StdDev(ticks), Iterations = iterations }); }
private void TestFindMatchBug_Internal(SnappyCompressor compressor) { var decompressor = new SnappyDecompressor(); var size = 1024; var data = new byte[size]; for (var i = 0; i < data.Length; ++i) { data[i] = (byte)(i & 0xff); } data[1021] = 5; data[1022] = 5; data[1023] = 5; var compressed = new byte[compressor.MaxCompressedLength(data.Length)]; var compressedLength = compressor.Compress(data, 0, data.Length, compressed, 0); var decompressed = decompressor.Decompress(compressed, 0, compressedLength); for (var i = 0; i < data.Length; ++i) { Assert.Equal(data[i], decompressed[i]); } }
private static void VerifyRoundTrip(string fileName) { int size = 0; byte[] uncompressed = File.ReadAllBytes(fileName); var compressor = new SnappyCompressor(); var result = new byte[compressor.MaxCompressedLength(uncompressed.Length)]; size = compressor.Compress(uncompressed, 0, uncompressed.Length, result); Array.Resize(ref result, size); var decompressor = new SnappyDecompressor(); var decompressed = decompressor.Decompress(result, 0, size); byte[] source = File.ReadAllBytes(fileName); if (source.Length != decompressed.Length) { throw new Exception(string.Format("Decompressed length {0} does not match original {1}", decompressed.Length, source.Length)); } for (int i = 0; i < uncompressed.Length; i++) { if (source[i] != decompressed[i]) { throw new Exception(string.Format("Decompressed data did not match original at index {0}", i)); } } }
public void Write(byte[] buffer, Stream destination) { var uncompressedLength = buffer.Length; var compressed = new byte[_snappyCompressor.MaxCompressedLength(uncompressedLength)]; _snappyCompressor.Compress(buffer, 0, uncompressedLength, compressed); destination.Write(compressed, 0, compressed.Length); }
public static byte[] Compress(byte[] uncompressed) { var target = new SnappyCompressor(); var result = new byte[target.MaxCompressedLength(uncompressed.Length)]; var count = target.Compress(uncompressed, 0, uncompressed.Length, result); return(result.Take(count).ToArray()); }
public void Write(byte[] buffer, Stream destination) { var uncompressedLength = buffer.Length; int compressedSize = _snappyCompressor.MaxCompressedLength(uncompressedLength); var compressed = new byte[compressedSize]; var length = _snappyCompressor.Compress(buffer, 0, uncompressedLength, compressed); destination.Write(compressed, 0, length); }
public void emit_literal_copies_bytes_to_destination(int dataSize, byte tagByteValue, int resultSizeExtension) { var target = new SnappyCompressor(); var data = GetRandomData(dataSize); var result = new byte[target.MaxCompressedLength(dataSize)]; var size = target.EmitLiteral(result, 0, data, 0, dataSize, true); Assert.Equal(data, result.Skip(size - dataSize).Take(dataSize)); }
public static Byte[] CompressToByteArray(string str) { var data = Encoding.Default.GetBytes(str); var snappy = new SnappyCompressor(); int compressedSize = snappy.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = snappy.Compress(data, 0, data.Length, compressed); return(compressed.Take(result).ToArray()); }
/// <summary> /// 使用snappy算法压缩 /// </summary> /// <param name="uncompressed"></param> /// <param name="offset"></param> /// <param name="uncompressedLength"></param> /// <returns></returns> public static CompressedBytes CompressInSnappy(byte[] uncompressed, int offset, int uncompressedLength) { SnappyCompressor sc = new SnappyCompressor(); //var bytes = Encoding.ASCII.GetBytes("HelloWor134ertegsdfgsfdgsdfgsdfgsfdgsdfgsdfgsdfgsdfgdsfgsdfgdsfgdfgdsfgld"); byte[] outBytes = new byte[sc.MaxCompressedLength(uncompressed.Length)]; int actualLength = sc.Compress(uncompressed, 0, uncompressedLength, outBytes); return(new CompressedBytes() { ContentBytes = outBytes, Length = actualLength }); }
public void round_trip_returns_original_data(string fileName) { byte[] uncompressed = File.ReadAllBytes(fileName); var target = new SnappyCompressor(); var result = new byte[target.MaxCompressedLength(uncompressed.Length)]; int size = target.Compress(uncompressed, 0, uncompressed.Length, result); var target2 = new SnappyDecompressor(); var sizes = target2.ReadUncompressedLength(result, 0); var bytes = new byte[sizes[0]]; target2.Decompress(result, 0 + sizes[1], size - sizes[1], bytes, 0, sizes[1]); Assert.Equal(uncompressed, bytes); }
public void compress_random_data() { var data = GetRandomData(4096); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.True(result < compressedSize); var decomperssor = new SnappyDecompressor(); var bytes = decomperssor.Decompress(compressed, 0, result); Assert.Equal(data, bytes); }
public void round_trip_returns_original_data(string fileName) { byte[] uncompressed = File.ReadAllBytes(fileName); var target = new SnappyCompressor(); var result = new byte[target.MaxCompressedLength(uncompressed.Length)]; int size = target.Compress(uncompressed, 0, uncompressed.Length, result); var target2 = new SnappyDecompressor(); var sizes = target2.ReadUncompressedLength(result, 0); var bytes = new byte[sizes[0]]; target2.Decompress(result, 0 + sizes[1], size - sizes[1], bytes, 0, sizes[1]); Assert.Equal(uncompressed, bytes); }
public void compress_writes_uncompressed_length_first(int dataSize, int storageBytes) { var data = GetRandomData(dataSize); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; target.Compress(data, 0, data.Length, compressed); var decompressor = new SnappyDecompressor(); var result = decompressor.ReadUncompressedLength(compressed, 0); Assert.Equal(dataSize, result[0]); Assert.Equal(storageBytes, result[1]); }
public void compress_constant_data_is_one_literal_and_one_copy() { // 1024 a's var data = Encoding.Default.GetBytes("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.Equal(50, result); var decompressor = new SnappyDecompressor(); var bytes = decompressor.Decompress(compressed, 0, result); Assert.Equal(data, bytes); }
public void compress_random_data() { var data = GetRandomData(4096); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.True(result < compressedSize); var decomperssor = new SnappyDecompressor(); var bytes = decomperssor.Decompress(compressed, 0, result); Assert.Equal(data, bytes); }
public void compress_constant_data_is_one_literal_and_one_copy() { // 1024 a's var data = Encoding.UTF8.GetBytes("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.Equal(52, result); var decompressor = new SnappyDecompressor(); var bytes = decompressor.Decompress(compressed, 0, result); Assert.Equal(data, bytes); }
public void compress_returns_bytes_copied() { var data = Encoding.Default.GetBytes("ThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThis"); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.True(result < compressedSize); Assert.Equal(15, result); // TODO: instead of decompressing, we should traverse the buffer looking for tag bytes and interpreting them. var decompressor = new SnappyDecompressor(); var bytes = decompressor.Decompress(compressed, 0, result); Console.Write(Encoding.Default.GetString(bytes)); }
public void compress_returns_bytes_copied() { var data = Encoding.UTF8.GetBytes("ThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThis"); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.True(result < compressedSize); Assert.Equal(12, result); // TODO: instead of decompressing, we should traverse the buffer looking for tag bytes and interpreting them. var decompressor = new SnappyDecompressor(); var bytes = decompressor.Decompress(compressed, 0, result); Console.Write(Encoding.UTF8.GetString(bytes)); }
public void compress_multiple_blocks() { /*! * this ends up being uncompressible because it is random. */ var rand = new Random().Next(1 << 4, 1 << 10); var data = GetRandomData((1 << 20) + rand); // 1MB + a bit random so there is an uneven block at end. var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.True(result < compressedSize); var decompressor = new SnappyDecompressor(); var bytes = decompressor.Decompress(compressed, 0, result); Assert.Equal(data, bytes); }
public void compress_multiple_blocks() { /*! * this ends up being uncompressible because it is random. */ var rand = new Random().Next(1 << 4, 1 << 10); var data = GetRandomData((1 << 20) + rand); // 1MB + a bit random so there is an uneven block at end. var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.True(result < compressedSize); var decompressor = new SnappyDecompressor(); var bytes = decompressor.Decompress(compressed, 0, result); Assert.Equal(data, bytes); }
public static int MaxCompressedLength(int sourceLength) { var compressor = new SnappyCompressor(); return(compressor.MaxCompressedLength(sourceLength)); }
public void emit_literal_copies_bytes_to_destination(int dataSize, byte tagByteValue, int resultSizeExtension) { var target = new SnappyCompressor(); var data = GetRandomData(dataSize); var result = new byte[target.MaxCompressedLength(dataSize)]; var size = target.EmitLiteral(result, 0, data, 0, dataSize, true); Assert.Equal(data, result.Skip(size - dataSize).Take(dataSize)); }
public void compress_writes_uncompressed_length_first(int dataSize, int storageBytes) { var data = GetRandomData(dataSize); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; target.Compress(data, 0, data.Length, compressed); var decompressor = new SnappyDecompressor(); var result = decompressor.ReadUncompressedLength(compressed, 0); Assert.Equal(dataSize, result[0]); Assert.Equal(storageBytes, result[1]); }