static CompressionResult RunCompression(string fileName, int iterations)
{
int size = 0;
long[] ticks = new long[iterations];
byte[] uncompressed = File.ReadAllBytes(fileName);
var target = new SnappyCompressor();
var s = new Stopwatch();
for (int i = 0; i < iterations; i++)
{
var result = new byte[target.MaxCompressedLength(uncompressed.Length)];
s.Start();
size = target.Compress(uncompressed, 0, uncompressed.Length, result);
s.Stop();
ticks[i] = s.ElapsedTicks;
s.Reset();
}
return(new CompressionResult
{
Direction = CompressionDirection.Compress,
FileName = FileMap[Path.GetFileName(fileName)],
CompressedSize = size,
FileBytes = uncompressed.Length,
ElapsedTime = new TimeSpan(ticks.Sum()),
StandardDeviation = StdDev(ticks),
Iterations = iterations
});
}
private void TestFindMatchBug_Internal(SnappyCompressor compressor)
{
var decompressor = new SnappyDecompressor();
var size = 1024;
var data = new byte[size];
for (var i = 0; i < data.Length; ++i)
{
data[i] = (byte)(i & 0xff);
}
data[1021] = 5;
data[1022] = 5;
data[1023] = 5;
var compressed = new byte[compressor.MaxCompressedLength(data.Length)];
var compressedLength = compressor.Compress(data, 0, data.Length, compressed, 0);
var decompressed = decompressor.Decompress(compressed, 0, compressedLength);
for (var i = 0; i < data.Length; ++i)
{
Assert.Equal(data[i], decompressed[i]);
}
}
private static void VerifyRoundTrip(string fileName)
{
int size = 0;
byte[] uncompressed = File.ReadAllBytes(fileName);
var compressor = new SnappyCompressor();
var result = new byte[compressor.MaxCompressedLength(uncompressed.Length)];
size = compressor.Compress(uncompressed, 0, uncompressed.Length, result);
Array.Resize(ref result, size);
var decompressor = new SnappyDecompressor();
var decompressed = decompressor.Decompress(result, 0, size);
byte[] source = File.ReadAllBytes(fileName);
if (source.Length != decompressed.Length)
{
throw new Exception(string.Format("Decompressed length {0} does not match original {1}", decompressed.Length, source.Length));
}
for (int i = 0; i < uncompressed.Length; i++)
{
if (source[i] != decompressed[i])
{
throw new Exception(string.Format("Decompressed data did not match original at index {0}", i));
}
}
}
public void Write(byte[] buffer, Stream destination)
{
var uncompressedLength = buffer.Length;
var compressed = new byte[_snappyCompressor.MaxCompressedLength(uncompressedLength)];
_snappyCompressor.Compress(buffer, 0, uncompressedLength, compressed);
destination.Write(compressed, 0, compressed.Length);
}
public static byte[] Compress(byte[] uncompressed)
{
var target = new SnappyCompressor();
var result = new byte[target.MaxCompressedLength(uncompressed.Length)];
var count = target.Compress(uncompressed, 0, uncompressed.Length, result);
return(result.Take(count).ToArray());
}
public void Write(byte[] buffer, Stream destination)
{
var uncompressedLength = buffer.Length;
int compressedSize = _snappyCompressor.MaxCompressedLength(uncompressedLength);
var compressed = new byte[compressedSize];
var length = _snappyCompressor.Compress(buffer, 0, uncompressedLength, compressed);
destination.Write(compressed, 0, length);
}
public void emit_literal_copies_bytes_to_destination(int dataSize, byte tagByteValue, int resultSizeExtension)
{
var target = new SnappyCompressor();
var data = GetRandomData(dataSize);
var result = new byte[target.MaxCompressedLength(dataSize)];
var size = target.EmitLiteral(result, 0, data, 0, dataSize, true);
Assert.Equal(data, result.Skip(size - dataSize).Take(dataSize));
}
public static Byte[] CompressToByteArray(string str)
{
var data = Encoding.Default.GetBytes(str);
var snappy = new SnappyCompressor();
int compressedSize = snappy.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
int result = snappy.Compress(data, 0, data.Length, compressed);
return(compressed.Take(result).ToArray());
}
/// <summary>
/// 使用snappy算法压缩
/// </summary>
/// <param name="uncompressed"></param>
/// <param name="offset"></param>
/// <param name="uncompressedLength"></param>
/// <returns></returns>
public static CompressedBytes CompressInSnappy(byte[] uncompressed, int offset, int uncompressedLength)
{
SnappyCompressor sc = new SnappyCompressor();
//var bytes = Encoding.ASCII.GetBytes("HelloWor134ertegsdfgsfdgsdfgsdfgsfdgsdfgsdfgsdfgsdfgdsfgsdfgdsfgdfgdsfgld");
byte[] outBytes = new byte[sc.MaxCompressedLength(uncompressed.Length)];
int actualLength = sc.Compress(uncompressed, 0, uncompressedLength, outBytes);
return(new CompressedBytes()
{
ContentBytes = outBytes, Length = actualLength
});
}
public void round_trip_returns_original_data(string fileName)
{
byte[] uncompressed = File.ReadAllBytes(fileName);
var target = new SnappyCompressor();
var result = new byte[target.MaxCompressedLength(uncompressed.Length)];
int size = target.Compress(uncompressed, 0, uncompressed.Length, result);
var target2 = new SnappyDecompressor();
var sizes = target2.ReadUncompressedLength(result, 0);
var bytes = new byte[sizes[0]];
target2.Decompress(result, 0 + sizes[1], size - sizes[1], bytes, 0, sizes[1]);
Assert.Equal(uncompressed, bytes);
}
public void compress_random_data()
{
var data = GetRandomData(4096);
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
int result = target.Compress(data, 0, data.Length, compressed);
Assert.True(result < compressedSize);
var decomperssor = new SnappyDecompressor();
var bytes = decomperssor.Decompress(compressed, 0, result);
Assert.Equal(data, bytes);
}
public void round_trip_returns_original_data(string fileName)
{
byte[] uncompressed = File.ReadAllBytes(fileName);
var target = new SnappyCompressor();
var result = new byte[target.MaxCompressedLength(uncompressed.Length)];
int size = target.Compress(uncompressed, 0, uncompressed.Length, result);
var target2 = new SnappyDecompressor();
var sizes = target2.ReadUncompressedLength(result, 0);
var bytes = new byte[sizes[0]];
target2.Decompress(result, 0 + sizes[1], size - sizes[1], bytes, 0, sizes[1]);
Assert.Equal(uncompressed, bytes);
}
public void compress_writes_uncompressed_length_first(int dataSize, int storageBytes)
{
var data = GetRandomData(dataSize);
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
target.Compress(data, 0, data.Length, compressed);
var decompressor = new SnappyDecompressor();
var result = decompressor.ReadUncompressedLength(compressed, 0);
Assert.Equal(dataSize, result[0]);
Assert.Equal(storageBytes, result[1]);
}
public void compress_constant_data_is_one_literal_and_one_copy()
{
// 1024 a's
var data = Encoding.Default.GetBytes("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
int result = target.Compress(data, 0, data.Length, compressed);
Assert.Equal(50, result);
var decompressor = new SnappyDecompressor();
var bytes = decompressor.Decompress(compressed, 0, result);
Assert.Equal(data, bytes);
}
public void compress_random_data()
{
var data = GetRandomData(4096);
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
int result = target.Compress(data, 0, data.Length, compressed);
Assert.True(result < compressedSize);
var decomperssor = new SnappyDecompressor();
var bytes = decomperssor.Decompress(compressed, 0, result);
Assert.Equal(data, bytes);
}
public void compress_constant_data_is_one_literal_and_one_copy()
{
// 1024 a's
var data = Encoding.UTF8.GetBytes("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
int result = target.Compress(data, 0, data.Length, compressed);
Assert.Equal(52, result);
var decompressor = new SnappyDecompressor();
var bytes = decompressor.Decompress(compressed, 0, result);
Assert.Equal(data, bytes);
}
public void compress_returns_bytes_copied()
{
var data = Encoding.Default.GetBytes("ThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThis");
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
int result = target.Compress(data, 0, data.Length, compressed);
Assert.True(result < compressedSize);
Assert.Equal(15, result);
// TODO: instead of decompressing, we should traverse the buffer looking for tag bytes and interpreting them.
var decompressor = new SnappyDecompressor();
var bytes = decompressor.Decompress(compressed, 0, result);
Console.Write(Encoding.Default.GetString(bytes));
}
public void compress_returns_bytes_copied()
{
var data = Encoding.UTF8.GetBytes("ThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThis");
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
int result = target.Compress(data, 0, data.Length, compressed);
Assert.True(result < compressedSize);
Assert.Equal(12, result);
// TODO: instead of decompressing, we should traverse the buffer looking for tag bytes and interpreting them.
var decompressor = new SnappyDecompressor();
var bytes = decompressor.Decompress(compressed, 0, result);
Console.Write(Encoding.UTF8.GetString(bytes));
}
public void compress_multiple_blocks()
{
/*!
* this ends up being uncompressible because it is random.
*/
var rand = new Random().Next(1 << 4, 1 << 10);
var data = GetRandomData((1 << 20) + rand); // 1MB + a bit random so there is an uneven block at end.
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
int result = target.Compress(data, 0, data.Length, compressed);
Assert.True(result < compressedSize);
var decompressor = new SnappyDecompressor();
var bytes = decompressor.Decompress(compressed, 0, result);
Assert.Equal(data, bytes);
}
public void compress_multiple_blocks()
{
/*!
* this ends up being uncompressible because it is random.
*/
var rand = new Random().Next(1 << 4, 1 << 10);
var data = GetRandomData((1 << 20) + rand); // 1MB + a bit random so there is an uneven block at end.
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
int result = target.Compress(data, 0, data.Length, compressed);
Assert.True(result < compressedSize);
var decompressor = new SnappyDecompressor();
var bytes = decompressor.Decompress(compressed, 0, result);
Assert.Equal(data, bytes);
}
public static int MaxCompressedLength(int sourceLength)
{
var compressor = new SnappyCompressor();
return(compressor.MaxCompressedLength(sourceLength));
}
public void emit_literal_copies_bytes_to_destination(int dataSize, byte tagByteValue, int resultSizeExtension)
{
var target = new SnappyCompressor();
var data = GetRandomData(dataSize);
var result = new byte[target.MaxCompressedLength(dataSize)];
var size = target.EmitLiteral(result, 0, data, 0, dataSize, true);
Assert.Equal(data, result.Skip(size - dataSize).Take(dataSize));
}
public void compress_writes_uncompressed_length_first(int dataSize, int storageBytes)
{
var data = GetRandomData(dataSize);
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
target.Compress(data, 0, data.Length, compressed);
var decompressor = new SnappyDecompressor();
var result = decompressor.ReadUncompressedLength(compressed, 0);
Assert.Equal(dataSize, result[0]);
Assert.Equal(storageBytes, result[1]);
}
