private static void VerifyRoundTrip(string fileName)
{
int size = 0;
byte[] uncompressed = File.ReadAllBytes(fileName);
var compressor = new SnappyCompressor();
var result = new byte[compressor.MaxCompressedLength(uncompressed.Length)];
size = compressor.Compress(uncompressed, 0, uncompressed.Length, result);
Array.Resize(ref result, size);
var decompressor = new SnappyDecompressor();
var decompressed = decompressor.Decompress(result, 0, size);
byte[] source = File.ReadAllBytes(fileName);
if (source.Length != decompressed.Length)
{
throw new Exception(string.Format("Decompressed length {0} does not match original {1}", decompressed.Length, source.Length));
}
for (int i = 0; i < uncompressed.Length; i++)
{
if (source[i] != decompressed[i])
{
throw new Exception(string.Format("Decompressed data did not match original at index {0}", i));
}
}
}
private void TestFindMatchBug_Internal(SnappyCompressor compressor)
{
var decompressor = new SnappyDecompressor();
var size = 1024;
var data = new byte[size];
for (var i = 0; i < data.Length; ++i)
{
data[i] = (byte)(i & 0xff);
}
data[1021] = 5;
data[1022] = 5;
data[1023] = 5;
var compressed = new byte[compressor.MaxCompressedLength(data.Length)];
var compressedLength = compressor.Compress(data, 0, data.Length, compressed, 0);
var decompressed = decompressor.Decompress(compressed, 0, compressedLength);
for (var i = 0; i < data.Length; ++i)
{
Assert.Equal(data[i], decompressed[i]);
}
}
static void Main(string[] args)
{
var lz4 = new LZ4Compressor();
var snappy = new SnappyCompressor();
for (var i = 0; i < 100; i++)
{
var model = new WorldState2()
{
Players = Enumerable.Range(0, 1000).Select(_ => 1).ToArray()
};
byte[] serialized;
using (var m = new MemoryStream())
{
ProtoBuf.Serializer.NonGeneric.Serialize(m, model);
serialized = m.ToArray();
}
var lz4Length = lz4.Compress(serialized).Length;
var snappLength = snappy.Compress(serialized).Length;
Console.WriteLine($"{serialized.Length}\t{lz4Length}\t{snappLength}\t{lz4Length - snappLength}");
}
Console.ReadLine();
}
static CompressionResult RunCompression(string fileName, int iterations)
{
int size = 0;
long[] ticks = new long[iterations];
byte[] uncompressed = File.ReadAllBytes(fileName);
var target = new SnappyCompressor();
var s = new Stopwatch();
for (int i = 0; i < iterations; i++)
{
var result = new byte[target.MaxCompressedLength(uncompressed.Length)];
s.Start();
size = target.Compress(uncompressed, 0, uncompressed.Length, result);
s.Stop();
ticks[i] = s.ElapsedTicks;
s.Reset();
}
return(new CompressionResult
{
Direction = CompressionDirection.Compress,
FileName = FileMap[Path.GetFileName(fileName)],
CompressedSize = size,
FileBytes = uncompressed.Length,
ElapsedTime = new TimeSpan(ticks.Sum()),
StandardDeviation = StdDev(ticks),
Iterations = iterations
});
}
public void Write(byte[] buffer, Stream destination)
{
var uncompressedLength = buffer.Length;
var compressed = new byte[_snappyCompressor.MaxCompressedLength(uncompressedLength)];
_snappyCompressor.Compress(buffer, 0, uncompressedLength, compressed);
destination.Write(compressed, 0, compressed.Length);
}
public static byte[] Compress(byte[] uncompressed)
{
var target = new SnappyCompressor();
var result = new byte[target.MaxCompressedLength(uncompressed.Length)];
var count = target.Compress(uncompressed, 0, uncompressed.Length, result);
return(result.Take(count).ToArray());
}
public void Write(byte[] buffer, Stream destination)
{
var uncompressedLength = buffer.Length;
int compressedSize = _snappyCompressor.MaxCompressedLength(uncompressedLength);
var compressed = new byte[compressedSize];
var length = _snappyCompressor.Compress(buffer, 0, uncompressedLength, compressed);
destination.Write(compressed, 0, length);
}
public static byte[] Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
var decompressed = Conversion.DecompressBody(req.Body);
if (decompressed != null)
{
Initialize();
var readrequest = ReadRequest.Parser.ParseFrom(decompressed);
log.LogMetric("querycount", readrequest.Queries.Count, new Dictionary <String, object>()
{
{ "type", "count" }
});
ReadResponse response = CreateResponse(readrequest, log);
log.LogMetric("result", response.Results.Count, new Dictionary <String, object>()
{
{ "type", "count" }
});
log.LogMetric("timeseriesread", response.Results.Select(_ => _.Timeseries.Count).Sum(__ => __), new Dictionary <String, object>()
{
{ "type", "count" }
});
MemoryStream ms = new MemoryStream();
Google.Protobuf.CodedOutputStream output = new Google.Protobuf.CodedOutputStream(ms);
response.WriteTo(output);
output.Flush();
var resultUncompressed = ms.ToArray();
if (resultUncompressed.Length > 0)
{
//should be at least the size of the uncompressed one
byte[] resultCompressed = new byte[resultUncompressed.Length * 2];
var compressedSize = compressor.Compress(resultUncompressed, 0, resultUncompressed.Length, resultCompressed);
Array.Resize(ref resultCompressed, compressedSize);
return(resultCompressed);
}
else
{
return(resultUncompressed);
}
}
return(null);
}
public static Byte[] CompressToByteArray(string str)
{
var data = Encoding.Default.GetBytes(str);
var snappy = new SnappyCompressor();
int compressedSize = snappy.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
int result = snappy.Compress(data, 0, data.Length, compressed);
return(compressed.Take(result).ToArray());
}
/// <summary>
/// 使用snappy算法压缩
/// </summary>
/// <param name="uncompressed"></param>
/// <param name="offset"></param>
/// <param name="uncompressedLength"></param>
/// <returns></returns>
public static CompressedBytes CompressInSnappy(byte[] uncompressed, int offset, int uncompressedLength)
{
SnappyCompressor sc = new SnappyCompressor();
//var bytes = Encoding.ASCII.GetBytes("HelloWor134ertegsdfgsfdgsdfgsdfgsfdgsdfgsdfgsdfgsdfgdsfgsdfgdsfgdfgdsfgld");
byte[] outBytes = new byte[sc.MaxCompressedLength(uncompressed.Length)];
int actualLength = sc.Compress(uncompressed, 0, uncompressedLength, outBytes);
return(new CompressedBytes()
{
ContentBytes = outBytes, Length = actualLength
});
}
public void round_trip_returns_original_data(string fileName)
{
byte[] uncompressed = File.ReadAllBytes(fileName);
var target = new SnappyCompressor();
var result = new byte[target.MaxCompressedLength(uncompressed.Length)];
int size = target.Compress(uncompressed, 0, uncompressed.Length, result);
var target2 = new SnappyDecompressor();
var sizes = target2.ReadUncompressedLength(result, 0);
var bytes = new byte[sizes[0]];
target2.Decompress(result, 0 + sizes[1], size - sizes[1], bytes, 0, sizes[1]);
Assert.Equal(uncompressed, bytes);
}
public static async Task<IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)]
HttpRequest req,
ILogger log)
{
var decompressed = HttpConversion.DecompressBody(req.Body);
if (decompressed != null)
{
InitializeKustoClient(log);
var readRequest = ReadRequest.Parser.ParseFrom(decompressed);
log.LogMetric("querycount", readRequest.Queries.Count,
new Dictionary<string, object>() { { "type", "count" } });
var response = CreateResponse(readRequest, log);
log.LogMetric("result", response.Results.Count, new Dictionary<string, object>() { { "type", "count" } });
log.LogMetric("timeseriesread", response.Results.Select(_ => _.Timeseries.Count).Sum(__ => __),
new Dictionary<string, object>() { { "type", "count" } });
var ms = new MemoryStream();
var output = new Google.Protobuf.CodedOutputStream(ms);
response.WriteTo(output);
output.Flush();
var resultUncompressed = ms.ToArray();
if (resultUncompressed.Length > 0)
{
// Should be at least the size of the uncompressed one
byte[] resultCompressed = new byte[resultUncompressed.Length * 2];
var compressedSize = Compressor.Compress(resultUncompressed, 0, resultUncompressed.Length,
resultCompressed);
Array.Resize(ref resultCompressed, compressedSize);
return new FileContentResult(resultCompressed, "application/x-protobuf");
}
else
{
return new FileContentResult(resultUncompressed, "application/x-protobuf");
}
}
return null;
} // - function Read
public void compress_writes_uncompressed_length_first(int dataSize, int storageBytes)
{
var data = GetRandomData(dataSize);
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
target.Compress(data, 0, data.Length, compressed);
var decompressor = new SnappyDecompressor();
var result = decompressor.ReadUncompressedLength(compressed, 0);
Assert.Equal(dataSize, result[0]);
Assert.Equal(storageBytes, result[1]);
}
public void compress_random_data()
{
var data = GetRandomData(4096);
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
int result = target.Compress(data, 0, data.Length, compressed);
Assert.True(result < compressedSize);
var decomperssor = new SnappyDecompressor();
var bytes = decomperssor.Decompress(compressed, 0, result);
Assert.Equal(data, bytes);
}
public void round_trip_returns_original_data(string fileName)
{
byte[] uncompressed = File.ReadAllBytes(fileName);
var target = new SnappyCompressor();
var result = new byte[target.MaxCompressedLength(uncompressed.Length)];
int size = target.Compress(uncompressed, 0, uncompressed.Length, result);
var target2 = new SnappyDecompressor();
var sizes = target2.ReadUncompressedLength(result, 0);
var bytes = new byte[sizes[0]];
target2.Decompress(result, 0 + sizes[1], size - sizes[1], bytes, 0, sizes[1]);
Assert.Equal(uncompressed, bytes);
}
public void compress_random_data()
{
var data = GetRandomData(4096);
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
int result = target.Compress(data, 0, data.Length, compressed);
Assert.True(result < compressedSize);
var decomperssor = new SnappyDecompressor();
var bytes = decomperssor.Decompress(compressed, 0, result);
Assert.Equal(data, bytes);
}
public void compress_constant_data_is_one_literal_and_one_copy()
{
// 1024 a's
var data = Encoding.Default.GetBytes("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
int result = target.Compress(data, 0, data.Length, compressed);
Assert.Equal(50, result);
var decompressor = new SnappyDecompressor();
var bytes = decompressor.Decompress(compressed, 0, result);
Assert.Equal(data, bytes);
}
public void compress_constant_data_is_one_literal_and_one_copy()
{
// 1024 a's
var data = Encoding.UTF8.GetBytes("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
int result = target.Compress(data, 0, data.Length, compressed);
Assert.Equal(52, result);
var decompressor = new SnappyDecompressor();
var bytes = decompressor.Decompress(compressed, 0, result);
Assert.Equal(data, bytes);
}
public void compress_returns_bytes_copied()
{
var data = Encoding.Default.GetBytes("ThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThis");
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
int result = target.Compress(data, 0, data.Length, compressed);
Assert.True(result < compressedSize);
Assert.Equal(15, result);
// TODO: instead of decompressing, we should traverse the buffer looking for tag bytes and interpreting them.
var decompressor = new SnappyDecompressor();
var bytes = decompressor.Decompress(compressed, 0, result);
Console.Write(Encoding.Default.GetString(bytes));
}
public void compress_returns_bytes_copied()
{
var data = Encoding.UTF8.GetBytes("ThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThis");
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
int result = target.Compress(data, 0, data.Length, compressed);
Assert.True(result < compressedSize);
Assert.Equal(12, result);
// TODO: instead of decompressing, we should traverse the buffer looking for tag bytes and interpreting them.
var decompressor = new SnappyDecompressor();
var bytes = decompressor.Decompress(compressed, 0, result);
Console.Write(Encoding.UTF8.GetString(bytes));
}
public void compress_multiple_blocks()
{
/*!
* this ends up being uncompressible because it is random.
*/
var rand = new Random().Next(1 << 4, 1 << 10);
var data = GetRandomData((1 << 20) + rand); // 1MB + a bit random so there is an uneven block at end.
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
int result = target.Compress(data, 0, data.Length, compressed);
Assert.True(result < compressedSize);
var decompressor = new SnappyDecompressor();
var bytes = decompressor.Decompress(compressed, 0, result);
Assert.Equal(data, bytes);
}
public void compress_multiple_blocks()
{
/*!
* this ends up being uncompressible because it is random.
*/
var rand = new Random().Next(1 << 4, 1 << 10);
var data = GetRandomData((1 << 20) + rand); // 1MB + a bit random so there is an uneven block at end.
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
int result = target.Compress(data, 0, data.Length, compressed);
Assert.True(result < compressedSize);
var decompressor = new SnappyDecompressor();
var bytes = decompressor.Decompress(compressed, 0, result);
Assert.Equal(data, bytes);
}
public static int Compress(byte[] uncompressed, int offset, int size, byte[] output, int outputOffset)
{
var target = new SnappyCompressor();
return(target.Compress(uncompressed, offset, size, output, outputOffset));
}
/// <summary>
/// Compress a block of Snappy data.
/// </summary>
/// <param name="input">Data to compress.</param>
/// <param name="output">Buffer to receive the compressed data.</param>
/// <returns>Number of bytes written to <paramref name="output"/>.</returns>
/// <remarks>
/// The output buffer must be large enough to contain the compressed output.
/// </remarks>
public static int Compress(ReadOnlySpan <byte> input, Span <byte> output)
{
var compressor = new SnappyCompressor();
return(compressor.Compress(input, output));
}
public void compress_writes_uncompressed_length_first(int dataSize, int storageBytes)
{
var data = GetRandomData(dataSize);
var target = new SnappyCompressor();
int compressedSize = target.MaxCompressedLength(data.Length);
var compressed = new byte[compressedSize];
target.Compress(data, 0, data.Length, compressed);
var decompressor = new SnappyDecompressor();
var result = decompressor.ReadUncompressedLength(compressed, 0);
Assert.Equal(dataSize, result[0]);
Assert.Equal(storageBytes, result[1]);
}
