public void RoundTrip_Chunks()
{
int chunkSize = 100;
int totalSize = 20000;
BrotliEncoder encoder;
BrotliDecoder decoder;
for (int i = 0; i < totalSize; i += chunkSize)
{
byte[] uncompressed = new byte[chunkSize];
new Random().NextBytes(uncompressed);
byte[] compressed = new byte[BrotliEncoder.GetMaxCompressedLength(chunkSize)];
byte[] decompressed = new byte[chunkSize];
var uncompressedSpan = new ReadOnlySpan <byte>(uncompressed);
var compressedSpan = new Span <byte>(compressed);
var decompressedSpan = new Span <byte>(decompressed);
int totalWrittenThisIteration = 0;
var compress = encoder.Compress(uncompressedSpan, compressedSpan, out int bytesConsumed, out int bytesWritten, isFinalBlock: false);
totalWrittenThisIteration += bytesWritten;
compress = encoder.Flush(compressedSpan.Slice(bytesWritten), out bytesWritten);
totalWrittenThisIteration += bytesWritten;
var res = decoder.Decompress(compressedSpan.Slice(0, totalWrittenThisIteration), decompressedSpan, out int decompressbytesConsumed, out int decompressbytesWritten);
Assert.Equal(totalWrittenThisIteration, decompressbytesConsumed);
Assert.Equal(bytesConsumed, decompressbytesWritten);
Assert.Equal <byte>(uncompressed, decompressedSpan.ToArray());
}
}
public void GetMaxCompressedSize_Basic()
{
Assert.Throws <ArgumentOutOfRangeException>("length", () => BrotliEncoder.GetMaxCompressedLength(-1));
Assert.Throws <ArgumentOutOfRangeException>("length", () => BrotliEncoder.GetMaxCompressedLength(2147483133));
Assert.InRange(BrotliEncoder.GetMaxCompressedLength(2147483132), 0, int.MaxValue);
Assert.Equal(1, BrotliEncoder.GetMaxCompressedLength(0));
}
public byte[] Compress(Span <byte> bytes)
{
var target = ArrayPool <byte> .Shared.Rent(BrotliEncoder.GetMaxCompressedLength(bytes.Length));
Compress(bytes, target);
return(target);
}
public void GetMaxCompressedSize()
{
string uncompressedFile = UncompressedTestFile();
string compressedFile = CompressedTestFile(uncompressedFile);
int maxCompressedSize = BrotliEncoder.GetMaxCompressedLength((int)new FileInfo(uncompressedFile).Length);
int actualCompressedSize = (int)new FileInfo(compressedFile).Length;
Assert.True(maxCompressedSize >= actualCompressedSize, $"MaxCompressedSize: {maxCompressedSize}, ActualCompressedSize: {actualCompressedSize}");
}
public void WriteWithoutState(string testFile)
{
byte[] correctUncompressedBytes = File.ReadAllBytes(testFile);
byte[] compressedBytes = new byte[BrotliEncoder.GetMaxCompressedLength(correctUncompressedBytes.Length)];
byte[] actualUncompressedBytes = new byte[correctUncompressedBytes.Length];
Compress_WithoutState(correctUncompressedBytes, compressedBytes);
Decompress_WithoutState(compressedBytes, actualUncompressedBytes);
Assert.Equal <byte>(correctUncompressedBytes, actualUncompressedBytes);
}
public Span <byte> Compress(Span <byte> bytes, Span <byte> target)
{
var needMinLength = BrotliEncoder.GetMaxCompressedLength(bytes.Length);
Guard.Argument(target.Length).GreaterThan(needMinLength);
var success = BrotliEncoder.TryCompress(bytes, target, out var bytesWritten, 2, 10);
Guard.Argument(success).Require(true);
return(target.Slice(0, bytesWritten));
}
public void WriteFully(string testFile)
{
byte[] correctUncompressedBytes = File.ReadAllBytes(testFile);
byte[] compressedBytes = new byte[BrotliEncoder.GetMaxCompressedLength(correctUncompressedBytes.Length)];
byte[] actualUncompressedBytes = new byte[BrotliEncoder.GetMaxCompressedLength(correctUncompressedBytes.Length)];
Span <byte> destination = new Span <byte>(compressedBytes);
Assert.True(BrotliEncoder.TryCompress(correctUncompressedBytes, destination, out int bytesWritten));
Assert.True(BrotliDecoder.TryDecompress(destination, actualUncompressedBytes, out bytesWritten));
Assert.Equal(correctUncompressedBytes.Length, bytesWritten);
Assert.Equal <byte>(correctUncompressedBytes, actualUncompressedBytes.AsSpan(0, correctUncompressedBytes.Length).ToArray());
}
/// <summary>
/// Compresses data. Uses <see cref="BrotliEncoder"/>.
/// </summary>
/// <param name="data">Data. See also: <see cref="MemoryMarshal.AsBytes"/>, <see cref="CollectionsMarshal.AsSpan"/>.</param>
/// <param name="level">Compression level, 0 (no compression) to 11 (maximal compression). Default 6. Bigger levels don't make much smaller but can make much slower.</param>
/// <exception cref="ArgumentOutOfRangeException">Invalid <i>level</i>.</exception>
/// <exception cref="OutOfMemoryException"></exception>
public static unsafe byte[] BrotliCompress(ReadOnlySpan <byte> data, int level = 6)
{
int n = BrotliEncoder.GetMaxCompressedLength(data.Length);
var b = MemoryUtil.Alloc(n);
try {
if (!BrotliEncoder.TryCompress(data, new(b, n), out n, level, 22))
{
throw new AuException();
}
return(new Span <byte>(b, n).ToArray());
}
finally { MemoryUtil.Free(b); }
}
public void WriteWithState(string testFile)
{
byte[] correctUncompressedBytes = File.ReadAllBytes(testFile);
byte[] compressedBytes = new byte[BrotliEncoder.GetMaxCompressedLength(correctUncompressedBytes.Length)];
byte[] actualUncompressedBytes = new byte[correctUncompressedBytes.Length];
Compress_WithState(correctUncompressedBytes, compressedBytes);
Decompress_WithState(compressedBytes, actualUncompressedBytes);
for (int i = 0; i < correctUncompressedBytes.Length; i++)
{
Assert.Equal(correctUncompressedBytes[i], actualUncompressedBytes[i]);
}
}
private static byte[] CompressBrotli(ReadOnlySpan <byte> chunk, out int compressedLength)
{
compressedLength = 0;
int maxLength = BrotliEncoder.GetMaxCompressedLength(chunk.Length);
var buffer = ArrayPool <byte> .Shared.Rent(maxLength);
if (!BrotliEncoder.TryCompress(chunk, buffer, out int written, 11, 22))
{
ArrayPool <byte> .Shared.Return(buffer);
return(null);
}
compressedLength = written;
return(buffer);
}
private static Task WriteBrotliCompressedDynamicResponse(ReadOnlySpan <byte> input, HttpResponse response)
{
byte[] output = null;
var arrayPool = ArrayPool <byte> .Shared;
try
{
output = arrayPool.Rent(BrotliEncoder.GetMaxCompressedLength(input.Length));
if (BrotliEncoder.TryCompress(input, output, out var bytesWritten, 4, 22))
{
response.ContentLength = bytesWritten;
response.Headers[ContentEncoding] = Brotli;
return(response.Body.WriteAsync(output, 0, bytesWritten));
}
else
{
return(TryCompressFalse());
}
}
public static bool WriteBlockAndPreamble(TrackingPipeWriter writer, NettraceBlock block)
{
if (block.Type.Name != KnownTypeNames.EventBlock && block.Type.Name != KnownTypeNames.StackBlock)
{
return(false);
}
var blockSequence = block.BlockBody;
var maxLength = BrotliEncoder.GetMaxCompressedLength((int)blockSequence.Length);
var padding = BlockHelpers.GetPadding(writer !, block);
var prefixLength = padding + sizeof(int);
var memory = writer !.GetMemory(maxLength + prefixLength);
// clear padding bits
memory.Slice(0, prefixLength).Span.Clear();
using var encoder = new BrotliEncoder(quality: 9, window: 10);
var slicedMemory = memory.Slice(prefixLength);
var totalWritten = 0;
OperationStatus status;
foreach (var sequence in blockSequence)
{
status = encoder.Compress(sequence.Span, slicedMemory.Span, out var consumed, out var written, false);
Debug.Assert(consumed == sequence.Span.Length);
Debug.Assert(status == OperationStatus.Done);
slicedMemory = slicedMemory.Slice(written);
totalWritten += written;
}
status = encoder.Compress(ReadOnlySpan <byte> .Empty, slicedMemory.Span, out var _, out var written2, true);
Debug.Assert(status == OperationStatus.Done);
totalWritten += written2;
// Write size
BitConverter.TryWriteBytes(memory.Span, totalWritten);
writer.Advance(totalWritten + prefixLength);
return(true);
}
public void TestCompressionBrotli()
{
var watch = new Stopwatch();
watch.Start();
string exeName = Assembly.GetExecutingAssembly().Location;
string fileName = Path.Combine(Path.GetDirectoryName(exeName), @"..\..\..\..\..\..\Data\mnist_png.zip");
long compLength = 0;
using var zip = ZipFile.OpenRead(fileName);
foreach (var entry in zip.Entries)
{
if (string.Compare(Path.GetExtension(entry.FullName), ".png", true) != 0)
{
continue;
}
using var stream = entry.Open();
using var memStream = new MemoryStream();
stream.CopyTo(memStream);
byte[] image = memStream.ToArray();
Utils.ReadStreamToBuffer(stream, image);
int maxCompSize = BrotliEncoder.GetMaxCompressedLength(image.Length);
byte[] compImage = new byte[maxCompSize];
if (!BrotliEncoder.TryCompress(new ReadOnlySpan <byte>(image), new Span <byte>(compImage), out int bytesWritten, 11, 16))
{
throw new Exception("Cannot compress image");
}
compLength += bytesWritten;
}
watch.Stop();
output.WriteLine($"Brotli: {watch.ElapsedMilliseconds:N0} msec.");
output.WriteLine($"Summary length: {compLength:N0}");
}