public void Compress_WithEmptySource()
{
string testFile = UncompressedTestFile();
byte[] sourceBytes = new byte[0];
byte[] destinationBytes = new byte[100000];
ReadOnlySpan <byte> source = new ReadOnlySpan <byte>(sourceBytes);
Span <byte> destination = new Span <byte>(destinationBytes);
Assert.True(BrotliEncoder.TryCompress(source, destination, out int bytesWritten));
// The only byte written should be the Brotli end of stream byte which varies based on the window/quality
Assert.Equal(1, bytesWritten);
BrotliEncoder encoder = default;
var result = encoder.Compress(source, destination, out int bytesConsumed, out bytesWritten, false);
Assert.Equal(0, bytesWritten);
Assert.Equal(0, bytesConsumed);
Assert.Equal(OperationStatus.Done, result);
result = encoder.Compress(source, destination, out bytesConsumed, out bytesWritten, isFinalBlock: true);
Assert.Equal(1, bytesWritten);
Assert.Equal(0, bytesConsumed);
Assert.Equal(OperationStatus.Done, result);
}
public void Compress_WithEmptyDestination()
{
string testFile = UncompressedTestFile();
byte[] correctUncompressedBytes = File.ReadAllBytes(testFile);
byte[] compressedBytes = File.ReadAllBytes(CompressedTestFile(testFile));
byte[] empty = new byte[0];
ReadOnlySpan <byte> source = new ReadOnlySpan <byte>(correctUncompressedBytes);
Span <byte> destination = new Span <byte>(empty);
Assert.False(BrotliEncoder.TryCompress(source, destination, out int bytesWritten), "TryCompress completed successfully but should have failed due to too short of a destination array");
Assert.Equal(0, bytesWritten);
BrotliEncoder encoder = default;
var result = encoder.Compress(source, destination, out int bytesConsumed, out bytesWritten, false);
Assert.Equal(0, bytesWritten);
Assert.Equal(0, bytesConsumed);
Assert.Equal(OperationStatus.DestinationTooSmall, result);
result = encoder.Compress(source, destination, out bytesConsumed, out bytesWritten, isFinalBlock: true);
Assert.Equal(0, bytesWritten);
Assert.Equal(0, bytesConsumed);
Assert.Equal(OperationStatus.DestinationTooSmall, result);
}
public static void Compress_WithState(ReadOnlySpan <byte> input, Span <byte> output)
{
BrotliEncoder encoder = default;
while (!input.IsEmpty && !output.IsEmpty)
{
encoder.Compress(input, output, out int bytesConsumed, out int written, isFinalBlock: false);
input = input.Slice(bytesConsumed);
output = output.Slice(written);
}
encoder.Compress(ReadOnlySpan <byte> .Empty, output, out int bytesConsumed2, out int bytesWritten, isFinalBlock: true);
}
public void RoundTrip_Chunks()
{
int chunkSize = 100;
int totalSize = 20000;
BrotliEncoder encoder = default;
BrotliDecoder decoder = default;
for (int i = 0; i < totalSize; i += chunkSize)
{
byte[] uncompressed = new byte[chunkSize];
new Random().NextBytes(uncompressed);
byte[] compressed = new byte[BrotliEncoder.GetMaxCompressedLength(chunkSize)];
byte[] deompressed = new byte[chunkSize];
var uncompressedSpan = new ReadOnlySpan <byte>(uncompressed);
var compressedSpan = new Span <byte>(compressed);
var decompressedSpan = new Span <byte>(deompressed);
int totalWrittenThisIteration = 0;
var compress = encoder.Compress(uncompressedSpan, compressedSpan, out int bytesConsumed, out int bytesWritten, isFinalBlock: false);
totalWrittenThisIteration += bytesWritten;
compress = encoder.Flush(compressedSpan.Slice(bytesWritten), out bytesWritten);
totalWrittenThisIteration += bytesWritten;
var res = decoder.Decompress(compressedSpan.Slice(0, totalWrittenThisIteration), decompressedSpan, out int decompressbytesConsumed, out int decompressbytesWritten);
Assert.Equal(totalWrittenThisIteration, decompressbytesConsumed);
Assert.Equal(bytesConsumed, decompressbytesWritten);
for (int j = 0; j < bytesConsumed; j++)
{
Assert.Equal(uncompressed[j], decompressedSpan[j]);
}
}
}
public static bool TryWrite(uint version, ReadOnlySequence <byte> sequence, IBufferWriter <byte> writer)
{
Varint.SetUInt32(version, writer);
Varint.SetUInt32((uint)ConvertCompressionAlgorithm.Brotli, writer);
using var encoder = new BrotliEncoder(0, 10);
var reader = new SequenceReader <byte>(sequence);
for (; ;)
{
var status = encoder.Compress(reader.UnreadSpan, writer.GetSpan(), out var bytesConsumed, out var bytesWritten, false);
if (status == OperationStatus.InvalidData)
{
_logger.Warn("invalid data");
return(false);
}
reader.Advance(bytesConsumed);
writer.Advance(bytesWritten);
if (status == OperationStatus.Done)
{
return(true);
}
}
}
public static bool WriteBlockAndPreamble(TrackingPipeWriter writer, NettraceBlock block)
{
if (block.Type.Name != KnownTypeNames.EventBlock && block.Type.Name != KnownTypeNames.StackBlock)
{
return(false);
}
var blockSequence = block.BlockBody;
var maxLength = BrotliEncoder.GetMaxCompressedLength((int)blockSequence.Length);
var padding = BlockHelpers.GetPadding(writer !, block);
var prefixLength = padding + sizeof(int);
var memory = writer !.GetMemory(maxLength + prefixLength);
// clear padding bits
memory.Slice(0, prefixLength).Span.Clear();
using var encoder = new BrotliEncoder(quality: 9, window: 10);
var slicedMemory = memory.Slice(prefixLength);
var totalWritten = 0;
OperationStatus status;
foreach (var sequence in blockSequence)
{
status = encoder.Compress(sequence.Span, slicedMemory.Span, out var consumed, out var written, false);
Debug.Assert(consumed == sequence.Span.Length);
Debug.Assert(status == OperationStatus.Done);
slicedMemory = slicedMemory.Slice(written);
totalWritten += written;
}
status = encoder.Compress(ReadOnlySpan <byte> .Empty, slicedMemory.Span, out var _, out var written2, true);
Debug.Assert(status == OperationStatus.Done);
totalWritten += written2;
// Write size
BitConverter.TryWriteBytes(memory.Span, totalWritten);
writer.Advance(totalWritten + prefixLength);
return(true);
}
public static int Brotli(int quality, int window, ref TimeSpan span)
{
int bytesConsumed;
int bytesWritten;
BrotliEncoder encoder = new BrotliEncoder(quality, window);
byte[] data = CompressionExamples.MarbibmUncompressed;
byte[] destination = new byte[(int)(data.Length * 1.2)];
Stopwatch sw = Stopwatch.StartNew();
encoder.Compress(data.AsSpan(), destination.AsSpan(), out bytesConsumed, out bytesWritten, true);
span += sw.Elapsed;
return(bytesWritten);
}
private static async Task BrotliCompressAsync(PipeReader rawReader, PipeWriter writer)
{
using (var brotli = new BrotliEncoder(1, 24))
{
while (true)
{
var result = await rawReader.ReadAsync();
if (result.Buffer.Length > int.MaxValue)
{
throw new Exception();
}
if (!result.Buffer.IsEmpty)
{
using (var sourceMemoryOwner = MemoryPool <byte> .Shared.Rent((int)result.Buffer.Length))
using (var destMemoryOwner = MemoryPool <byte> .Shared.Rent((int)result.Buffer.Length))
{
var sourceMemory = sourceMemoryOwner.Memory.Slice(0, (int)result.Buffer.Length);
result.Buffer.CopyTo(sourceMemory.Span);
brotli.Compress(sourceMemory.Span, destMemoryOwner.Memory.Span, out var bytesConsumed,
out var bytesWritten, result.IsCompleted);
rawReader.AdvanceTo(result.Buffer.GetPosition(bytesConsumed));
var destMemory = destMemoryOwner.Memory.Slice(0, bytesWritten);
await writer.WriteAsync(destMemory);
}
}
if (result.IsCompleted)
{
break;
}
}
rawReader.Complete();
writer.Complete();
}
}
public static bool TryWrite(ReadOnlySequence <byte> sequence, IBufferWriter <byte> writer)
{
Varint.SetUInt32((uint)FormatType.Version1, writer);
Varint.SetUInt32((uint)CompressionAlgorithm.Brotli, writer);
var reader = new SequenceReader <byte>(sequence);
using var encoder = new BrotliEncoder(0, 10);
var crc32 = default(Crc32_Castagnoli);
for (; ;)
{
var source = reader.UnreadSpan;
var destination = writer.GetSpan();
var status = encoder.Compress(source, destination, out var bytesConsumed, out var bytesWritten, false);
if (status == OperationStatus.InvalidData)
{
_logger.Warn("invalid data");
return(false);
}
reader.Advance(bytesConsumed);
crc32.Compute(destination.Slice(0, bytesWritten));
writer.Advance(bytesWritten);
if (status == OperationStatus.Done)
{
break;
}
}
BinaryPrimitives.WriteUInt32BigEndian(writer.GetSpan(4), crc32.GetResult());
writer.Advance(4);
return(true);
}