public void WriteFully(string testFile)
{
byte[] correctUncompressedBytes = File.ReadAllBytes(testFile);
byte[] compressedBytes = new byte[BrotliEncoder.GetMaxCompressedLength(correctUncompressedBytes.Length)];
byte[] actualUncompressedBytes = new byte[BrotliEncoder.GetMaxCompressedLength(correctUncompressedBytes.Length)];
Span <byte> destination = new Span <byte>(compressedBytes);
Assert.True(BrotliEncoder.TryCompress(correctUncompressedBytes, destination, out int bytesWritten));
Assert.True(BrotliDecoder.TryDecompress(destination, actualUncompressedBytes, out bytesWritten));
Assert.Equal(correctUncompressedBytes.Length, bytesWritten);
Assert.Equal <byte>(correctUncompressedBytes, actualUncompressedBytes.AsSpan(0, correctUncompressedBytes.Length).ToArray());
}
public void WriteWithoutState(string testFile)
{
byte[] correctUncompressedBytes = File.ReadAllBytes(testFile);
byte[] compressedBytes = new byte[BrotliEncoder.GetMaxCompressedLength(correctUncompressedBytes.Length)];
byte[] actualUncompressedBytes = new byte[correctUncompressedBytes.Length];
Compress_WithoutState(correctUncompressedBytes, compressedBytes);
Decompress_WithoutState(compressedBytes, actualUncompressedBytes);
for (int i = 0; i < correctUncompressedBytes.Length; i++)
{
Assert.Equal(correctUncompressedBytes[i], actualUncompressedBytes[i]);
}
}
private static byte[] CompressBrotli(ReadOnlySpan <byte> chunk, out int compressedLength)
{
compressedLength = 0;
int maxLength = BrotliEncoder.GetMaxCompressedLength(chunk.Length);
var buffer = ArrayPool <byte> .Shared.Rent(maxLength);
if (!BrotliEncoder.TryCompress(chunk, buffer, out int written, 11, 22))
{
ArrayPool <byte> .Shared.Return(buffer);
return(null);
}
compressedLength = written;
return(buffer);
}
public static int Brotli(int quality, int window, ref TimeSpan span)
{
int bytesConsumed;
int bytesWritten;
BrotliEncoder encoder = new BrotliEncoder(quality, window);
byte[] data = CompressionExamples.MarbibmUncompressed;
byte[] destination = new byte[(int)(data.Length * 1.2)];
Stopwatch sw = Stopwatch.StartNew();
encoder.Compress(data.AsSpan(), destination.AsSpan(), out bytesConsumed, out bytesWritten, true);
span += sw.Elapsed;
return(bytesWritten);
}
private static Task WriteBrotliCompressedDynamicResponse(ReadOnlySpan <byte> input, HttpResponse response)
{
byte[] output = null;
var arrayPool = ArrayPool <byte> .Shared;
try
{
output = arrayPool.Rent(BrotliEncoder.GetMaxCompressedLength(input.Length));
if (BrotliEncoder.TryCompress(input, output, out var bytesWritten, 4, 22))
{
response.ContentLength = bytesWritten;
response.Headers[ContentEncoding] = Brotli;
return(response.Body.WriteAsync(output, 0, bytesWritten));
}
else
{
return(TryCompressFalse());
}
}
private static async Task BrotliCompressAsync(PipeReader rawReader, PipeWriter writer)
{
using (var brotli = new BrotliEncoder(1, 24))
{
while (true)
{
var result = await rawReader.ReadAsync();
if (result.Buffer.Length > int.MaxValue)
{
throw new Exception();
}
if (!result.Buffer.IsEmpty)
{
using (var sourceMemoryOwner = MemoryPool <byte> .Shared.Rent((int)result.Buffer.Length))
using (var destMemoryOwner = MemoryPool <byte> .Shared.Rent((int)result.Buffer.Length))
{
var sourceMemory = sourceMemoryOwner.Memory.Slice(0, (int)result.Buffer.Length);
result.Buffer.CopyTo(sourceMemory.Span);
brotli.Compress(sourceMemory.Span, destMemoryOwner.Memory.Span, out var bytesConsumed,
out var bytesWritten, result.IsCompleted);
rawReader.AdvanceTo(result.Buffer.GetPosition(bytesConsumed));
var destMemory = destMemoryOwner.Memory.Slice(0, bytesWritten);
await writer.WriteAsync(destMemory);
}
}
if (result.IsCompleted)
{
break;
}
}
rawReader.Complete();
writer.Complete();
}
}
public async Task FasterLog_Test1Write()
{
const int entryLength = 1 << 10;
byte[] staticEntry1 = new byte[entryLength];
byte[] staticEntry1Brotli = new byte[entryLength];
for (int i = 0; i < entryLength; i++)
{
staticEntry1[i] = (byte)i;
}
byte[] staticEntry2 = new byte[entryLength];
byte[] staticEntry2Brotli = new byte[entryLength];
for (int i = 0; i < entryLength; i++)
{
staticEntry2[i] = (byte)(entryLength - i);
}
var path = GetPath();
using (var logCommitManager = new DeviceLogCommitCheckpointManager(
new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme(path), true, false)
) {
using (IDevice device = Devices.CreateLogDevice(path + "hlog.log")) {
//FasterLogScanIterator iter;
var logSettings = new FASTER.core.FasterLogSettings()
{
LogDevice = device,
LogChecksum = LogChecksumType.PerEntry,
LogCommitManager = logCommitManager,
};
using (var log = await FASTER.core.FasterLog.CreateAsync(logSettings)) {
//log.TruncateUntilPageStart(0);
//await log.CommitAsync();
for (int i = 0; i < 1000; i++)
{
if (BrotliEncoder.TryCompress(staticEntry1, staticEntry1Brotli, out var bytesWritten1, 4, 10))
{
await log.EnqueueAsync(new Memory <byte>(staticEntry1Brotli).Slice(0, bytesWritten1));
}
public static bool WriteBlockAndPreamble(TrackingPipeWriter writer, NettraceBlock block)
{
if (block.Type.Name != KnownTypeNames.EventBlock && block.Type.Name != KnownTypeNames.StackBlock)
{
return(false);
}
var blockSequence = block.BlockBody;
var maxLength = BrotliEncoder.GetMaxCompressedLength((int)blockSequence.Length);
var padding = BlockHelpers.GetPadding(writer !, block);
var prefixLength = padding + sizeof(int);
var memory = writer !.GetMemory(maxLength + prefixLength);
// clear padding bits
memory.Slice(0, prefixLength).Span.Clear();
using var encoder = new BrotliEncoder(quality: 9, window: 10);
var slicedMemory = memory.Slice(prefixLength);
var totalWritten = 0;
OperationStatus status;
foreach (var sequence in blockSequence)
{
status = encoder.Compress(sequence.Span, slicedMemory.Span, out var consumed, out var written, false);
Debug.Assert(consumed == sequence.Span.Length);
Debug.Assert(status == OperationStatus.Done);
slicedMemory = slicedMemory.Slice(written);
totalWritten += written;
}
status = encoder.Compress(ReadOnlySpan <byte> .Empty, slicedMemory.Span, out var _, out var written2, true);
Debug.Assert(status == OperationStatus.Done);
totalWritten += written2;
// Write size
BitConverter.TryWriteBytes(memory.Span, totalWritten);
writer.Advance(totalWritten + prefixLength);
return(true);
}
public void TestCompressionBrotli()
{
var watch = new Stopwatch();
watch.Start();
string exeName = Assembly.GetExecutingAssembly().Location;
string fileName = Path.Combine(Path.GetDirectoryName(exeName), @"..\..\..\..\..\..\Data\mnist_png.zip");
long compLength = 0;
using var zip = ZipFile.OpenRead(fileName);
foreach (var entry in zip.Entries)
{
if (string.Compare(Path.GetExtension(entry.FullName), ".png", true) != 0)
{
continue;
}
using var stream = entry.Open();
using var memStream = new MemoryStream();
stream.CopyTo(memStream);
byte[] image = memStream.ToArray();
Utils.ReadStreamToBuffer(stream, image);
int maxCompSize = BrotliEncoder.GetMaxCompressedLength(image.Length);
byte[] compImage = new byte[maxCompSize];
if (!BrotliEncoder.TryCompress(new ReadOnlySpan <byte>(image), new Span <byte>(compImage), out int bytesWritten, 11, 16))
{
throw new Exception("Cannot compress image");
}
compLength += bytesWritten;
}
watch.Stop();
output.WriteLine($"Brotli: {watch.ElapsedMilliseconds:N0} msec.");
output.WriteLine($"Summary length: {compLength:N0}");
}
public static bool TryWrite(ReadOnlySequence <byte> sequence, IBufferWriter <byte> writer)
{
Varint.SetUInt32((uint)FormatType.Version1, writer);
Varint.SetUInt32((uint)CompressionAlgorithm.Brotli, writer);
var reader = new SequenceReader <byte>(sequence);
using var encoder = new BrotliEncoder(0, 10);
var crc32 = default(Crc32_Castagnoli);
for (; ;)
{
var source = reader.UnreadSpan;
var destination = writer.GetSpan();
var status = encoder.Compress(source, destination, out var bytesConsumed, out var bytesWritten, false);
if (status == OperationStatus.InvalidData)
{
_logger.Warn("invalid data");
return(false);
}
reader.Advance(bytesConsumed);
crc32.Compute(destination.Slice(0, bytesWritten));
writer.Advance(bytesWritten);
if (status == OperationStatus.Done)
{
break;
}
}
BinaryPrimitives.WriteUInt32BigEndian(writer.GetSpan(4), crc32.GetResult());
writer.Advance(4);
return(true);
}
public bool TryCompress(byte[] source, int sourceOffset, int sourceLength, byte[] destination, int destinationOffset, int destinationLength, out int written)
{
return(BrotliEncoder.TryCompress(new ReadOnlySpan <byte>(source, sourceOffset, sourceLength),
new Span <byte>(destination, destinationOffset, destinationLength), out written, _quality, _window));
}
private static void Compress_WithoutState(ReadOnlySpan <byte> input, Span <byte> output)
{
BrotliEncoder.TryCompress(input, output, out int bytesWritten);
}
