protected async void HandleConnection([NotNull] TcpClient client)
{
await Task.Yield();
using (client)
{
Console.WriteLine("Connection accepted");
using var networkStream = client.GetStream();
try
{
var tmp = new byte[1024];
var rawData = new List <byte>();
int bytesRead = 0;
do
{
bytesRead = networkStream.Read(tmp);
rawData.AddRange(tmp);
} while (networkStream.DataAvailable && bytesRead != 0);
var data = Brotli.Decompress(rawData.ToArray());
var message = System.Text.Json.JsonSerializer.Deserialize <Message>(Encoding.ASCII.GetString(data));
Route(message);
networkStream.Write(Brotli.Compress(data));
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
}
}
public void Compress()
{
// Run tests on data
foreach (var file in CompressTestFiles)
{
var filePath = Path.Combine(TestdataDir, file);
Assert.IsTrue(File.Exists(filePath), "Unable to find the test file: " + file);
foreach (var quality in CompressQualities)
{
// Compress using the current quality
var original = File.ReadAllBytes(filePath);
var compressed = Brotli.CompressBuffer(original, 0, original.Length, quality);
// Decompress and verify with original
try
{
var decompressed = Brotli.DecompressBuffer(compressed, 0, compressed.Length);
CompareBuffers(original, decompressed, file);
}
catch (Exception e)
{
throw new Exception("Decompress failed with compressed buffer quality " + quality + " for " + file, e);
}
}
}
}
private void ValidateCompressedData(Span <byte> data, byte[] expected)
{
byte[] decompressed = new byte[expected.Length];
TransformationStatus result = Brotli.Decompress(data, decompressed, out int consumed, out int written);
Assert.Equal <TransformationStatus>(TransformationStatus.Done, result);
Assert.Equal <byte>(expected, decompressed);
}
public static async Task <byte[]> DecompressAsync(ReadOnlyMemory <byte> data, CompressionMethod method)
{
return(method switch
{
CompressionMethod.LZ4 => await LZ4.DecompressAsync(data).ConfigureAwait(false),
CompressionMethod.Deflate => await Deflate.DecompressAsync(data).ConfigureAwait(false),
CompressionMethod.Brotli => await Brotli.DecompressAsync(data).ConfigureAwait(false),
CompressionMethod.Gzip => await Gzip.DecompressAsync(data).ConfigureAwait(false),
_ => await Gzip.DecompressAsync(data).ConfigureAwait(false)
});
private void ValidateCompressedData(byte[] data, byte[] expected)
{
byte[] decompressed = new byte[expected.Length];
Brotli.State state = new Brotli.State();
TransformationStatus result = Brotli.Decompress(data, decompressed, out int consumed, out int written, ref state);
Assert.Equal <TransformationStatus>(TransformationStatus.Done, result);
Assert.Equal <long>(expected.Length, written);
Assert.Equal <long>(consumed, 0);
Assert.Equal <byte>(expected, decompressed);
}
public override string Process(string[] args)
{
var source = args[ExtraArgumentCount + 0];
var output = args.Length >= ExtraArgumentCount + 2 ? args[ExtraArgumentCount + 1] : File.Exists(source) ? Path.GetDirectoryName(source) : source;
if (output == null || !Directory.Exists(output))
{
throw new ArgumentException("Output folder does not exist.");
}
Setup(args);
var items = SelectFiles(Brotli.ListPath(source)).ToArray();
int errors = 0;
using (var progress = new Progress(items.Length)){
items.Parallelize().ForAll(item => {
var(group, file) = item;
string outputFile = Path.Combine(output, file.FullName + AppendFileName);
try{
progress.Start($"Processing {file}");
Directory.CreateDirectory(Path.GetDirectoryName(outputFile));
if (Path.GetFullPath(outputFile) == Path.GetFullPath(file.Path))
{
string outputFileTmp = outputFile + ".tmp";
using (var stream = new FileStream(outputFileTmp, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read)){
MapFile(group, file, stream);
}
File.Move(outputFileTmp, outputFile, true);
}
else
{
using var stream = new FileStream(outputFile, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read);
MapFile(group, file, stream);
}
progress.Finish($"Completed {file}");
}catch (Exception e) {
Interlocked.Increment(ref errors);
progress.Print(ConsoleColor.Red, $"Error processing {file}: {e.Message}");
Debug.WriteLine(e.ToString());
}
});
}
return($"{WorkDesc} {items.Length} file(s) with {errors} error(s).");
}
public void RoundtripCompressDecompress(int totalSize)
{
byte[] data = new byte[totalSize];
new Random(42).NextBytes(data);
Span <byte> compressed = new byte[Brotli.GetMaximumCompressedSize(totalSize)];
TransformationStatus result = Brotli.Compress(data, compressed, out int consumed, out int written);
Assert.Equal(TransformationStatus.Done, result);
Assert.Equal(totalSize, consumed);
compressed = compressed.Slice(0, written);
ValidateCompressedData(compressed, data);
}
public void Compress(CompressionType type)
{
byte[] bytes = CreateBytesToCompress(type);
foreach (var iteration in Benchmark.Iterations)
{
byte[] compressed = new byte[bytes.Length];
using (iteration.StartMeasurement())
{
Brotli.Compress(bytes, compressed, out int consumed, out int writen);
}
}
}
protected override async Task <HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
var response = await base.SendAsync(request, cancellationToken);
if (!response.Content.Headers.TryGetValues("Content-Encoding", out var ce) || ce.First() != "br")
{
return(response);
}
var buffer = await response.Content.ReadAsByteArrayAsync();
response.Content = new ByteArrayContent(Brotli.DecompressBuffer(buffer, 0, buffer.Length));
return(response);
}
public async Task <Transaction> SendAsync(string data, string endpoint)
{
Transaction transaction = new Transaction();
Message request = null;
Message response = null;
try
{
using var clientSocket = new TcpClient();
request = new Message(data, endpoint)
{
TransactionId = transaction.Id,
};
// Send request
var serializedObj = JsonSerializer.Serialize(request);
var socketData = Brotli.Compress(serializedObj);
await clientSocket.ConnectAsync(IP, Port);
var stream = clientSocket.GetStream();
await stream.WriteAsync(socketData);
// Read response
var tmp = new byte[1024];
var rawResponseData = new List <byte>();
int bytesRead = 0;
do
{
bytesRead = await stream.ReadAsync(tmp);
rawResponseData.AddRange(tmp);
}while (stream.DataAvailable && bytesRead != 0);
var responseData = Brotli.Decompress(rawResponseData.ToArray());
var responseString = Encoding.ASCII.GetString(responseData);
response = JsonSerializer.Deserialize <Message>(responseString);
}
catch (Exception ex)
{
Console.Error.WriteLine(ex.ToString());
return(null);
}
transaction.Request = request;
if (response != null)
{
transaction.Response = response;
}
_transactions[transaction.Id] = transaction;
return(transaction);
}
public async Task <ByteArrayContent> ReadRequestContentAsync(WebRequest request)
{
byte[] payload = await GetPayload(GetStream(), request.ContentLength).ConfigureAwait(false);
if (payload == null)
{
return(null);
}
if (request.Headers[HttpRequestHeader.ContentEncoding] == "br")
{
request.Headers[HttpRequestHeader.ContentEncoding] = ""; // No longer encoded.
payload = Brotli.DecompressBuffer(payload, 0, payload.Length);
}
return(new ByteArrayContent(payload));
}
public string Process(string[] args)
{
var path = args[0];
var qualities = args[1] == "all" ? QualityRange : ParseQualityRange(args[1]);
var wbits = args.Length >= 3 ? new WindowSize(int.Parse(args[2])).Bits : AutoWindowSize;
if (!qualities.Values.All(QualityRange.Contains))
{
throw new ArgumentException($"Compression quality must be in the range {QualityRange}.");
}
var groups = Brotli.ListPath(path).ToArray();
int totalFiles = groups.Length;
if (totalFiles > FileLimit)
{
throw new InvalidOperationException($"Too many files to process ({totalFiles} > {FileLimit}), cancelling command for safety.");
}
var items = qualities.Values.Cartesian(groups).ToArray();
using (var progress = new Progress(items.Length)){
try{
items.Parallelize().ForAll(item => {
var(quality, group) = item;
var file = group.Uncompressed;
progress.Start($"Processing {file} (quality {quality})");
Compress(wbits, quality, file.Path);
progress.Finish($"Completed {file} (quality {quality})");
});
}catch (Exception e) when(HasWin32Exception(e, out var we))
{
if (CustomExePath == null)
{
throw new InvalidOperationException("Brotli executable must be named 'brotli' and placed into the working directory or environment path.", we);
}
else
{
throw new InvalidOperationException("Brotli executable failed to start.", we);
}
}
}
return($"Compressed {totalFiles} file(s).");
}
public sealed override string Process(string[] args)
{
Setup(args);
var items = SelectFiles(Brotli.ListPath(args[ExtraArgumentCount + 0]));
var output = args.Length >= ExtraArgumentCount + 2 ? args[ExtraArgumentCount + 1] : GetTemporaryOutputFile();
using var table = new Table.CSV(output, Columns);
var result = new FileWorker <T> {
Work = GenerateRows,
Error = OnError
}.Start(table, items);
Finalize(table);
return($"{WorkDesc} {result.TotalProcessed} file(s) with {result.TotalErrors} error(s).");
}
public void Decompress(CompressionType type)
{
string testFilePath = CreateCompressedFile(type);
int bufferSize = 1000000;
byte[] data = File.ReadAllBytes(testFilePath);
var bytes = new byte[bufferSize];
foreach (var iteration in Benchmark.Iterations)
{
using (iteration.StartMeasurement())
for (int i = 0; i < Benchmark.InnerIterationCount; i++)
{
Brotli.Decompress(data, bytes, out int consumed, out int written);
}
}
File.Delete(testFilePath);
}
public void Decompress()
{
// Run tests on data
foreach (var kvp in DecompressTestFiles)
{
var compressedFilePath = Path.Combine(TestdataDir, kvp.Key);
var originalFilePath = Path.Combine(TestdataDir, kvp.Value);
Assert.IsTrue(File.Exists(compressedFilePath), "Unable to find the compressed test file: " + kvp.Key);
Assert.IsTrue(File.Exists(originalFilePath), "Unable to find the test file: " + kvp.Value);
// Decompress the compressed data
var compressed = File.ReadAllBytes(compressedFilePath);
var decompressed = Brotli.DecompressBuffer(compressed, 0, compressed.Length);
// Compare the decompressed version with the original
var original = File.ReadAllBytes(originalFilePath);
CompareBuffers(original, decompressed, kvp.Key + " --> " + kvp.Value);
}
}
public void RoundtripCompressDecompress(int totalSize)
{
byte[] data = new byte[totalSize];
new Random(42).NextBytes(data);
byte[] compressed = new byte[Brotli.GetMaximumCompressedSize(totalSize)];
Assert.NotEqual(compressed.Length, 0);
Brotli.State state = new Brotli.State();
TransformationStatus result = Brotli.Compress(data, compressed, out int consumed, out int written, ref state);
while (consumed != 0 || result != TransformationStatus.Done)
{
result = Brotli.Compress(data, compressed, out consumed, out written, ref state);
}
byte[] flush = new byte[0];
result = Brotli.FlushEncoder(flush, compressed, out consumed, out written, ref state);
Assert.Equal(TransformationStatus.Done, result);
Assert.Equal(consumed, 0);
byte[] resultCompressed = new byte[written];
Array.Copy(compressed, resultCompressed, written);
ValidateCompressedData(resultCompressed, data);
}
public WebFile(EndianBinaryReader reader)
{
var magic = reader.ReadBytes(2);
reader.Position = 0;
if (gzipMagic.SequenceEqual(magic))
{
var stream = new MemoryStream();
using (var gs = new GZipStream(reader.BaseStream, CompressionMode.Decompress))
{
gs.CopyTo(stream);
}
stream.Position = 0;
using (reader = new EndianBinaryReader(stream, EndianType.LittleEndian))
{
ReadWebData(reader);
}
}
else
{
reader.Position = 0x20;
magic = reader.ReadBytes(6);
reader.Position = 0;
if (brotliMagic.SequenceEqual(magic))
{
var buff = reader.ReadBytes((int)reader.BaseStream.Length);
var uncompressedData = Brotli.DecompressBuffer(buff, 0, buff.Length);
var stream = new MemoryStream(uncompressedData);
using (reader = new EndianBinaryReader(stream, EndianType.LittleEndian))
{
ReadWebData(reader);
}
}
else
{
reader.endian = EndianType.LittleEndian;
ReadWebData(reader);
}
}
}
// Compression
/// <summary>
/// Automatically decompresses the current message's content if encoded with the Brotli compression algorithm
/// </summary>
/// <returns>This <see cref="HttpResponseReader"/></returns>
public HttpResponseReader <T> UseBrotliDecompression()
{
this.funcs.Enqueue(async(message, _) => {
if (message.Content.Headers.ContentEncoding.FirstOrDefault() == "br")
{
byte[] raw = await message.Content.ReadAsByteArrayAsync();
byte[] decompressed = Brotli.DecompressBuffer(raw, 0, raw.Length);
var newContent = new ByteArrayContent(decompressed);
newContent.Headers.Clear();
TransferHeaders(message.Content, newContent);
message.Content.Dispose();
message.Content = newContent;
}
return(true);
});
return(this);
}
public void CompressViaStream()
{
// Run tests on data
foreach (var file in CompressTestFiles)
{
var filePath = Path.Combine(TestdataDir, file);
Assert.IsTrue(File.Exists(filePath), "Unable to find the test file: " + file);
foreach (var quality in CompressQualities)
{
// Compress using the current quality
using (var fs = File.OpenRead(filePath))
using (var ms = new MemoryStream())
{
using (var bs = new BrotliStream(ms, CompressionMode.Compress))
{
bs.SetQuality(quality);
fs.CopyTo(bs);
bs.Dispose();
var compressed = ms.ToArray();
// Decompress and verify with original
try
{
var decompressed = Brotli.DecompressBuffer(compressed, 0, compressed.Length);
CompareBuffers(File.ReadAllBytes(filePath), decompressed, file);
}
catch (Exception e)
{
throw new Exception("Decompress failed with compressed buffer quality " + quality + " for " + file, e);
}
}
}
}
}
}
public async Task WriteRequestContentAsync(WebRequest request, HttpContent content)
{
byte[] payload = null;
if (content is StreamContent streamContent)
{
// TODO:
throw new NotSupportedException();
}
else
{
payload = await content.ReadAsByteArrayAsync().ConfigureAwait(false);
}
if (request.Headers[HttpRequestHeader.ContentEncoding] == "br")
{
payload = Brotli.CompressBuffer(payload, 0, payload.Length);
}
request.ContentLength = payload.Length;
using (Stream output = await request.GetRequestStreamAsync().ConfigureAwait(false))
{
await output.WriteAsync(payload, 0, payload.Length).ConfigureAwait(false);
}
}
static void DecompressBrotli(byte[] compressed, Stream output)
{
var decompressed = Brotli.DecompressBuffer(compressed, 0, compressed.Length);
}
internal byte[] UnBrotliJson(byte[] compressedData)
{
return(Brotli.DecompressBuffer(compressedData, 0, compressedData.Length /**, customDictionary **/));
}
internal byte[] BrotliJson(byte[] uncompressedData)
{
return(Brotli.CompressBuffer(uncompressedData, 0, uncompressedData.Length, 4));
}
public void DecompressSetQuality()
{
Brotli.State state = new Brotli.State();
state.SetQuality(1);
Assert.Throws <System.Exception>(delegate { Brotli.Decompress(Span <byte> .Empty, Span <byte> .Empty, out int consumed, out int written, ref state); });
public void TestMethodCompressEx(CompressionLevel quality, int lgWinSize)
{
Assert.Throws <ArgumentOutOfRangeException>(() => Brotli.Compress(new byte[1], new byte[1], out int consumed, out int written, quality, lgWinSize));
}
internal override byte[] Compress(byte[] uncompressedData)
{
return(Brotli.CompressBuffer(uncompressedData, 0, uncompressedData.Length, 4));
}
internal override byte[] Decompress(byte[] compressedData)
{
return(Brotli.DecompressBuffer(compressedData, 0, compressedData.Length));
}
public string Process(string[] args)
{
int totalFiles = 0;
int failedFiles = 0;
var items = Brotli.ListPath(args[0]).SelectCompressedFiles().ToArray();
using (var progress = new Progress(items.Length))
using (var table = new Table.CSV(args[1], new [] {
"File", "Quality", "Original Bytes", "Reserialize Bytes", "Rebuild Bytes", "Avg Reserialize Time (ms)", "Avg Rebuild Time (ms)"
})){
foreach (var(group, file) in items)
{
progress.Start($"Processing {file}");
int? originalBytes = file.SizeBytes;
int? reserializeBytes = null;
int? rebuildBytes = null;
long?reserializeTotalTime = 0L;
long?rebuildTotalTime = 0L;
for (int run = 1; run <= SkippedRuns + CountedRuns; run++)
{
Stopwatch swReserialize = Stopwatch.StartNew();
try{
reserializeBytes = group.CountBytesAndValidate(file.Reader);
}catch (Exception e) {
Debug.WriteLine(e.ToString());
++failedFiles;
reserializeTotalTime = null;
rebuildTotalTime = null;
break;
}finally{
swReserialize.Stop();
}
Stopwatch swRebuild = Stopwatch.StartNew();
try{
rebuildBytes = group.CountBytesAndValidate(file.Transforming(new TransformRebuild()));
}catch (Exception e) {
Debug.WriteLine(e.ToString());
++failedFiles;
reserializeTotalTime = null;
rebuildTotalTime = null;
break;
}finally{
swRebuild.Stop();
}
if (run > SkippedRuns)
{
reserializeTotalTime += swReserialize.ElapsedMilliseconds;
rebuildTotalTime += swRebuild.ElapsedMilliseconds;
}
}
++totalFiles;
table.AddRow(file.Name, file.Identifier, originalBytes, reserializeBytes, rebuildBytes, reserializeTotalTime / CountedRuns, rebuildTotalTime / CountedRuns); // subtraction propagates null
progress.Finish($"Completed {file}");
}
}
return($"Processed {totalFiles} file(s) with {failedFiles} error(s).");
}
