public void FrameRoundTrip(
[ValueSource(typeof(Utilities), nameof(Utilities.CompressionLevels))]
int compressionLevel,
[Values(false, true)] bool useDictionary
)
{
#pragma warning disable 8632
var dict = useDictionary
? Utilities.CreateDictionaryFromSamples(32 * 1024, 30, compressionLevel)
// ReSharper disable once RedundantCast // downlevel impl requires
: (ZStdDictionaryBuilder?)null;
#pragma warning restore 8632
var sample = Utilities.GenerateSampleBuffer(1000);
Console.WriteLine($"Compression Level: {compressionLevel}");
// compression
using var cDict = dict?.CreateCompressorDictionary(compressionLevel);
using var cCtx = new ZStdCompressor();
cCtx.UseDictionary(cDict);
var compressBufferSize = CCtx.GetUpperBound((UIntPtr)sample.Length).ToUInt32();
var compressBuffer = new byte[compressBufferSize];
cCtx.Set(CompressionParameter.CompressionLevel, compressionLevel);
var compressedSize = cCtx.Compress(compressBuffer, sample);
compressedSize.ToUInt64().Should().NotBe(0);
var compressedFrame = new ArraySegment <byte>(compressBuffer, 0, (int)compressedSize);
Console.WriteLine($"Compressed to: {compressedSize} ({(double) compressedSize / (double) sample.Length:P})");
// decompression
using var dDict = dict?.CreateDecompressorDictionary();
using var dCtx = new ZStdDecompressor();
dCtx.UseDictionary(dDict);
var decompressBufferSize = DCtx.GetUpperBound(compressedFrame);
decompressBufferSize.Should().NotBe(0);
decompressBufferSize.Should().BeGreaterOrEqualTo((ulong)sample.Length);
var decompressBuffer = new byte[decompressBufferSize];
var decompressedSize = dCtx.Decompress(decompressBuffer, compressedFrame);
(decompressedSize).Should().Be((UIntPtr)sample.Length);
decompressBuffer.Take((int)decompressedSize).Should().Equal(sample);
}
public void PiecemealStreamRoundTrip(
[ValueSource(typeof(Utilities), nameof(Utilities.CompressionLevels))]
int compressionLevel,
[Values(0, 1, 2)] int flushMode,
[Values(false, true)] bool useDictionary
)
{
#pragma warning disable 8632
var dict = useDictionary
? Utilities.CreateDictionaryFromSamples(32 * 1024, 30, compressionLevel)
// ReSharper disable once RedundantCast // downlevel impl requires
: (ZStdDictionaryBuilder?)null;
#pragma warning restore 8632
var sample = Utilities.GenerateSampleBuffer(1000);
Console.WriteLine($"Compression Level: {compressionLevel}");
// lol "min" compression level
var bufSize = (int)(CCtx.GetUpperBound((UIntPtr)sample.Length) * 1.05);
var buffer = new byte[bufSize]; // compression is much worse when fed piecemeal
fixed(byte *pBuffer = &buffer[0])
{
using var compressed = new MemoryRegionStream(pBuffer, bufSize);
using var toDecompress = new MemoryRegionStream(pBuffer, bufSize);
// compression
var run = 0;
var wroteThisRun = 0L;
IEnumerable <long> Feed()
{
using (var cDict = dict?.CreateCompressorDictionary())
using (var compressStream = new ZStdCompressStream(compressed)) {
compressStream.Compressor.Set(CompressionParameter.CompressionLevel, compressionLevel);
compressStream.Compressor.UseDictionary(cDict);
for (var i = 0; i < sample.Length;)
{
// NOTE: failures happen if fed with below 12 bytes at a time
var x = Utilities.Random.Next(1, 12) * (1 + run);
var rem = sample.Length - (i + x);
if (rem < 0)
{
x += rem;
}
compressStream.Write(sample, i, x);
if (flushMode != 0)
{
compressStream.Flush(flushMode == 2);
}
wroteThisRun = compressed.Length - toDecompress.Length;
if (wroteThisRun > 0)
{
toDecompress.SetLength(compressed.Length);
}
i += x;
yield return(wroteThisRun);
}
}
wroteThisRun = compressed.Length - toDecompress.Length;
if (wroteThisRun > 0)
{
toDecompress.SetLength(compressed.Length);
}
Console.WriteLine($"Compressed to: {compressed.Length} ({(double) compressed.Length / (double) sample.Length:P})");
yield return(wroteThisRun);
}
// decompression
compressed.Position = 0;
using var dDict = dict?.CreateDecompressorDictionary();
using var decompressStream = new ZStdDecompressStream(toDecompress);
decompressStream.Decompressor.UseDictionary(dDict);
var decompressed = new byte[sample.Length];
using var feed = Feed().GetEnumerator();
var totalRead = 0;
do
{
int read;
var readThisRun = 0;
do
{
read = decompressStream.Read(decompressed, totalRead, decompressed.Length - totalRead);
decompressed.Skip(totalRead).Take(read).Should().Equal(sample.Skip(totalRead).Take(read));
totalRead += read;
readThisRun += read;
} while (read > 0);
if (wroteThisRun != readThisRun)
{
Debugger.Break();
}
Console.WriteLine($"Run: {run++} Wrote: {wroteThisRun}, Read: {readThisRun}");
} while (feed.MoveNext());
decompressStream.ReadByte().Should().Be(-1);
decompressed.Should().Equal(sample);
}
}
public void StreamRoundTrip(
[ValueSource(typeof(Utilities), nameof(Utilities.CompressionLevels))]
int compressionLevel,
[Values(false, true)] bool useDictionary
)
{
#pragma warning disable 8632
var dict = useDictionary
? Utilities.CreateDictionaryFromSamples(32 * 1024, 30, compressionLevel)
// ReSharper disable once RedundantCast // downlevel impl requires
: (ZStdDictionaryBuilder?)null;
#pragma warning restore 8632
var sample = Utilities.GenerateSampleBuffer(1000);
Console.WriteLine($"Compression Level: {compressionLevel}");
// compression
using var cDict = dict?.CreateCompressorDictionary(compressionLevel);
using var cCtx = new ZStdCompressor();
cCtx.UseDictionary(cDict);
var compressBufferSize = CCtx.GetUpperBound((UIntPtr)sample.Length).ToUInt32();
var compressBuffer = new byte[compressBufferSize];
UIntPtr compressedSize;
cCtx.Set(CompressionParameter.CompressionLevel, compressionLevel);
{
var outBuf = new ArraySegment <byte>(compressBuffer, 0, compressBuffer.Length);
var inBuf = new ArraySegment <byte>(sample, 0, sample.Length);
var amountRemaining = cCtx.StreamCompress(ref outBuf, ref inBuf, EndDirective.End);
amountRemaining.ToUInt64().Should().Be(0);
outBuf.Offset.Should().NotBe(0);
inBuf.Offset.Should().NotBe(0);
inBuf.Offset.Should().Be(sample.Length);
compressedSize = (UIntPtr)outBuf.Offset;
}
var compressedFrame = new ArraySegment <byte>(compressBuffer, 0, (int)compressedSize);
Console.WriteLine($"Compressed to: {compressedSize} ({(double) compressedSize / (double) sample.Length:P})");
// decompression
using var dDict = dict?.CreateDecompressorDictionary();
using var dCtx = new ZStdDecompressor();
dCtx.UseDictionary(dDict);
var decompressBufferSize = DCtx.GetUpperBound(compressedFrame);
var decompressBuffer = new byte[decompressBufferSize];
{
var outBuf = new ArraySegment <byte>(decompressBuffer, 0, decompressBuffer.Length);
var inBuf = compressedFrame;
var amountRemaining = dCtx.StreamDecompress(ref outBuf, ref inBuf);
amountRemaining.ToUInt64().Should().Be(0);
outBuf.Count.Should().Be(0);
inBuf.Count.Should().Be(0);
outBuf.Offset.Should().NotBe(0);
inBuf.Offset.Should().NotBe(0);
outBuf.Offset.Should().Be(sample.Length);
decompressBuffer.Take(outBuf.Offset).Should().Equal(sample);
}
}