public void CompressionImprovesWithDictionary()
{
var dict = TrainDict();
var dataStream = DataGenerator.GetSmallStream(DataFill.Sequential);
var normalResultStream = new MemoryStream();
using (var compressionStream = new CompressionStream(normalResultStream))
dataStream.CopyTo(compressionStream);
dataStream.Seek(0, SeekOrigin.Begin);
var dictResultStream = new MemoryStream();
using (var compressionStream = new CompressionStream(dictResultStream))
{
compressionStream.LoadDictionary(dict);
dataStream.CopyTo(compressionStream);
}
Assert.True(normalResultStream.Length > dictResultStream.Length);
dictResultStream.Seek(0, SeekOrigin.Begin);
var resultStream = new MemoryStream();
using (var decompressionStream = new DecompressionStream(dictResultStream))
{
decompressionStream.LoadDictionary(dict);
decompressionStream.CopyTo(resultStream);
}
Assert.True(dataStream.ToArray().SequenceEqual(resultStream.ToArray()));
}
public async Task RoundTrip_StreamingToStreamingAsync(
[CombinatorialValues(false, true)] bool useDict, [CombinatorialValues(false, true)] bool advanced,
[CombinatorialValues(1, 2, 7, 101, 1024, 65535, DataGenerator.LargeBufferSize,
DataGenerator.LargeBufferSize + 1)]
int zstdBufferSize,
[CombinatorialValues(1, 2, 7, 101, 1024, 65535, DataGenerator.LargeBufferSize,
DataGenerator.LargeBufferSize + 1)]
int copyBufferSize)
{
var dict = useDict ? TrainDict() : null;
var testStream = DataGenerator.GetLargeStream(DataFill.Sequential);
const int offset = 1;
var buffer = new byte[copyBufferSize + offset + 1];
var tempStream = new MemoryStream();
await using (var compressionStream =
new CompressionStream(tempStream, Compressor.DefaultCompressionLevel, zstdBufferSize))
{
compressionStream.LoadDictionary(dict);
if (advanced)
{
compressionStream.SetParameter(ZSTD_cParameter.ZSTD_c_windowLog, 11);
compressionStream.SetParameter(ZSTD_cParameter.ZSTD_c_checksumFlag, 1);
}
int bytesRead;
while ((bytesRead = await testStream.ReadAsync(buffer, offset, copyBufferSize)) > 0)
{
await compressionStream.WriteAsync(buffer, offset, bytesRead);
}
}
tempStream.Seek(0, SeekOrigin.Begin);
var resultStream = new MemoryStream();
await using (var decompressionStream = new DecompressionStream(tempStream, zstdBufferSize))
{
decompressionStream.LoadDictionary(dict);
if (advanced)
{
decompressionStream.SetParameter(ZSTD_dParameter.ZSTD_d_windowLogMax, 11);
}
int bytesRead;
while ((bytesRead = await decompressionStream.ReadAsync(buffer, offset, copyBufferSize)) > 0)
{
await resultStream.WriteAsync(buffer, offset, bytesRead);
}
}
Assert.True(testStream.ToArray().SequenceEqual(resultStream.ToArray()));
}
public void RoundTrip_StreamingToStreaming_Stress([CombinatorialValues(true, false)] bool useDict,
[CombinatorialValues(true, false)] bool async)
{
long i = 0;
var dict = useDict ? TrainDict() : null;
Enumerable.Range(0, 10000)
.AsParallel()
.WithDegreeOfParallelism(Environment.ProcessorCount * 4)
.ForAll(n =>
{
var testStream = DataGenerator.GetSmallStream(DataFill.Sequential);
var cBuffer = new byte[1 + (int)(n % (testStream.Length * 11))];
var dBuffer = new byte[1 + (int)(n % (testStream.Length * 13))];
var tempStream = new MemoryStream();
using (var compressionStream = new CompressionStream(tempStream, Compressor.DefaultCompressionLevel,
1 + (int)(n % (testStream.Length * 17))))
{
compressionStream.LoadDictionary(dict);
int bytesRead;
int offset = n % cBuffer.Length;
while ((bytesRead = testStream.Read(cBuffer, offset, cBuffer.Length - offset)) > 0)
{
if (async)
{
compressionStream.WriteAsync(cBuffer, offset, bytesRead).GetAwaiter().GetResult();
}
else
{
compressionStream.Write(cBuffer, offset, bytesRead);
}
if (Interlocked.Increment(ref i) % 100 == 0)
{
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, true, true);
}
}
}
tempStream.Seek(0, SeekOrigin.Begin);
var resultStream = new MemoryStream();
using (var decompressionStream =
new DecompressionStream(tempStream, 1 + (int)(n % (testStream.Length * 19))))
{
decompressionStream.LoadDictionary(dict);
int bytesRead;
int offset = n % dBuffer.Length;
while ((bytesRead = async
? decompressionStream.ReadAsync(dBuffer, offset, dBuffer.Length - offset).GetAwaiter()
.GetResult()
: decompressionStream.Read(dBuffer, offset, dBuffer.Length - offset)) > 0)
{
resultStream.Write(dBuffer, offset, bytesRead);
if (Interlocked.Increment(ref i) % 100 == 0)
{
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, true, true);
}
}
}
Assert.True(testStream.ToArray().SequenceEqual(resultStream.ToArray()));
});
}