public void CompressionImprovesWithDictionary()
{
var dict = TrainDict();
var compressionOptions = new CompressionOptions(dict);
var dataStream = DataGenerator.GetSmallStream(DataFill.Sequential);
var normalResultStream = new MemoryStream();
using (var compressionStream = new CompressionStream(normalResultStream))
dataStream.CopyTo(compressionStream);
dataStream.Seek(0, SeekOrigin.Begin);
var dictResultStream = new MemoryStream();
using (var compressionStream = new CompressionStream(dictResultStream, compressionOptions))
dataStream.CopyTo(compressionStream);
Assert.Greater(normalResultStream.Length, dictResultStream.Length);
dictResultStream.Seek(0, SeekOrigin.Begin);
var resultStream = new MemoryStream();
using (var decompressionStream = new DecompressionStream(dictResultStream, new DecompressionOptions(dict)))
decompressionStream.CopyTo(resultStream);
Assert.AreEqual(dataStream.ToArray(), resultStream.ToArray());
}
public void RoundTrip_StreamingToStreaming_Stress([Values(true, false)] bool useDict, [Values(true, false)] bool async)
{
long i = 0;
var dict = useDict ? TrainDict() : null;
var compressionOptions = new CompressionOptions(dict);
var decompressionOptions = new DecompressionOptions(dict);
Enumerable.Range(0, 10000)
.AsParallel()
.WithDegreeOfParallelism(Environment.ProcessorCount * 4)
.ForAll(n =>
{
var testStream = DataGenerator.GetSmallStream(DataFill.Sequential);
var cBuffer = new byte[1 + (int)(n % (testStream.Length * 11))];
var dBuffer = new byte[1 + (int)(n % (testStream.Length * 13))];
var tempStream = new MemoryStream();
using (var compressionStream = new CompressionStream(tempStream, compressionOptions, 1 + (int)(n % (testStream.Length * 17))))
{
int bytesRead;
int offset = n % cBuffer.Length;
while ((bytesRead = testStream.Read(cBuffer, offset, cBuffer.Length - offset)) > 0)
{
if (async)
{
compressionStream.WriteAsync(cBuffer, offset, bytesRead).GetAwaiter().GetResult();
}
else
{
compressionStream.Write(cBuffer, offset, bytesRead);
}
if (Interlocked.Increment(ref i) % 100 == 0)
{
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, true, true);
}
}
}
tempStream.Seek(0, SeekOrigin.Begin);
var resultStream = new MemoryStream();
using (var decompressionStream = new DecompressionStream(tempStream, decompressionOptions, 1 + (int)(n % (testStream.Length * 19))))
{
int bytesRead;
int offset = n % dBuffer.Length;
while ((bytesRead = async ? decompressionStream.ReadAsync(dBuffer, offset, dBuffer.Length - offset).GetAwaiter().GetResult() : decompressionStream.Read(dBuffer, offset, dBuffer.Length - offset)) > 0)
{
resultStream.Write(dBuffer, offset, bytesRead);
if (Interlocked.Increment(ref i) % 100 == 0)
{
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, true, true);
}
}
}
Assert.AreEqual(testStream.ToArray(), resultStream.ToArray());
});
GC.KeepAlive(compressionOptions);
GC.KeepAlive(decompressionOptions);
}
public void CompressionImprovesWithDictionary()
{
var dict = TrainDict();
var dataStream = DataGenerator.GetSmallStream(DataFill.Sequential);
var normalResultStream = new MemoryStream();
using (var compressionStream = new CompressionStream(normalResultStream))
dataStream.CopyTo(compressionStream);
dataStream.Seek(0, SeekOrigin.Begin);
var dictResultStream = new MemoryStream();
using (var compressionStream = new CompressionStream(dictResultStream))
{
compressionStream.LoadDictionary(dict);
dataStream.CopyTo(compressionStream);
}
Assert.True(normalResultStream.Length > dictResultStream.Length);
dictResultStream.Seek(0, SeekOrigin.Begin);
var resultStream = new MemoryStream();
using (var decompressionStream = new DecompressionStream(dictResultStream))
{
decompressionStream.LoadDictionary(dict);
decompressionStream.CopyTo(resultStream);
}
Assert.True(dataStream.ToArray().SequenceEqual(resultStream.ToArray()));
}
public void StreamingDecompressionSimpleRead(int readCount)
{
var data = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
var tempStream = new MemoryStream();
using (var compressionStream = new CompressionStream(tempStream))
compressionStream.Write(data, 0, data.Length);
tempStream.Seek(0, SeekOrigin.Begin);
var buffer = new byte[data.Length];
using (var decompressionStream = new DecompressionStream(tempStream))
{
int bytesRead;
int totalBytesRead = 0;
while ((bytesRead = decompressionStream.Read(buffer, totalBytesRead, Math.Min(readCount, buffer.Length - totalBytesRead))) > 0)
{
Assert.LessOrEqual(bytesRead, readCount);
totalBytesRead += bytesRead;
}
Assert.AreEqual(data.Length, totalBytesRead);
}
Assert.AreEqual(data, buffer);
}
public UnbufferedLongReader(string FilePath)
{
int ioBufSize = Math.Min(BufSize * sizeof(long), 131072);
file = new FileStream(FilePath, FileMode.Open, FileAccess.Read, FileShare.Read, ioBufSize, FileOptions.SequentialScan);
zip = new DecompressionStream(file, ioBufSize);
}
public static async Task Main()
{
var engine = new RazorLightEngineBuilder()
.UseMemoryCachingProvider()
.UseEmbeddedResourcesProject(typeof(Program).Assembly, rootNamespace: "RazorLight.Sandbox.Views")
.Build();
//string result = await engine.CompileRenderAsync<object>("Home", null, null);
//Console.WriteLine(result);
await using var tempStream = new MemoryStream();
await using var compressionStream = new CompressionStream(tempStream);
await GetStream(64).CopyToAsync(compressionStream);
await using var decompressionStream = new DecompressionStream(tempStream);
await UnwrapStream(decompressionStream);
//System.GC.SuppressFinalize(decompressionStream);
var model = new { UnwrappedValue = new Lazy <string>(() =>
{
var x = UnwrapStream(decompressionStream).Result;
return(Encoding.Default.GetString(x));
}) };
string dynamicTemplateResult =
await engine.CompileRenderStringAsync("templateKey", @"@using RazorLight
@Model.UnwrappedValue.Value", model, null);
Console.WriteLine(dynamicTemplateResult);
Console.WriteLine("Finished");
}
static async Task <byte[]> UnwrapStream(DecompressionStream decompressionStream)
{
var result = new byte[64];
await decompressionStream.ReadAsync(result, 0, 64);
return(result);
}
public async Task DecompressStreamAsync(int zstdBufferSize, int copyBufferSize)
{
#if !NET48
await
#endif
using var decompressionStream = new DecompressionStream(new MemoryStream(CompressedStreamData), zstdBufferSize);
await decompressionStream.CopyToAsync(Stream.Null, copyBufferSize);
}
public async Task RoundTrip_StreamingToStreamingAsync(
[CombinatorialValues(false, true)] bool useDict, [CombinatorialValues(false, true)] bool advanced,
[CombinatorialValues(1, 2, 7, 101, 1024, 65535, DataGenerator.LargeBufferSize,
DataGenerator.LargeBufferSize + 1)]
int zstdBufferSize,
[CombinatorialValues(1, 2, 7, 101, 1024, 65535, DataGenerator.LargeBufferSize,
DataGenerator.LargeBufferSize + 1)]
int copyBufferSize)
{
var dict = useDict ? TrainDict() : null;
var testStream = DataGenerator.GetLargeStream(DataFill.Sequential);
const int offset = 1;
var buffer = new byte[copyBufferSize + offset + 1];
var tempStream = new MemoryStream();
await using (var compressionStream =
new CompressionStream(tempStream, Compressor.DefaultCompressionLevel, zstdBufferSize))
{
compressionStream.LoadDictionary(dict);
if (advanced)
{
compressionStream.SetParameter(ZSTD_cParameter.ZSTD_c_windowLog, 11);
compressionStream.SetParameter(ZSTD_cParameter.ZSTD_c_checksumFlag, 1);
}
int bytesRead;
while ((bytesRead = await testStream.ReadAsync(buffer, offset, copyBufferSize)) > 0)
{
await compressionStream.WriteAsync(buffer, offset, bytesRead);
}
}
tempStream.Seek(0, SeekOrigin.Begin);
var resultStream = new MemoryStream();
await using (var decompressionStream = new DecompressionStream(tempStream, zstdBufferSize))
{
decompressionStream.LoadDictionary(dict);
if (advanced)
{
decompressionStream.SetParameter(ZSTD_dParameter.ZSTD_d_windowLogMax, 11);
}
int bytesRead;
while ((bytesRead = await decompressionStream.ReadAsync(buffer, offset, copyBufferSize)) > 0)
{
await resultStream.WriteAsync(buffer, offset, bytesRead);
}
}
Assert.True(testStream.ToArray().SequenceEqual(resultStream.ToArray()));
}
public BufferedLongReader(string FilePath)
{
int ioBufSize = Math.Min(BufSize * sizeof(long), 131072);
file = new FileStream(FilePath, FileMode.Open, FileAccess.Read, FileShare.Read, ioBufSize, FileOptions.SequentialScan);
zip = new DecompressionStream(file, ioBufSize);
//最初はここで強制的に読ませる
FillNextBuf();
ActualReadAuto();
}
public void RoundTrip_BatchToStreaming()
{
var data = DataGenerator.GetLargeBuffer(DataFill.Sequential);
byte[] compressed;
using (var compressor = new Compressor())
compressed = compressor.Wrap(data);
var resultStream = new MemoryStream();
using (var decompressionStream = new DecompressionStream(new MemoryStream(compressed)))
decompressionStream.CopyTo(resultStream);
Assert.AreEqual(data, resultStream.ToArray());
}
public async Task RoundTrip_StreamingToStreamingAsync(
[Values(false, true)] bool useDict, [Values(false, true)] bool advanced,
[Values(1, 2, 7, 101, 1024, 65535, DataGenerator.LargeBufferSize, DataGenerator.LargeBufferSize + 1)] int zstdBufferSize,
[Values(1, 2, 7, 101, 1024, 65535, DataGenerator.LargeBufferSize, DataGenerator.LargeBufferSize + 1)] int copyBufferSize)
{
var dict = useDict ? TrainDict() : null;
var testStream = DataGenerator.GetLargeStream(DataFill.Sequential);
const int offset = 1;
var buffer = new byte[copyBufferSize + offset + 1];
var tempStream = new MemoryStream();
using (var compressionOptions = new CompressionOptions(dict, advanced ? new Dictionary <ZSTD_cParameter, int> {
{ ZSTD_cParameter.ZSTD_c_windowLog, 11 }, { ZSTD_cParameter.ZSTD_c_checksumFlag, 1 }, { ZSTD_cParameter.ZSTD_c_nbWorkers, 4 }
} : null))
await using (var compressionStream = new CompressionStream(tempStream, compressionOptions, zstdBufferSize))
{
int bytesRead;
while ((bytesRead = await testStream.ReadAsync(buffer, offset, copyBufferSize)) > 0)
{
await compressionStream.WriteAsync(buffer, offset, bytesRead);
}
}
tempStream.Seek(0, SeekOrigin.Begin);
var resultStream = new MemoryStream();
using (var decompressionOptions = new DecompressionOptions(dict, advanced ? new Dictionary <ZSTD_dParameter, int> {
{ ZSTD_dParameter.ZSTD_d_windowLogMax, 11 }
} : null))
await using (var decompressionStream = new DecompressionStream(tempStream, decompressionOptions, zstdBufferSize))
{
int bytesRead;
while ((bytesRead = await decompressionStream.ReadAsync(buffer, offset, copyBufferSize)) > 0)
{
await resultStream.WriteAsync(buffer, offset, bytesRead);
}
}
Assert.AreEqual(testStream.ToArray(), resultStream.ToArray());
}
public void StreamingCompressionSimpleWrite(byte[] data, int offset, int count)
{
var tempStream = new MemoryStream();
using (var compressionStream = new CompressionStream(tempStream))
compressionStream.Write(data, offset, count);
tempStream.Seek(0, SeekOrigin.Begin);
var resultStream = new MemoryStream();
using (var decompressionStream = new DecompressionStream(tempStream))
decompressionStream.CopyTo(resultStream);
var dataToCompress = new byte[count];
Array.Copy(data, offset, dataToCompress, 0, count);
Assert.AreEqual(dataToCompress, resultStream.ToArray());
}
public void StreamingCompressionFlushDataFromInternalBuffers()
{
var testBuffer = new byte[1];
var tempStream = new MemoryStream();
using var compressionStream = new CompressionStream(tempStream);
compressionStream.Write(testBuffer, 0, testBuffer.Length);
compressionStream.Flush();
Assert.True(tempStream.Length > 0);
tempStream.Seek(0, SeekOrigin.Begin);
//NOTE: without ZSTD_endStream call on compression
var resultStream = new MemoryStream();
using (var decompressionStream = new DecompressionStream(tempStream))
decompressionStream.CopyTo(resultStream);
Assert.True(testBuffer.SequenceEqual(resultStream.ToArray()));
}
public void StreamingCompressionZeroAndOneByte()
{
var data = new byte[] { 0, 0, 0, 1, 2, 3, 4, 0, 0, 0 };
var tempStream = new MemoryStream();
using (var compressionStream = new CompressionStream(tempStream))
{
compressionStream.Write(data, 0, 0);
compressionStream.Write(ReadOnlySpan <byte> .Empty);
compressionStream.WriteAsync(data, 0, 0).GetAwaiter().GetResult();
compressionStream.WriteAsync(ReadOnlyMemory <byte> .Empty).GetAwaiter().GetResult();
compressionStream.Write(data, 3, 1);
compressionStream.Write(new ReadOnlySpan <byte>(data, 4, 1));
compressionStream.Flush();
compressionStream.WriteAsync(data, 5, 1).GetAwaiter().GetResult();
compressionStream.WriteAsync(new ReadOnlyMemory <byte>(data, 6, 1)).GetAwaiter().GetResult();
compressionStream.FlushAsync().GetAwaiter().GetResult();
}
tempStream.Seek(0, SeekOrigin.Begin);
var result = new byte[data.Length];
using (var decompressionStream = new DecompressionStream(tempStream))
{
Assert.AreEqual(0, decompressionStream.Read(result, 0, 0));
Assert.AreEqual(0, decompressionStream.Read(Span <byte> .Empty));
Assert.AreEqual(0, decompressionStream.ReadAsync(result, 0, 0).GetAwaiter().GetResult());
Assert.AreEqual(0, decompressionStream.ReadAsync(Memory <byte> .Empty).GetAwaiter().GetResult());
Assert.AreEqual(1, decompressionStream.Read(result, 3, 1));
Assert.AreEqual(1, decompressionStream.Read(new Span <byte>(result, 4, 1)));
Assert.AreEqual(1, decompressionStream.ReadAsync(result, 5, 1).GetAwaiter().GetResult());
Assert.AreEqual(1, decompressionStream.ReadAsync(new Memory <byte>(result, 6, 1)).GetAwaiter().GetResult());
}
Assert.AreEqual(data, result);
}
public void DecompressStream(int zstdBufferSize, int copyBufferSize)
{
using var decompressionStream = new DecompressionStream(new MemoryStream(CompressedStreamData), zstdBufferSize);
decompressionStream.CopyTo(Stream.Null, copyBufferSize);
}
