static void Main(string[] args)
{
var data = Encoding.ASCII.GetBytes("Hello World!");
Console.WriteLine("SnappyCodec roundtrip: {0}", Encoding.ASCII.GetString(SnappyCodec.Uncompress(SnappyCodec.Compress(data))));
var buffer = new MemoryStream();
var stream = new SnappyStream(buffer, CompressionMode.Compress);
stream.Write(data, 0, data.Length);
stream.Close();
buffer = new MemoryStream(buffer.ToArray());
stream = new SnappyStream(buffer, CompressionMode.Decompress);
var roundtrip = new byte[data.Length];
int read = stream.Read(roundtrip, 0, data.Length);
if (read != data.Length)
{
throw new ApplicationException();
}
if (0 != stream.Read(roundtrip, 0, data.Length))
{
throw new ApplicationException();
}
Console.WriteLine("SnappyStream roundtrip: {0}", Encoding.ASCII.GetString(roundtrip));
}
public async Task CompressAndDecompressAsync(string filename)
{
using var resource =
typeof(SnappyStreamTests).Assembly.GetManifestResourceStream($"Snappier.Tests.TestData.{filename}");
Assert.NotNull(resource);
using var output = new MemoryStream();
using (var compressor = new SnappyStream(output, CompressionMode.Compress, true))
{
await resource.CopyToAsync(compressor);
}
output.Position = 0;
using var decompressor = new SnappyStream(output, CompressionMode.Decompress, true);
using var streamReader = new StreamReader(decompressor, Encoding.UTF8);
var decompressedText = await streamReader.ReadToEndAsync();
_outputHelper.WriteLine(decompressedText);
using var sourceResource = typeof(SnappyStreamTests).Assembly.GetManifestResourceStream($"Snappier.Tests.TestData.{filename}");
Assert.NotNull(sourceResource);
using var streamReader2 = new StreamReader(sourceResource, Encoding.UTF8);
var sourceText = await streamReader2.ReadToEndAsync();
Assert.Equal(sourceText, decompressedText);
}
public void stream_cannot_read_when_compressing()
{
using (var ms = new MemoryStream())
{
var target = new SnappyStream(ms, CompressionMode.Compress);
Assert.False(target.CanRead);
}
}
public void stream_can_read_when_decompressing()
{
using (var ms = new MemoryStream(new byte[] { 255, 115, 78, 97, 80, 112, 89, 0, 0, 0, 8, 0, 100, 0, 254, 1, 0, 130, 1, 0, 0 }))
{
var target = new SnappyStream(ms, CompressionMode.Decompress);
Assert.True(target.CanRead);
}
}
public void identifier_and_header_written_on_compression_construction()
{
var ms = new MemoryStream();
using (var target = new SnappyStream(ms, CompressionMode.Compress, true, false))
{
}
Assert.Equal(new byte[] {0xff, (byte)'s', (byte)'N', (byte)'a', (byte)'P', (byte)'p', (byte)'Y'}, ms.GetBuffer().Take((int) ms.Length));
}
public void underlying_stream_closed_on_dispose()
{
var ms = new MemoryStream();
using (var target = new SnappyStream(ms, CompressionMode.Compress))
{
}
Assert.Throws<ObjectDisposedException>(() => ms.Capacity);
}
public void underlying_stream_not_closed_when_contructor_says_no()
{
var ms = new MemoryStream();
using (var target = new SnappyStream(ms, CompressionMode.Compress, true, false))
{
}
Assert.True(ms.CanWrite && ms.CanRead);
}
public void stream_cannot_read_when_compressing()
{
using (var ms = new MemoryStream())
{
var target = new SnappyStream(ms, CompressionMode.Compress);
Assert.False(target.CanRead);
}
}
public void stream_can_read_when_decompressing()
{
using (var ms = new MemoryStream(new byte[] {255, 115, 78, 97, 80, 112, 89, 0, 0, 0, 8, 0, 100, 0, 254, 1, 0, 130, 1, 0, 0 }))
{
var target = new SnappyStream(ms, CompressionMode.Decompress);
Assert.True(target.CanRead);
}
}
public void Snappier()
{
_source.Position = 0;
_destination.Position = 0;
using var stream = new SnappyStream(_destination, CompressionMode.Compress, true);
_source.CopyTo(_destination, ReadSize);
}
public void Snappier()
{
_memoryStream.Position = 0;
using var stream = new SnappyStream(_memoryStream, CompressionMode.Decompress, true);
while (stream.Read(_buffer, 0, ReadSize) > 0)
{
}
}
public void underlying_stream_closed_on_dispose()
{
var ms = new MemoryStream();
using (var target = new SnappyStream(ms, CompressionMode.Compress))
{
}
Assert.Throws <ObjectDisposedException>(() => ms.Capacity);
}
public void stream_can_never_seek()
{
using (var ms = new MemoryStream())
{
Assert.True(ms.CanSeek);
var target = new SnappyStream(ms, CompressionMode.Compress);
Assert.False(target.CanSeek);
}
}
public void identifier_and_header_written_on_compression_construction()
{
var ms = new MemoryStream();
using (var target = new SnappyStream(ms, CompressionMode.Compress, true, false))
{
}
Assert.Equal(new byte[] { 0xff, (byte)'s', (byte)'N', (byte)'a', (byte)'P', (byte)'p', (byte)'Y' }, ms.ToArray().Take((int)ms.Length));
}
public void stream_can_never_seek()
{
using (var ms = new MemoryStream())
{
Assert.True(ms.CanSeek);
var target = new SnappyStream(ms, CompressionMode.Compress);
Assert.False(target.CanSeek);
}
}
public void underlying_stream_not_closed_when_contructor_says_no()
{
var ms = new MemoryStream();
using (var target = new SnappyStream(ms, CompressionMode.Compress, true, false))
{
}
Assert.True(ms.CanWrite && ms.CanRead);
}
byte[] GetCompressedFile(byte[] uncompressed)
{
var compressed = new MemoryStream();
using (var compressor = new SnappyStream(compressed, CompressionMode.Compress, true))
compressor.Write(uncompressed, 0, uncompressed.Length);
compressed.Close();
return(compressed.ToArray());
}
public void stream_write_throws_exception_on_chunk_length_too_long()
{
var ms = new MemoryStream();
using (var target = new SnappyStream(ms, CompressionMode.Compress, true, false))
{
byte[] buffer = new byte[1 << 20];
Assert.Throws <InvalidOperationException>(() => target.Write(buffer, 0, 1 << 20 + 1));
}
}
public override byte[] Compress(byte[] uncompressedData)
{
var outStream = new MemoryStream();
using (var compress = new SnappyStream(outStream, CompressionMode.Compress))
{
compress.Write(uncompressedData, 0, uncompressedData.Length);
}
return(outStream.ToArray());
}
public override byte[] Decompress(byte[] compressedData)
{
var inStream = new MemoryStream(compressedData);
var outStream = new MemoryStream();
using (var decompress = new SnappyStream(inStream, CompressionMode.Decompress))
{
CopyTo(decompress, outStream);
}
return(outStream.ToArray());
}
public void Decompress(Stream input, Stream output)
{
#if NET452 || NETSTANDARD2_0
using (var snappyStream = new SnappyStream(input, CompressionMode.Decompress, true))
{
snappyStream.CopyTo(output);
}
#else
throw new NotSupportedException();
#endif
}
public void data_written_to_stream()
{
var ms = new MemoryStream();
using (var target = new SnappyStream(ms, CompressionMode.Compress, true, false))
{
byte[] buffer = new byte[100];
target.Write(buffer, 0, buffer.Length);
}
Assert.Equal(new byte[] { 255, 115, 78, 97, 80, 112, 89, 0, 0, 0, 8, 0, 100, 0, 254, 1, 0, 130, 1, 0, 0 }, ms.ToArray().Take((int)ms.Length));
}
public void UnCompress(string compressedFile, string destinationFile)
{
using (Stream source = File.OpenRead(compressedFile))
{
using (var decoder = new SnappyStream(source, CompressionMode.Decompress))
{
using (var file = new FileStream(destinationFile, FileMode.Create, FileAccess.Write))
{
decoder.CopyTo(file);
}
}
}
}
public void Compress(string sourceFile, string compressedFile)
{
using (Stream source = File.OpenRead(sourceFile))
{
using (var file = new FileStream(compressedFile, FileMode.Create, FileAccess.Write))
{
using (var compressor = new SnappyStream(file, CompressionMode.Compress))
{
source.CopyTo(compressor);
}
}
}
}
static void ProcessFile(Stream input, Stream output, Options options)
{
var buffer = new byte[8192];
if (options.Mode == CompressionMode.Compress)
{
using (var compressor = new SnappyStream(output, CompressionMode.Compress, true))
{
while (true)
{
int read = input.Read(buffer, 0, buffer.Length);
if (read == 0)
{
break;
}
compressor.Write(buffer, 0, read);
}
}
}
else if (!options.Test)
{
using (var decompressor = new SnappyStream(input, CompressionMode.Decompress, true))
{
while (true)
{
int read = decompressor.Read(buffer, 0, buffer.Length);
if (read == 0)
{
break;
}
output.Write(buffer, 0, read);
}
}
}
else
{
using (var decompressor = new SnappyStream(input, CompressionMode.Decompress, true))
{
while (true)
{
int read = input.Read(buffer, 0, buffer.Length);
if (read == 0)
{
break;
}
}
}
}
}
public void LoadToMemory()
{
_memoryStream = new MemoryStream();
using var resource =
typeof(DecompressAll).Assembly.GetManifestResourceStream("Snappier.Benchmarks.TestData." + FileName);
using var compressStream = new SnappyStream(_memoryStream, CompressionMode.Compress, true);
// ReSharper disable once PossibleNullReferenceException
resource.CopyTo(compressStream);
compressStream.Flush();
_buffer = new byte[65536];
}
public void AsyncDecompression(string name)
{
Benchmark.Run("Async-decompressing", name, benchmark =>
{
var stream = new RepeaterStream(GetCompressedFile(benchmark.Input));
var decompressor = new SnappyStream(stream, CompressionMode.Decompress);
var decompressed = new byte[benchmark.Input.Length];
benchmark.Stopwatch.Start();
for (int i = 0; i < benchmark.Iterations; ++i)
{
ReadFullyAsync(decompressor, decompressed, 0, decompressed.Length).Wait();
}
benchmark.Stopwatch.Stop();
});
}
public void AsyncCompression(string name)
{
Benchmark.Run("Async-compressing", name, benchmark =>
{
var stream = new NullStream();
var compressor = new SnappyStream(stream, CompressionMode.Compress);
benchmark.Stopwatch.Start();
for (int i = 0; i < benchmark.Iterations; ++i)
{
compressor.WriteAsync(benchmark.Input, 0, benchmark.Input.Length).Wait();
compressor.FlushAsync().Wait();
}
benchmark.Stopwatch.Stop();
benchmark.Note = String.Format(" ({0:0.00 %})", stream.Written / (double)benchmark.Input.Length / benchmark.Iterations);
});
}
static void Main(string[] args)
{
try
{
var folder = Path.Combine(Path.GetTempPath(), "Snappy.NET-" + "XXX");
Console.WriteLine($"Temp folder: {folder}");
var file = File.OpenWrite("mydata.sz");
var compressor = new SnappyStream(file, CompressionMode.Compress);
var writer = new StreamWriter(compressor);
writer.WriteLine("Hello World!Hello World!Hello World!Hello World!Hello World!Hello World!Hello World!");
writer.Close();
var data = File.ReadAllBytes("mydata.sz");
Console.WriteLine($"Length: {data.Length}");
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
public static PositionTrackingStream CreateWriter(
Stream nakedStream, CompressionMethod compressionMethod)
{
Stream dest = nakedStream;
switch (compressionMethod)
{
case CompressionMethod.Gzip:
dest = new GZipStream(dest, CompressionLevel.Optimal, false);
break;
case CompressionMethod.Snappy:
dest = new SnappyStream(dest, CompressionMode.Compress, false, false);
break;
case CompressionMethod.None:
break;
default:
throw new NotImplementedException();
}
return(new PositionTrackingStream(dest));
}
public void stream_write_throws_exception_on_chunk_length_too_long()
{
var ms = new MemoryStream();
using (var target = new SnappyStream(ms, CompressionMode.Compress, true, false))
{
byte[] buffer = new byte[1 << 20];
Assert.Throws<InvalidOperationException>(() => target.Write(buffer, 0, 1 << 20));
}
}
public void data_written_to_stream()
{
var ms = new MemoryStream();
using (var target = new SnappyStream(ms, CompressionMode.Compress, true, false))
{
byte[] buffer = new byte[100];
target.Write(buffer, 0, buffer.Length);
}
Assert.Equal(new byte[] { 255, 115, 78, 97, 80, 112, 89, 0, 0, 0, 8, 0, 100, 0, 254, 1, 0, 130, 1, 0, 0 }, ms.GetBuffer().Take((int) ms.Length));
}
public void Twister()
{
var testdata = Directory.GetFiles(Benchmark.DataPath).Select(f => File.ReadAllBytes(f)).ToArray();
long totalData = 0;
var stopwatch = new Stopwatch();
stopwatch.Start();
while (stopwatch.Elapsed < TimeSpan.FromSeconds(3))
{
int count = Random.Next(0, 21);
var sequence = Enumerable.Range(0, count).Select(n => testdata[Random.Next(testdata.Length)]).ToArray();
totalData += sequence.Sum(f => f.Length);
var stream = new RandomChunkStream();
ManualResetEvent doneReading = new ManualResetEvent(false);
ThreadPool.QueueUserWorkItem(ctx =>
{
try
{
using (var decompressor = new SnappyStream(stream, CompressionMode.Decompress))
{
foreach (var file in sequence)
{
var decompressed = new byte[file.Length];
if (decompressed.Length < 500)
{
for (int i = 0; i < decompressed.Length; ++i)
{
decompressed[i] = checked ((byte)decompressor.ReadByte());
}
}
else
{
#if SNAPPY_ASYNC
if (ReadRandom.Next(2) == 0)
{
ReadAllAsync(decompressor, decompressed, 0, decompressed.Length).Wait();
}
else
#endif
ReadAll(decompressor, decompressed, 0, decompressed.Length);
}
CheckBuffers(file, decompressed);
}
Assert.AreEqual(-1, decompressor.ReadByte());
}
doneReading.Set();
}
catch (Exception e)
{
Console.WriteLine("Read thread failed: {0}", e);
throw;
}
});
using (var compressor = new SnappyStream(stream, CompressionMode.Compress))
{
foreach (var file in sequence)
{
if (file.Length < 500)
{
for (int i = 0; i < file.Length; ++i)
{
compressor.WriteByte(file[i]);
}
}
else
{
#if SNAPPY_ASYNC
if (WriteRandom.Next(2) == 0)
{
compressor.WriteAsync(file, 0, file.Length).Wait();
}
else
#endif
compressor.Write(file, 0, file.Length);
}
if (WriteRandom.Next(10) == 0)
{
#if SNAPPY_ASYNC
if (WriteRandom.Next(2) == 0)
{
compressor.FlushAsync().Wait();
}
else
#endif
compressor.Flush();
}
}
#if SNAPPY_ASYNC
if (WriteRandom.Next(3) == 0)
{
compressor.FlushAsync().Wait();
}
else
#endif
{
if (WriteRandom.Next(2) == 0)
{
compressor.Flush();
}
}
}
doneReading.WaitOne();
}
stopwatch.Stop();
Console.WriteLine("Ran {0} MB through the stream, that's {1:0.0} MB/s", totalData / 1024 / 1024, totalData / stopwatch.Elapsed.TotalSeconds / 1024 / 1024);
}
