public void CompressUncompressEmpty()
{
var compressed = SnappyCodec.Compress(new byte[0]);
Assert.That(compressed.Length, Is.GreaterThan(0));
Assert.AreEqual(0, SnappyCodec.Uncompress(compressed).Length);
}
public bool uncompressVoxData()
{
//0 -> chunk isn't in use
//1 -> chunk is in use
if (0 == Interlocked.Exchange(ref this.currentlyWorked, 1))
{
Console.WriteLine("Chunk compressing at thread {0}", System.Threading.Thread.CurrentThread.ManagedThreadId);
if (this.chunkData.Length == 1)
{
this.chunkData = SnappyCodec.Uncompress(this.compChunkData);
}
Interlocked.Exchange(ref this.currentlyWorked, 0);
return(true);
}
else
{
while (Interlocked.Read(ref this.currentlyWorked) == 1)
{
Console.WriteLine("Waiting on other thread to complete their operation: {0}", System.Threading.Thread.CurrentThread.ManagedThreadId);
}
Console.WriteLine("Chunk already uncompressed");
return(false);
}
}
bool ReadBlock()
{
if (!StreamUtils.ReadAll(_base, _compressedBuffer, 4))
{
return(false);
}
var blockSize = BigEndianConverter.ReadInt32(_compressedBuffer);
if (blockSize > _compressedBuffer.Length)
{
_compressedBuffer = new byte[blockSize];
}
if (!StreamUtils.ReadAll(_base, _compressedBuffer, blockSize))
{
throw new InvalidDataException("Unexpected end of snappy data block");
}
// Reallocate compressed buffer if needed
var uncompressedLen = Snappy.SnappyCodec.GetUncompressedLength(_compressedBuffer, 0, blockSize);
if (uncompressedLen > _uncompressedBuffer.Length)
{
_uncompressedBuffer = new byte[uncompressedLen];
}
var read = SnappyCodec.Uncompress(_compressedBuffer, 0, blockSize, _uncompressedBuffer, 0);
_bufferLen = read;
_bufferPtr = 0;
return(true);
}
public void CompressUncompressEmpty()
{
var compressed = SnappyCodec.Compress(new byte[0]);
Assert.True(compressed.Length > 0);
Assert.Equal(0, SnappyCodec.Uncompress(compressed).Length);
}
static void Main(string[] args)
{
var data = Encoding.ASCII.GetBytes("Hello World!");
Console.WriteLine("SnappyCodec roundtrip: {0}", Encoding.ASCII.GetString(SnappyCodec.Uncompress(SnappyCodec.Compress(data))));
var buffer = new MemoryStream();
var stream = new SnappyStream(buffer, CompressionMode.Compress);
stream.Write(data, 0, data.Length);
stream.Close();
buffer = new MemoryStream(buffer.ToArray());
stream = new SnappyStream(buffer, CompressionMode.Decompress);
var roundtrip = new byte[data.Length];
int read = stream.Read(roundtrip, 0, data.Length);
if (read != data.Length)
{
throw new ApplicationException();
}
if (0 != stream.Read(roundtrip, 0, data.Length))
{
throw new ApplicationException();
}
Console.WriteLine("SnappyStream roundtrip: {0}", Encoding.ASCII.GetString(roundtrip));
}
private static void Uncompress(ReusableMemoryStream uncompressed, byte[] body, int offset, int length, CompressionCodec codec)
{
try
{
if (codec == CompressionCodec.Snappy)
{
#if NET_CORE
throw new NotImplementedException();
#else
uncompressed.SetLength(SnappyCodec.GetUncompressedLength(body, offset, length));
SnappyCodec.Uncompress(body, offset, length, uncompressed.GetBuffer(), 0);
#endif
}
else // compression == CompressionCodec.Gzip
{
using (var compressed = new MemoryStream(body, offset, length, false))
{
using (var gzip = new GZipStream(compressed, CompressionMode.Decompress))
{
using (var tmp = uncompressed.Pool.Reserve())
{
gzip.ReusableCopyTo(uncompressed, tmp);
}
}
}
}
uncompressed.Position = 0;
}
catch (Exception ex)
{
throw new UncompressException("Invalid compressed data.", codec, ex);
}
}
public byte[] Decode(byte[] encoded, int uncompressedLength)
{
var target = new byte[uncompressedLength];
SnappyCodec.Uncompress(encoded, 0, encoded.Length, target, 0);
return(target);
}
protected override void ChannelRead0(IChannelHandlerContext ctx, Packet msg)
{
if (SnappyEnabled)
{
if (SnappyCodec.GetUncompressedLength(msg.Data) > SnappyDecoder.MaxSnappyLength)
{
throw new Exception("Max message size exceeeded"); // TODO: disconnect here
}
if (msg.Data.Length > SnappyDecoder.MaxSnappyLength / 4)
{
if (_logger.IsWarn)
{
_logger.Warn($"Big Snappy message of length {msg.Data.Length}");
}
}
else
{
if (_logger.IsTrace)
{
_logger.Trace($"Uncompressing with Snappy a message of length {msg.Data.Length}");
}
}
msg.Data = SnappyCodec.Uncompress(msg.Data);
}
if (_logger.IsTrace)
{
_logger.Trace($"Channel read... data length {msg.Data.Length}");
}
_session.ReceiveMessage(msg);
}
protected override void Decode(IChannelHandlerContext context, Packet message, List <object> output)
{
if (SnappyCodec.GetUncompressedLength(message.Data) > SnappyParameters.MaxSnappyLength)
{
throw new Exception("Max message size exceeeded");
}
if (message.Data.Length > SnappyParameters.MaxSnappyLength / 4)
{
if (_logger.IsWarn)
{
_logger.Warn($"Big Snappy message of length {message.Data.Length}");
}
}
else
{
if (_logger.IsTrace)
{
_logger.Trace($"Uncompressing with Snappy a message of length {message.Data.Length}");
}
}
try
{
message.Data = SnappyCodec.Uncompress(message.Data);
}
catch
{
_logger.Error($"{message.Data.ToHexString()}");
throw;
}
output.Add(message);
}
protected override void Decode(IChannelHandlerContext context, Packet message, List <object> output)
{
if (SnappyCodec.GetUncompressedLength(message.Data) > MaxSnappyLength)
{
throw new Exception("Max message size exceeeded"); // TODO: disconnect here
}
if (message.Data.Length > MaxSnappyLength / 4)
{
if (_logger.IsWarn)
{
_logger.Warn($"Big Snappy message of length {message.Data.Length}");
}
}
else
{
if (_logger.IsTrace)
{
_logger.Trace($"Uncompressing with Snappy a message of length {message.Data.Length}");
}
}
message.Data = SnappyCodec.Uncompress(message.Data);
output.Add(message);
}
public void Uncompress_with_advanced_parameters_should_throw_if_parameters_incorrect(bool?provideCorrectCompressedBytes, int inputOffset, int inputLength, int?outputCount, int outputOffset, Type expectedException)
{
var uncompressedBytes = Encoding.ASCII.GetBytes("Hello, hello, howdy?"); // 20 bytes
byte[] compressedBytes = null;
if (provideCorrectCompressedBytes.HasValue)
{
if (provideCorrectCompressedBytes.Value)
{
compressedBytes = SnappyCodec.Compress(uncompressedBytes);
}
else
{
compressedBytes = new byte[inputLength];
new Random(0).NextBytes(compressedBytes);
}
}
var outputBytes = outputCount.HasValue ? new byte[outputCount.Value] : null;
var outputLength = (outputBytes?.Length ?? 0) - outputOffset;
var exception = Record.Exception(() => SnappyCodec.Uncompress(compressedBytes, inputOffset, inputLength, outputBytes, outputOffset, outputLength));
exception.Should().BeOfType(expectedException);
}
/// <summary>
/// Extract existing memory-mapped-file,
/// decompress with the proper algorithm.
/// </summary>
/// <param name="output"></param>
public void ExtractExistingMMF(Stream output, MemoryMappedFile memorymappedbundle)
{
/* var hash = *//*@"Global\" + *//*Archive.FileName.GetHashMD5();
* System.Console.WriteLine(hash);
* using (MemoryMappedFile mmf = MemoryMappedFile.OpenExisting(hash, MemoryMappedFileRights.Read, HandleInheritability.Inheritable))
*/
using (var viewstream = memorymappedbundle.CreateViewStream(PageOffset, ZSize, MemoryMappedFileAccess.Read))
{
switch (CompressionType)
{
case "None":
{
viewstream.CopyTo(output);
break;
}
case "Lz4":
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = LZ4Codec.Decode(buffer, 0, c, (int)Size);
output.Write(uncompressed, 0, uncompressed.Length);
break;
}
case "Snappy":
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = SnappyCodec.Uncompress(buffer);
output.Write(uncompressed, 0, uncompressed.Length);
break;
}
case "Doboz":
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = DobozCodec.Decode(buffer, 0, c);
output.Write(uncompressed, 0, uncompressed.Length);
break;
}
case "Zlib":
{
var zlib = new ZlibStream(viewstream, CompressionMode.Decompress);
zlib.CopyTo(output);
break;
}
default:
throw new MissingCompressionException("Unhandled compression algorithm.")
{
Compression = Compression
};
}
viewstream.Close();
}
}
public byte[] UncompressedData()
{
using (MemoryMappedFile file = MemoryMappedFile.CreateFromFile(Bundle.FileName, FileMode.Open))
{
using (MemoryMappedViewStream viewstream = file.CreateViewStream(PageOFfset, ZSize, MemoryMappedFileAccess.Read))
{
switch (CompressionType)
{
case "None":
{
using (MemoryStream destination = new MemoryStream())
{
viewstream.CopyTo(destination);
return(destination.ToArray());
}
}
case "Lz4":
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = LZ4Codec.Decode(buffer, 0, c, (int)Size);
return(uncompressed);
}
case "Snappy":
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = SnappyCodec.Uncompress(buffer);
return(uncompressed);
}
case "Doboz":
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = DobozCodec.Decode(buffer, 0, c);
return(uncompressed);
}
case "Zlib":
{
using (MemoryStream destination = new MemoryStream())
{
var zlib = new ZlibStream(viewstream, CompressionMode.Decompress);
zlib.CopyTo(destination);
return(destination.ToArray());
}
}
default:
throw new MissingCompressionException("Unhandled compression algorithm.")
{
Compression = Compression
};
}
}
}
}
public void Extract(Stream output)
{
using (var file = MemoryMappedFile.CreateFromFile(Bundle.FileName, FileMode.Open))
{
using (var viewstream = file.CreateViewStream(Offset, ZSize, MemoryMappedFileAccess.Read))
{
switch (CompressionType)
{
case "None":
{
viewstream.CopyTo(output);
break;
}
case "Lz4":
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = LZ4Codec.Decode(buffer, 0, c, (int)Size);
output.Write(uncompressed, 0, uncompressed.Length);
break;
}
case "Snappy":
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = SnappyCodec.Uncompress(buffer);
output.Write(uncompressed, 0, uncompressed.Length);
break;
}
case "Doboz":
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = DobozCodec.Decode(buffer, 0, c);
output.Write(uncompressed, 0, uncompressed.Length);
break;
}
case "Zlib":
{
var zlib = new ZlibStream(viewstream, CompressionMode.Decompress);
zlib.CopyTo(output);
break;
}
default:
throw new MissingCompressionException("Unhandled compression algorithm.")
{
Compression = Compression
};
}
viewstream.Close();
}
}
}
public void CompressRange()
{
var input = Encoding.ASCII.GetBytes("ByeHelloBye");
var output = new byte[100];
var length = SnappyCodec.Compress(input, 3, 5, output, 10);
Assert.AreEqual("Hello", Encoding.ASCII.GetString(SnappyCodec.Uncompress(output.Skip(10).Take(length).ToArray())));
}
public void UncompressRange()
{
var howdy = Encoding.ASCII.GetBytes("Howdy");
var padded = howdy.Take(3).Concat(SnappyCodec.Compress(Encoding.ASCII.GetBytes("Hello"))).Concat(howdy.Skip(3)).ToArray();
var output = new byte[100];
var length = SnappyCodec.Uncompress(padded, 3, padded.Length - 5, output, 10);
Assert.AreEqual(5, length);
Assert.AreEqual("Hello", Encoding.ASCII.GetString(output.Skip(10).Take(5).ToArray()));
}
/// <summary>
/// Decompresses the remainder of <paramref name="input"/>, writing the uncompressed data to <paramref name="output"/>.
/// </summary>
/// <param name="input">The input stream.</param>
/// <param name="output">The output stream.</param>
public void Decompress(Stream input, Stream output)
{
var compressedSize = (int)(input.Length - input.Position);
var compressedBytes = new byte[compressedSize];
input.ReadBytes(compressedBytes, offset: 0, count: compressedSize, CancellationToken.None);
var decompressedBytes = SnappyCodec.Uncompress(compressedBytes);
output.Write(decompressedBytes, offset: 0, count: decompressedBytes.Length);
}
public object Deserialize(Type type, byte[] data)
{
if (data == null)
{
throw new ArgumentNullException(nameof(data));
}
data = compressed ? SnappyCodec.Uncompress(data) : data;
return(XDocument.Parse(data.GetString()).XmlDeserialize(type));
}
/// <summary>
/// Decompresses the remainder of <paramref name="input"/>, writing the uncompressed data to <paramref name="output"/>.
/// </summary>
/// <param name="input">The input stream.</param>
/// <param name="output">The output stream.</param>
public void Decompress(Stream input, Stream output)
{
#if NET452 || NETSTANDARD2_0
var compressedSize = (int)(input.Length - input.Position);
var compressedBytes = new byte[compressedSize];
input.ReadBytes(compressedBytes, offset: 0, count: compressedSize, CancellationToken.None);
var decompressedBytes = SnappyCodec.Uncompress(compressedBytes);
output.Write(decompressedBytes, offset: 0, count: decompressedBytes.Length);
#else
throw new NotSupportedException();
#endif
}
public void Compress_should_uncompress_previously_compressed_message(string input)
{
var inputBytes = Encoding.ASCII.GetBytes(input);
var compressedBytes = SnappyCodec.Compress(inputBytes);
compressedBytes.Length.Should().BeGreaterThan(0);
var uncompressedBytes = SnappyCodec.Uncompress(compressedBytes);
var outputString = Encoding.ASCII.GetString(uncompressedBytes);
outputString.Should().Be(input);
}
public static T ProtoDecompressDeserialize <T>(this byte[] data) where T : class
{
if (data == null)
{
return(null);
}
using (MemoryStream memoryStream = new MemoryStream(SnappyCodec.Uncompress(data)))
{
return(Serializer.Deserialize <T>(memoryStream));
}
}
public static object ProtoDecompressDeserialize(this byte[] data, Type type)
{
if (data == null)
{
return(null);
}
using (MemoryStream memoryStream = new MemoryStream(SnappyCodec.Uncompress(data)))
{
return(Serializer.NonGeneric.Deserialize(type, memoryStream));
}
}
public byte[] Decompress(byte[] data)
{
byte[] result = null;
try
{
result = SnappyCodec.Uncompress(data);
}
catch (Exception ex)
{
Console.WriteLine($"EXCEPTION: {ex.GetType().Name}, {ex.Message}");
}
return(result);
}
/// <summary>Uncompresses the specified input file.</summary>
/// <param name="input">The input file.</param>
/// <param name="output">The output file.</param>
private static void Uncompress(string input, string output)
{
byte[] compressed = File.ReadAllBytes(input);
var timer = Stopwatch.StartNew();
var decompressed = SnappyCodec.Uncompress(compressed, 0, compressed.Length);
timer.Stop();
Console.WriteLine("Decompression:");
Console.WriteLine(" Speed: {0:0.00}MB/s", (double)decompressed.Length / 1024 / 1024 / timer.Elapsed.TotalSeconds);
Console.WriteLine(" Ratio: {0:0.00}%", (double)compressed.Length * 100 / decompressed.Length);
Console.WriteLine(" Hash: {0}", decompressed.MD5());
File.WriteAllBytes(output, decompressed);
}
public void Compress_with_advanced_options_should_work_as_expected(int inputOffset, int inputLength, int outputOffset, string expectedResult)
{
var input = Encoding.ASCII.GetBytes("ByeHelloBye");
var output = new byte[100];
var outputLength = output.Length - outputOffset;
var compressedLength = SnappyCodec.Compress(input, inputOffset, inputLength, output, outputOffset, outputLength);
var compressedOutputWithoutOffset = output.Skip(outputOffset).Take(compressedLength).ToArray();
var uncompressed = SnappyCodec.Uncompress(compressedOutputWithoutOffset);
var uncompressedString = Encoding.ASCII.GetString(uncompressed);
uncompressedString.Should().Be(expectedResult);
}
public byte[] Decompress(byte[] src, int srcPos, int srcSize)
{
var uncompressedLength = SnappyCodec.GetUncompressedLength(src, srcPos, srcSize);
var output = new byte[uncompressedLength];
var length = SnappyCodec.Uncompress(src, srcPos, srcSize, output, 0);
if (length == uncompressedLength)
{
return(output);
}
Array.Resize(ref output, length);
return(output);
}
protected override void ChannelRead0(IChannelHandlerContext ctx, Packet msg)
{
if (SnappyEnabled)
{
if (SnappyCodec.GetUncompressedLength(msg.Data) > SnappyParameters.MaxSnappyLength)
{
throw new Exception("Max message size exceeeded"); // TODO: disconnect here
}
if (msg.Data.Length > SnappyParameters.MaxSnappyLength / 4)
{
if (_logger.IsWarn)
{
_logger.Warn($"Big Snappy message of length {msg.Data.Length}");
}
}
else
{
if (_logger.IsTrace)
{
_logger.Trace($"Uncompressing with Snappy a message of length {msg.Data.Length}");
}
}
try
{
msg.Data = SnappyCodec.Uncompress(msg.Data);
}
catch
{
if (msg.Data.Length == 2 && msg.Data[0] == 193)
{
// this is a Parity disconnect sent as a non-snappy-encoded message
// e.g. 0xc103
}
else
{
throw;
}
}
}
if (_logger.IsTrace)
{
_logger.Trace($"Channel read... data length {msg.Data.Length}");
}
_session.ReceiveMessage(msg);
}
public void Uncompression(string name)
{
Benchmark.Run("Uncompressing", name, benchmark =>
{
var compressed = SnappyCodec.Compress(benchmark.Input);
var roundtrip = new byte[benchmark.Input.Length];
int length = 0;
benchmark.Stopwatch.Start();
for (int i = 0; i < benchmark.Iterations; ++i)
{
length = SnappyCodec.Uncompress(compressed, 0, compressed.Length, roundtrip, 0);
}
benchmark.Stopwatch.Stop();
CollectionAssert.AreEqual(benchmark.Input, roundtrip);
});
}
public void Evaluate(int spreadMax)
{
if (FStreamIn.SliceCount == 0 || FStreamIn[0] == null || FStreamIn[0].Length == 0)
{
spreadMax = 0;
}
else
{
spreadMax = FStreamIn.SliceCount;
}
FStreamOut.ResizeAndDispose(spreadMax, () => new MemoryStream());
for (int i = 0; i < spreadMax; i++)
{
var inputStream = FStreamIn[i];
var outputStream = FStreamOut[i];
inputStream.Position = 0;
outputStream.Position = 0;
FStreamOut[i].SetLength(0);
var compressedDataSize = (int)inputStream.Length;
IntPtr contentPtr = Marshal.AllocHGlobal(compressedDataSize);
byte[] compressedData = new byte[compressedDataSize];
inputStream.Read(compressedData, 0, compressedDataSize);
Marshal.Copy(compressedData, 0, contentPtr, compressedDataSize);
fixed(byte *bptr = &compressedData[0])
{
SnappyCodec.GetUncompressedLength(bptr, compressedDataSize, ref this.uncompressedSize);
IntPtr uncompressedDataPointer = Marshal.AllocHGlobal(this.uncompressedSize);
SnappyCodec.Uncompress(bptr, compressedDataSize, (byte *)uncompressedDataPointer, ref this.uncompressedSize);
this.uncompressedFrameData = new byte[this.uncompressedSize];
Marshal.Copy((IntPtr)uncompressedDataPointer, this.uncompressedFrameData, 0, this.uncompressedSize);
outputStream.Write(this.uncompressedFrameData, 0, this.uncompressedSize);
Marshal.FreeHGlobal(uncompressedDataPointer);
}
Marshal.FreeHGlobal(contentPtr);
}
FStreamOut.Flush(true);
}
