protected override void ChannelRead0(IChannelHandlerContext ctx, Packet msg)
{
if (SnappyEnabled)
{
if (SnappyCodec.GetUncompressedLength(msg.Data) > SnappyDecoder.MaxSnappyLength)
{
throw new Exception("Max message size exceeeded"); // TODO: disconnect here
}
if (msg.Data.Length > SnappyDecoder.MaxSnappyLength / 4)
{
if (_logger.IsWarn)
{
_logger.Warn($"Big Snappy message of length {msg.Data.Length}");
}
}
else
{
if (_logger.IsTrace)
{
_logger.Trace($"Uncompressing with Snappy a message of length {msg.Data.Length}");
}
}
msg.Data = SnappyCodec.Uncompress(msg.Data);
}
if (_logger.IsTrace)
{
_logger.Trace($"Channel read... data length {msg.Data.Length}");
}
_session.ReceiveMessage(msg);
}
protected override void Decode(IChannelHandlerContext context, Packet message, List <object> output)
{
if (SnappyCodec.GetUncompressedLength(message.Data) > MaxSnappyLength)
{
throw new Exception("Max message size exceeeded"); // TODO: disconnect here
}
if (message.Data.Length > MaxSnappyLength / 4)
{
if (_logger.IsWarn)
{
_logger.Warn($"Big Snappy message of length {message.Data.Length}");
}
}
else
{
if (_logger.IsTrace)
{
_logger.Trace($"Uncompressing with Snappy a message of length {message.Data.Length}");
}
}
message.Data = SnappyCodec.Uncompress(message.Data);
output.Add(message);
}
protected override void Decode(IChannelHandlerContext context, Packet message, List <object> output)
{
if (SnappyCodec.GetUncompressedLength(message.Data) > SnappyParameters.MaxSnappyLength)
{
throw new Exception("Max message size exceeeded");
}
if (message.Data.Length > SnappyParameters.MaxSnappyLength / 4)
{
if (_logger.IsWarn)
{
_logger.Warn($"Big Snappy message of length {message.Data.Length}");
}
}
else
{
if (_logger.IsTrace)
{
_logger.Trace($"Uncompressing with Snappy a message of length {message.Data.Length}");
}
}
try
{
message.Data = SnappyCodec.Uncompress(message.Data);
}
catch
{
_logger.Error($"{message.Data.ToHexString()}");
throw;
}
output.Add(message);
}
private static void Uncompress(ReusableMemoryStream uncompressed, byte[] body, int offset, int length, CompressionCodec codec)
{
try
{
if (codec == CompressionCodec.Snappy)
{
#if NET_CORE
throw new NotImplementedException();
#else
uncompressed.SetLength(SnappyCodec.GetUncompressedLength(body, offset, length));
SnappyCodec.Uncompress(body, offset, length, uncompressed.GetBuffer(), 0);
#endif
}
else // compression == CompressionCodec.Gzip
{
using (var compressed = new MemoryStream(body, offset, length, false))
{
using (var gzip = new GZipStream(compressed, CompressionMode.Decompress))
{
using (var tmp = uncompressed.Pool.Reserve())
{
gzip.ReusableCopyTo(uncompressed, tmp);
}
}
}
}
uncompressed.Position = 0;
}
catch (Exception ex)
{
throw new UncompressException("Invalid compressed data.", codec, ex);
}
}
protected override void ChannelRead0(IChannelHandlerContext ctx, ZeroPacket input)
{
IByteBuffer content = input.Content;
if (SnappyEnabled)
{
int uncompressedLength = SnappyCodec.GetUncompressedLength(content.Array, content.ArrayOffset + content.ReaderIndex, content.ReadableBytes);
if (uncompressedLength > SnappyParameters.MaxSnappyLength)
{
throw new Exception("Max message size exceeeded"); // TODO: disconnect here
}
if (content.ReadableBytes > SnappyParameters.MaxSnappyLength / 4)
{
if (_logger.IsWarn)
{
_logger.Warn($"Big Snappy message of length {content.ReadableBytes}");
}
}
else
{
if (_logger.IsTrace)
{
_logger.Trace($"Uncompressing with Snappy a message of length {content.ReadableBytes}");
}
}
IByteBuffer output = PooledByteBufferAllocator.Default.Buffer(uncompressedLength);
try
{
int length = SnappyCodec.Uncompress(content.Array, content.ArrayOffset + content.ReaderIndex, content.ReadableBytes, output.Array, output.ArrayOffset);
output.SetWriterIndex(output.WriterIndex + length);
}
catch (Exception)
{
if (content.ReadableBytes == 2 && content.ReadByte() == 193)
{
// this is a Parity disconnect sent as a non-snappy-encoded message
// e.g. 0xc103
}
else
{
content.SkipBytes(content.ReadableBytes);
throw;
}
}
content.SkipBytes(content.ReadableBytes);
ZeroPacket outputPacket = new ZeroPacket(output);
outputPacket.PacketType = input.PacketType;
_session.ReceiveMessage(outputPacket);
outputPacket.Release();
}
else
{
_session.ReceiveMessage(input);
}
}
public byte[] Decompress(byte[] src, int srcPos, int srcSize)
{
var uncompressedLength = SnappyCodec.GetUncompressedLength(src, srcPos, srcSize);
var output = new byte[uncompressedLength];
var length = SnappyCodec.Uncompress(src, srcPos, srcSize, output, 0);
if (length == uncompressedLength)
{
return(output);
}
Array.Resize(ref output, length);
return(output);
}
protected override void ChannelRead0(IChannelHandlerContext ctx, Packet msg)
{
if (SnappyEnabled)
{
if (SnappyCodec.GetUncompressedLength(msg.Data) > SnappyParameters.MaxSnappyLength)
{
throw new Exception("Max message size exceeeded"); // TODO: disconnect here
}
if (msg.Data.Length > SnappyParameters.MaxSnappyLength / 4)
{
if (_logger.IsWarn)
{
_logger.Warn($"Big Snappy message of length {msg.Data.Length}");
}
}
else
{
if (_logger.IsTrace)
{
_logger.Trace($"Uncompressing with Snappy a message of length {msg.Data.Length}");
}
}
try
{
msg.Data = SnappyCodec.Uncompress(msg.Data);
}
catch
{
if (msg.Data.Length == 2 && msg.Data[0] == 193)
{
// this is a Parity disconnect sent as a non-snappy-encoded message
// e.g. 0xc103
}
else
{
throw;
}
}
}
if (_logger.IsTrace)
{
_logger.Trace($"Channel read... data length {msg.Data.Length}");
}
_session.ReceiveMessage(msg);
}
public void GetUncompressedLengthExceptions()
{
var uncompressed = Encoding.ASCII.GetBytes("Hello, hello, howdy?");
var compressed = SnappyCodec.Compress(uncompressed);
var buffer = new byte[100];
Assert.Throws <ArgumentNullException>(() => SnappyCodec.GetUncompressedLength(null));
Assert.Throws <ArgumentNullException>(() => SnappyCodec.GetUncompressedLength(null, 0, 3));
Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.GetUncompressedLength(compressed, -1, uncompressed.Length));
Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.GetUncompressedLength(compressed, 0, -1));
Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.GetUncompressedLength(compressed, compressed.Length - 2, 4));
Assert.Throws <InvalidDataException>(() => SnappyCodec.GetUncompressedLength(compressed, 0, 0));
Assert.Throws <InvalidDataException>(() => SnappyCodec.GetUncompressedLength(compressed, compressed.Length, 0));
var rubbish = Enumerable.Repeat((byte)0xff, 10).ToArray();
Assert.Throws <InvalidDataException>(() => SnappyCodec.GetUncompressedLength(rubbish, 0, rubbish.Length));
}
public void Evaluate(int spreadMax)
{
if (FStreamIn.SliceCount == 0 || FStreamIn[0] == null || FStreamIn[0].Length == 0)
{
spreadMax = 0;
}
else
{
spreadMax = FStreamIn.SliceCount;
}
FStreamOut.ResizeAndDispose(spreadMax, () => new MemoryStream());
for (int i = 0; i < spreadMax; i++)
{
var inputStream = FStreamIn[i];
var outputStream = FStreamOut[i];
inputStream.Position = 0;
outputStream.Position = 0;
FStreamOut[i].SetLength(0);
var compressedDataSize = (int)inputStream.Length;
IntPtr contentPtr = Marshal.AllocHGlobal(compressedDataSize);
byte[] compressedData = new byte[compressedDataSize];
inputStream.Read(compressedData, 0, compressedDataSize);
Marshal.Copy(compressedData, 0, contentPtr, compressedDataSize);
fixed(byte *bptr = &compressedData[0])
{
SnappyCodec.GetUncompressedLength(bptr, compressedDataSize, ref this.uncompressedSize);
IntPtr uncompressedDataPointer = Marshal.AllocHGlobal(this.uncompressedSize);
SnappyCodec.Uncompress(bptr, compressedDataSize, (byte *)uncompressedDataPointer, ref this.uncompressedSize);
this.uncompressedFrameData = new byte[this.uncompressedSize];
Marshal.Copy((IntPtr)uncompressedDataPointer, this.uncompressedFrameData, 0, this.uncompressedSize);
outputStream.Write(this.uncompressedFrameData, 0, this.uncompressedSize);
Marshal.FreeHGlobal(uncompressedDataPointer);
}
Marshal.FreeHGlobal(contentPtr);
}
FStreamOut.Flush(true);
}
/// <summary>
/// </summary>
/// <param name="Data"></param>
/// <returns></returns>
public static bool DecompressInPlace(byte[] Data, long DataOffset, long DataLength, out long ResultLength)
{
byte[] Buffer = CompressBuffer.Value;
int MaxBufferSize = SnappyCodec.GetUncompressedLength(Data, (int)DataOffset, (int)DataLength);
if (Buffer.Length < MaxBufferSize)
{
Buffer = new byte[MaxBufferSize];
CompressBuffer.Value = Buffer;
}
#if LOG_COMPRESS_RATIO
Stopwatch watch = new Stopwatch();
watch.Start();
#endif
ResultLength = SnappyCodec.Uncompress(Data, (int)DataOffset, (int)DataLength, Buffer, 0);
if (ResultLength <= Buffer.Length)
{
Array.Copy(Buffer, 0, Data, (int)DataOffset, ResultLength);
#if LOG_COMPRESS_RATIO
watch.Stop();
float elapsed = ((float)watch.ElapsedTicks / (float)Stopwatch.Frequency) * 1000.0f;
CompressedBytes += ResultLength;
UncompressedBytes += DataLength;
float Reduction = ((float)CompressedBytes / (float)UncompressedBytes) * 100.0f;
Console.WriteLine("DecompressInPlace: elapsed={0}ms", elapsed);
#endif
return(true);
}
else
{
return(false);
}
}
public void GetUncompressedLength_with_advanced_options_should_throw_if_parameters_incorrect(bool?provideCorrectCompressedBytes, int inputOffset, int inputLength, Type expectedExceptionType)
{
var uncompressedBytes = Encoding.ASCII.GetBytes("Hello, hello, howdy?"); // 20 bytes
byte[] compressedBytes = null;
if (provideCorrectCompressedBytes.HasValue)
{
if (provideCorrectCompressedBytes.Value)
{
compressedBytes = SnappyCodec.Compress(uncompressedBytes); // 22 bytes
}
else
{
compressedBytes = Enumerable.Repeat((byte)0xff, 10).ToArray(); // 10 bytes
}
}
var exception = Record.Exception(() => SnappyCodec.GetUncompressedLength(compressedBytes, inputOffset, inputLength));
exception.Should().BeOfType(expectedExceptionType);
}
public void ReadMsg(BitStream stream)
{
//bool isFilenames;
if (stream.ReadChar() == ':')
{
//isFilenames = true;
}
else
{
stream.Seek(-8, System.IO.SeekOrigin.Current);
//isFilenames = false;
}
TableName = stream.ReadCString();
MaxEntries = stream.ReadUShort();
int encodeBits = ExtMath.Log2(MaxEntries);
Entries = stream.ReadUShort((byte)(encodeBits + 1));
ulong bitCount = stream.ReadVarUInt();
// userdatafixedsize
if (stream.ReadBool())
{
UserDataSize = stream.ReadUShort(12);
UserDataSizeBits = stream.ReadByte(4);
}
bool isCompressedData = stream.ReadBool();
Data = stream.Subsection(stream.Cursor, stream.Cursor + bitCount);
stream.Seek(bitCount, System.IO.SeekOrigin.Current);
if (isCompressedData)
{
uint decompressedNumBytes = Data.ReadUInt();
uint compressedNumBytes = Data.ReadUInt();
byte[] compressedData = Data.ReadBytes(compressedNumBytes);
char[] magic = Encoding.ASCII.GetChars(compressedData, 0, 4);
if (
magic[0] != 'S' ||
magic[1] != 'N' ||
magic[2] != 'A' ||
magic[3] != 'P')
{
throw new FormatException("Unknown format for compressed stringtable");
}
int snappyDecompressedNumBytes = SnappyCodec.GetUncompressedLength(compressedData, 4, compressedData.Length - 4);
if (snappyDecompressedNumBytes != decompressedNumBytes)
{
throw new FormatException("Mismatching decompressed data lengths");
}
byte[] decompressedData = new byte[snappyDecompressedNumBytes];
if (SnappyCodec.Uncompress(compressedData, 4, compressedData.Length - 4, decompressedData, 0) != decompressedNumBytes)
{
throw new FormatException("Snappy didn't decode all the bytes");
}
Data = new BitStream(decompressedData);
}
}
/// <summary>
///
/// </summary>
/// <param name="Data"></param>
/// <returns></returns>
public static int GetDecompressedLength(byte[] Data, long DataOffset, long DataLength)
{
return(SnappyCodec.GetUncompressedLength(Data, (int)DataOffset, (int)DataLength));
}
public void GetUncompressedLength_should_throw_if_parameter_null()
{
var exception = Record.Exception(() => SnappyCodec.GetUncompressedLength(null));
exception.Should().BeOfType <ArgumentNullException>();
}