private static void _SerializeMessages(ReusableMemoryStream stream, IEnumerable <Message> messages, SerializationInfo info)
{
if (info.CompressionCodec != CompressionCodec.None)
{
stream.Write(Basics.Zero64, 0, 8);
using (var msgsetStream = stream.Pool.Reserve())
{
SerializeMessagesUncompressed(msgsetStream, messages, info.Serializers, info.MessageVersion);
using (var compressed = stream.Pool.Reserve())
{
if (info.CompressionCodec == CompressionCodec.Gzip)
{
using (var gzip = new GZipStream(compressed, CompressionMode.Compress, true))
{
msgsetStream.WriteTo(gzip);
}
}
else // Snappy
{
#if NETSTANDARD1_3
throw new NotImplementedException();
#else
compressed.SetLength(SnappyCodec.GetMaxCompressedLength((int)msgsetStream.Length));
{
int size = SnappyCodec.Compress(msgsetStream.GetBuffer(), 0, (int)msgsetStream.Length,
compressed.GetBuffer(), 0);
compressed.SetLength(size);
}
#endif
}
var m = new Message
{
Value = compressed,
TimeStamp = Timestamp.Now
};
Basics.WriteSizeInBytes(stream, m,
new SerializationInfo
{
Serializers = SerializationConfig.ByteArraySerializers,
CompressionCodec = info.CompressionCodec,
MessageVersion = info.MessageVersion
}, SerializeMessageWithCodec);
}
}
}
else
{
SerializeMessagesUncompressed(stream, messages, info.Serializers, info.MessageVersion);
}
}
/// <summary>
/// Compresses the remainder of <paramref name="input"/>, writing the compressed data to
/// <paramref name="output"/>.
/// </summary>
/// <param name="input"> The input stream.</param>
/// <param name="output">The output stream.</param>
public void Compress(Stream input, Stream output)
{
#if NET452 || NETSTANDARD2_0
var uncompressedSize = (int)(input.Length - input.Position);
var uncompressedBytes = new byte[uncompressedSize]; // does not include uncompressed message headers
input.ReadBytes(uncompressedBytes, offset: 0, count: uncompressedSize, CancellationToken.None);
var maxCompressedSize = SnappyCodec.GetMaxCompressedLength(uncompressedSize);
var compressedBytes = new byte[maxCompressedSize];
var compressedSize = SnappyCodec.Compress(uncompressedBytes, 0, uncompressedSize, compressedBytes, 0);
output.Write(compressedBytes, 0, compressedSize);
#else
throw new NotSupportedException();
#endif
}
/// <summary>
/// Compresses the remainder of <paramref name="input"/>, writing the compressed data to
/// <paramref name="output"/>.
/// </summary>
/// <param name="input"> The input stream.</param>
/// <param name="output">The output stream.</param>
public void Compress(Stream input, Stream output)
{
var uncompressedSize = (int)(input.Length - input.Position);
var uncompressedBytes = new byte[uncompressedSize]; // does not include uncompressed message headers
input.ReadBytes(uncompressedBytes, offset: 0, count: uncompressedSize, CancellationToken.None);
var maxCompressedSize = SnappyCodec.GetMaxCompressedLength(uncompressedSize);
var compressedBytes = new byte[maxCompressedSize];
var compressedSize = SnappyCodec.Compress(
input: uncompressedBytes,
inputOffset: 0,
inputLength: uncompressedSize,
output: compressedBytes,
outputOffset: 0,
outputLength: compressedBytes.Length); // output.Length - outputOffset
output.Write(compressedBytes, 0, compressedSize);
}
public void Compression(string name)
{
Benchmark.Run("Compressing", name, benchmark =>
{
var output = new byte[SnappyCodec.GetMaxCompressedLength(benchmark.Input.Length)];
int length = 0;
benchmark.Stopwatch.Start();
for (int i = 0; i < benchmark.Iterations; ++i)
{
length = SnappyCodec.Compress(benchmark.Input, 0, benchmark.Input.Length, output, 0);
}
benchmark.Stopwatch.Stop();
var roundtrip = new byte[benchmark.Input.Length];
var roundtripLength = SnappyCodec.Uncompress(output, 0, length, roundtrip, 0);
CollectionAssert.AreEqual(benchmark.Input, roundtrip.Take(roundtripLength));
benchmark.Note = String.Format(" ({0:0.00 %})", length / (double)benchmark.Input.Length);
});
}
/// <summary>
/// </summary>
/// <param name="Data"></param>
/// <returns></returns>
public static bool CompressInPlace(byte[] Data, long DataOffset, long DataLength, out long ResultLength)
{
byte[] Buffer = CompressBuffer.Value;
#if LOG_COMPRESS_RATIO
Stopwatch watch = new Stopwatch();
watch.Start();
#endif
int MaxBufferSize = SnappyCodec.GetMaxCompressedLength((int)DataLength);
if (Buffer.Length < MaxBufferSize)
{
Buffer = new byte[MaxBufferSize];
CompressBuffer.Value = Buffer;
}
ResultLength = SnappyCodec.Compress(Data, (int)DataOffset, (int)DataLength, Buffer, 0);
if (ResultLength < DataLength)
{
Array.Copy(Buffer, 0, Data, (int)DataOffset, ResultLength);
#if LOG_COMPRESS_RATIO
watch.Stop();
float elapsed = ((float)watch.ElapsedTicks / (float)Stopwatch.Frequency) * 1000.0f;
CompressedBytes += ResultLength;
UncompressedBytes += DataLength;
float Reduction = ((float)CompressedBytes / (float)UncompressedBytes) * 100.0f;
Console.WriteLine("CompressInPlace: ratio={0} elapsed={1}ms compressed={2} uncompressed={3} saved={4}",
Reduction,
elapsed,
StringUtils.FormatAsSize(CompressedBytes),
StringUtils.FormatAsSize(UncompressedBytes),
StringUtils.FormatAsSize(UncompressedBytes - CompressedBytes));
#endif
return(true);
}
else
{
return(false);
}
}
protected override void Encode(IChannelHandlerContext context, IByteBuffer input, IByteBuffer output)
{
byte packetType = input.ReadByte();
output.EnsureWritable(1, true);
output.WriteByte(packetType);
output.EnsureWritable(SnappyCodec.GetMaxCompressedLength(input.ReadableBytes), true);
if (_logger.IsTrace)
{
_logger.Trace($"Compressing with Snappy a message of length {input.ReadableBytes}");
}
int length = SnappyCodec.Compress(
input.Array,
input.ArrayOffset + input.ReaderIndex,
input.ReadableBytes,
output.Array,
output.ArrayOffset + output.WriterIndex);
input.SetReaderIndex(input.ReaderIndex + input.ReadableBytes);
output.SetWriterIndex(output.WriterIndex + length);
}
/// <summary>
/// Compress a given stream using a given compression codec
/// </summary>
/// <param name="uncompressedStream"> The initial stream we want to compress</param>
/// <param name="compressedStream"> The stream that want to put the compressed data in (should be empty before calling the method).</param>
/// <param name="compression"> The compression we want to use.</param>
/// <returns></returns>
internal static void CompressStream(ReusableMemoryStream uncompressedStream,
ReusableMemoryStream compressedStream, CompressionCodec compression)
{
if (compression == CompressionCodec.None)
{
throw new ArgumentException("Compress a stream only when you want compression.");
}
switch (compression)
{
case CompressionCodec.Gzip:
using (var gzip = new GZipStream(compressedStream, CompressionMode.Compress, true))
{
uncompressedStream.WriteTo(gzip);
}
break;
case CompressionCodec.Lz4:
KafkaLz4.Compress(compressedStream, uncompressedStream.GetBuffer(),
(int)uncompressedStream.Length);
break;
case CompressionCodec.Snappy:
#if NETSTANDARD1_3
throw new NotImplementedException();
#else
compressedStream.SetLength(SnappyCodec.GetMaxCompressedLength((int)uncompressedStream.Length));
{
int size = SnappyCodec.Compress(uncompressedStream.GetBuffer(), 0, (int)uncompressedStream.Length,
compressedStream.GetBuffer(), 0);
compressedStream.SetLength(size);
}
#endif
break;
}
}
/// <summary>
/// Allocate buffers of size, recommended for kafka usage. CompressedBuffer will be allocated of enough size, which will not require reallocation.
/// </summary>
/// <param name="uncompressedBuffer"></param>
/// <param name="compressedBuffer"></param>
public static void AllocateBuffers(out byte[] uncompressedBuffer, out byte[] compressedBuffer)
{
uncompressedBuffer = new byte[32 * 1024]; // 32K is default buffer size in xerces
compressedBuffer = new byte[SnappyCodec.GetMaxCompressedLength(uncompressedBuffer.Length)];
}
