private static void Uncompress(ReusableMemoryStream uncompressed, byte[] body, int offset, int length, CompressionCodec codec) { try { if (codec == CompressionCodec.Snappy) { #if NET_CORE throw new NotImplementedException(); #else uncompressed.SetLength(SnappyCodec.GetUncompressedLength(body, offset, length)); SnappyCodec.Uncompress(body, offset, length, uncompressed.GetBuffer(), 0); #endif } else if (codec == CompressionCodec.Gzip) { using (var compressed = new MemoryStream(body, offset, length, false)) { using (var zipped = new GZipStream(compressed, CompressionMode.Decompress)) { using (var tmp = uncompressed.Pool.Reserve()) { zipped.ReusableCopyTo(uncompressed, tmp); } } } } else // compression == CompressionCodec.Lz4 { KafkaLz4.Uncompress(uncompressed, body, offset); } uncompressed.Position = 0; } catch (Exception ex) { throw new UncompressException("Invalid compressed data.", codec, ex); } }
/// <summary> /// Compress a given stream using a given compression codec /// </summary> /// <param name="uncompressedStream"> The initial stream we want to compress</param> /// <param name="compressedStream"> The stream that want to put the compressed data in (should be empty before calling the method).</param> /// <param name="compression"> The compression we want to use.</param> /// <returns></returns> internal static void CompressStream(ReusableMemoryStream uncompressedStream, ReusableMemoryStream compressedStream, CompressionCodec compression) { if (compression == CompressionCodec.None) { throw new ArgumentException("Compress a stream only when you want compression."); } switch (compression) { case CompressionCodec.Gzip: using (var gzip = new GZipStream(compressedStream, CompressionMode.Compress, true)) { uncompressedStream.WriteTo(gzip); } break; case CompressionCodec.Lz4: KafkaLz4.Compress(compressedStream, uncompressedStream.GetBuffer(), (int)uncompressedStream.Length); break; case CompressionCodec.Snappy: #if NETSTANDARD1_3 throw new NotImplementedException(); #else compressedStream.SetLength(SnappyCodec.GetMaxCompressedLength((int)uncompressedStream.Length)); { int size = SnappyCodec.Compress(uncompressedStream.GetBuffer(), 0, (int)uncompressedStream.Length, compressedStream.GetBuffer(), 0); compressedStream.SetLength(size); } #endif break; } }
private static void _SerializeMessages(ReusableMemoryStream stream, IEnumerable <Message> messages, SerializationInfo info) { if (info.CompressionCodec != CompressionCodec.None) { stream.Write(Basics.Zero64, 0, 8); using (var msgsetStream = stream.Pool.Reserve()) { SerializeMessagesUncompressed(msgsetStream, messages, info.Serializers, info.MessageVersion); using (var compressed = stream.Pool.Reserve()) { switch (info.CompressionCodec) { case CompressionCodec.Gzip: using (var gzip = new GZipStream(compressed, CompressionMode.Compress, true)) { msgsetStream.WriteTo(gzip); } break; case CompressionCodec.Lz4: KafkaLz4.Compress(compressed, msgsetStream.GetBuffer(), (int)msgsetStream.Length); break; case CompressionCodec.Snappy: { #if NET_CORE throw new NotImplementedException(); #else compressed.SetLength(SnappyCodec.GetMaxCompressedLength((int)msgsetStream.Length)); { int size = SnappyCodec.Compress(msgsetStream.GetBuffer(), 0, (int)msgsetStream.Length, compressed.GetBuffer(), 0); compressed.SetLength(size); } #endif } break; } var m = new Message { Value = compressed, TimeStamp = Timestamp.Now }; Basics.WriteSizeInBytes(stream, m, new SerializationInfo { Serializers = SerializationConfig.ByteArraySerializers, CompressionCodec = info.CompressionCodec, MessageVersion = info.MessageVersion }, SerializeMessageWithCodec); } } } else { SerializeMessagesUncompressed(stream, messages, info.Serializers, info.MessageVersion); } }