private static void _SerializeMessages(ReusableMemoryStream stream, IEnumerable <Message> messages, SerializationInfo info) { if (info.CompressionCodec != CompressionCodec.None) { stream.Write(Basics.Zero64, 0, 8); using (var msgsetStream = stream.Pool.Reserve()) { SerializeMessagesUncompressed(msgsetStream, messages, info.Serializers, info.MessageVersion); using (var compressed = stream.Pool.Reserve()) { if (info.CompressionCodec == CompressionCodec.Gzip) { using (var gzip = new GZipStream(compressed, CompressionMode.Compress, true)) { msgsetStream.WriteTo(gzip); } } else // Snappy { #if NETSTANDARD1_3 throw new NotImplementedException(); #else compressed.SetLength(SnappyCodec.GetMaxCompressedLength((int)msgsetStream.Length)); { int size = SnappyCodec.Compress(msgsetStream.GetBuffer(), 0, (int)msgsetStream.Length, compressed.GetBuffer(), 0); compressed.SetLength(size); } #endif } var m = new Message { Value = compressed, TimeStamp = Timestamp.Now }; Basics.WriteSizeInBytes(stream, m, new SerializationInfo { Serializers = SerializationConfig.ByteArraySerializers, CompressionCodec = info.CompressionCodec, MessageVersion = info.MessageVersion }, SerializeMessageWithCodec); } } } else { SerializeMessagesUncompressed(stream, messages, info.Serializers, info.MessageVersion); } }
/// <summary> /// Compresses the remainder of <paramref name="input"/>, writing the compressed data to /// <paramref name="output"/>. /// </summary> /// <param name="input"> The input stream.</param> /// <param name="output">The output stream.</param> public void Compress(Stream input, Stream output) { #if NET452 || NETSTANDARD2_0 var uncompressedSize = (int)(input.Length - input.Position); var uncompressedBytes = new byte[uncompressedSize]; // does not include uncompressed message headers input.ReadBytes(uncompressedBytes, offset: 0, count: uncompressedSize, CancellationToken.None); var maxCompressedSize = SnappyCodec.GetMaxCompressedLength(uncompressedSize); var compressedBytes = new byte[maxCompressedSize]; var compressedSize = SnappyCodec.Compress(uncompressedBytes, 0, uncompressedSize, compressedBytes, 0); output.Write(compressedBytes, 0, compressedSize); #else throw new NotSupportedException(); #endif }
/// <summary> /// Compresses the remainder of <paramref name="input"/>, writing the compressed data to /// <paramref name="output"/>. /// </summary> /// <param name="input"> The input stream.</param> /// <param name="output">The output stream.</param> public void Compress(Stream input, Stream output) { var uncompressedSize = (int)(input.Length - input.Position); var uncompressedBytes = new byte[uncompressedSize]; // does not include uncompressed message headers input.ReadBytes(uncompressedBytes, offset: 0, count: uncompressedSize, CancellationToken.None); var maxCompressedSize = SnappyCodec.GetMaxCompressedLength(uncompressedSize); var compressedBytes = new byte[maxCompressedSize]; var compressedSize = SnappyCodec.Compress( input: uncompressedBytes, inputOffset: 0, inputLength: uncompressedSize, output: compressedBytes, outputOffset: 0, outputLength: compressedBytes.Length); // output.Length - outputOffset output.Write(compressedBytes, 0, compressedSize); }
public void Compression(string name) { Benchmark.Run("Compressing", name, benchmark => { var output = new byte[SnappyCodec.GetMaxCompressedLength(benchmark.Input.Length)]; int length = 0; benchmark.Stopwatch.Start(); for (int i = 0; i < benchmark.Iterations; ++i) { length = SnappyCodec.Compress(benchmark.Input, 0, benchmark.Input.Length, output, 0); } benchmark.Stopwatch.Stop(); var roundtrip = new byte[benchmark.Input.Length]; var roundtripLength = SnappyCodec.Uncompress(output, 0, length, roundtrip, 0); CollectionAssert.AreEqual(benchmark.Input, roundtrip.Take(roundtripLength)); benchmark.Note = String.Format(" ({0:0.00 %})", length / (double)benchmark.Input.Length); }); }
/// <summary> /// </summary> /// <param name="Data"></param> /// <returns></returns> public static bool CompressInPlace(byte[] Data, long DataOffset, long DataLength, out long ResultLength) { byte[] Buffer = CompressBuffer.Value; #if LOG_COMPRESS_RATIO Stopwatch watch = new Stopwatch(); watch.Start(); #endif int MaxBufferSize = SnappyCodec.GetMaxCompressedLength((int)DataLength); if (Buffer.Length < MaxBufferSize) { Buffer = new byte[MaxBufferSize]; CompressBuffer.Value = Buffer; } ResultLength = SnappyCodec.Compress(Data, (int)DataOffset, (int)DataLength, Buffer, 0); if (ResultLength < DataLength) { Array.Copy(Buffer, 0, Data, (int)DataOffset, ResultLength); #if LOG_COMPRESS_RATIO watch.Stop(); float elapsed = ((float)watch.ElapsedTicks / (float)Stopwatch.Frequency) * 1000.0f; CompressedBytes += ResultLength; UncompressedBytes += DataLength; float Reduction = ((float)CompressedBytes / (float)UncompressedBytes) * 100.0f; Console.WriteLine("CompressInPlace: ratio={0} elapsed={1}ms compressed={2} uncompressed={3} saved={4}", Reduction, elapsed, StringUtils.FormatAsSize(CompressedBytes), StringUtils.FormatAsSize(UncompressedBytes), StringUtils.FormatAsSize(UncompressedBytes - CompressedBytes)); #endif return(true); } else { return(false); } }
protected override void Encode(IChannelHandlerContext context, IByteBuffer input, IByteBuffer output) { byte packetType = input.ReadByte(); output.EnsureWritable(1, true); output.WriteByte(packetType); output.EnsureWritable(SnappyCodec.GetMaxCompressedLength(input.ReadableBytes), true); if (_logger.IsTrace) { _logger.Trace($"Compressing with Snappy a message of length {input.ReadableBytes}"); } int length = SnappyCodec.Compress( input.Array, input.ArrayOffset + input.ReaderIndex, input.ReadableBytes, output.Array, output.ArrayOffset + output.WriterIndex); input.SetReaderIndex(input.ReaderIndex + input.ReadableBytes); output.SetWriterIndex(output.WriterIndex + length); }
/// <summary> /// Compress a given stream using a given compression codec /// </summary> /// <param name="uncompressedStream"> The initial stream we want to compress</param> /// <param name="compressedStream"> The stream that want to put the compressed data in (should be empty before calling the method).</param> /// <param name="compression"> The compression we want to use.</param> /// <returns></returns> internal static void CompressStream(ReusableMemoryStream uncompressedStream, ReusableMemoryStream compressedStream, CompressionCodec compression) { if (compression == CompressionCodec.None) { throw new ArgumentException("Compress a stream only when you want compression."); } switch (compression) { case CompressionCodec.Gzip: using (var gzip = new GZipStream(compressedStream, CompressionMode.Compress, true)) { uncompressedStream.WriteTo(gzip); } break; case CompressionCodec.Lz4: KafkaLz4.Compress(compressedStream, uncompressedStream.GetBuffer(), (int)uncompressedStream.Length); break; case CompressionCodec.Snappy: #if NETSTANDARD1_3 throw new NotImplementedException(); #else compressedStream.SetLength(SnappyCodec.GetMaxCompressedLength((int)uncompressedStream.Length)); { int size = SnappyCodec.Compress(uncompressedStream.GetBuffer(), 0, (int)uncompressedStream.Length, compressedStream.GetBuffer(), 0); compressedStream.SetLength(size); } #endif break; } }
/// <summary> /// Allocate buffers of size, recommended for kafka usage. CompressedBuffer will be allocated of enough size, which will not require reallocation. /// </summary> /// <param name="uncompressedBuffer"></param> /// <param name="compressedBuffer"></param> public static void AllocateBuffers(out byte[] uncompressedBuffer, out byte[] compressedBuffer) { uncompressedBuffer = new byte[32 * 1024]; // 32K is default buffer size in xerces compressedBuffer = new byte[SnappyCodec.GetMaxCompressedLength(uncompressedBuffer.Length)]; }