public static BufferedMessageSet Decompress(Message message, int partition) { switch (message.CompressionCodec) { case CompressionCodecs.DefaultCompressionCodec: case CompressionCodecs.GZIPCompressionCodec: var inputBytes = message.Payload; using (var outputStream = new MemoryStream()) { using (var inputStream = new MemoryStream(inputBytes)) { using (var gzipInputStream = new GZipStream(inputStream, CompressionMode.Decompress)) { try { gzipInputStream.CopyTo(outputStream); gzipInputStream.Close(); } catch (IOException ex) { Logger.InfoFormat("Error while reading from the GZIP input stream: {0}", ex.FormatException()); throw; } } } outputStream.Position = 0; using (var reader = new KafkaBinaryReader(outputStream)) { return(BufferedMessageSet.ParseFrom(reader, (int)outputStream.Length, partition)); } } case CompressionCodecs.SnappyCompressionCodec: try { using (var stream = new MemoryStream(SnappyHelper.Decompress(message.Payload))) { using (var reader = new KafkaBinaryReader(stream)) { return(BufferedMessageSet.ParseFrom(reader, (int)stream.Length, partition)); } } } catch (Exception ex) { Logger.ErrorFormat("Error while reading from the Snappy input stream {0}", ex.FormatException()); throw; } default: throw new UnknownCodecException(string.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}", message.CompressionCodec)); } }
public static Message Compress(IEnumerable <Message> messages, CompressionCodecs compressionCodec, int partition) { switch (compressionCodec) { case CompressionCodecs.DefaultCompressionCodec: case CompressionCodecs.GZIPCompressionCodec: using (var outputStream = new MemoryStream()) { using (var gZipStream = new GZipStream(outputStream, CompressionMode.Compress)) { if (Logger.IsDebugEnabled) { Logger.DebugFormat( "Allocating BufferedMessageSet of size = {0}", MessageSet.GetMessageSetSize(messages)); } var bufferedMessageSet = new BufferedMessageSet(messages, partition); using (var inputStream = new MemoryStream(bufferedMessageSet.SetSize)) { bufferedMessageSet.WriteTo(inputStream); inputStream.Position = 0; try { gZipStream.Write(inputStream.ToArray(), 0, inputStream.ToArray().Length); gZipStream.Close(); } catch (IOException ex) { Logger.ErrorFormat("Error while writing to the GZIP stream {0}", ex.FormatException()); throw; } } Message oneCompressedMessage = new Message(outputStream.ToArray(), compressionCodec) { PartitionId = partition }; return(oneCompressedMessage); } } case CompressionCodecs.SnappyCompressionCodec: Logger.DebugFormat( "Allocating BufferedMessageSet of size = {0}", MessageSet.GetMessageSetSize(messages)); var messageSet = new BufferedMessageSet(messages, partition); using (var inputStream = new MemoryStream(messageSet.SetSize)) { messageSet.WriteTo(inputStream); inputStream.Position = 0; try { return(new Message(SnappyHelper.Compress(inputStream.GetBuffer()), compressionCodec) { PartitionId = partition }); } catch (Exception ex) { Logger.ErrorFormat("Error while writing to the Snappy stream {0}", ex.FormatException()); throw; } } default: throw new UnknownCodecException(String.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}", compressionCodec)); } }