internal void TestWriteToWithMessageSet(MessageSet set) { // do the write twice to ensure the message set is restored to its orginal state for (var i = 0; i < 1; i++) { var stream = ByteBuffer.Allocate(1024); var written = set.WriteTo(stream, 0, 1024); stream.SetLength(written); Assert.Equal(set.SizeInBytes, written); stream.Position = 0; var newSet = new ByteBufferMessageSet(stream); Assert.Equal(TestUtil.IteratorToArray(set.Iterator()), TestUtil.IteratorToArray(newSet.Iterator())); } }
public static Message Compress(IEnumerable <Message> messages, CompressionCodecs compressionCodec) { switch (compressionCodec) { case CompressionCodecs.DefaultCompressionCodec: case CompressionCodecs.GZIPCompressionCodec: using (MemoryStream outputStream = new MemoryStream()) { using (GZipStream gZipStream = new GZipStream(outputStream, CompressionMode.Compress)) { if (Logger.IsDebugEnabled) { Logger.DebugFormat( CultureInfo.CurrentCulture, "Allocating BufferedMessageSet of size = {0}", MessageSet.GetMessageSetSize(messages)); } var bufferedMessageSet = new BufferedMessageSet(messages); using (MemoryStream inputStream = new MemoryStream(bufferedMessageSet.SetSize)) { bufferedMessageSet.WriteTo(inputStream); inputStream.Position = 0; try { gZipStream.Write(inputStream.ToArray(), 0, inputStream.ToArray().Length); gZipStream.Close(); } catch (IOException ex) { Logger.Error("Error while writing to the GZIP stream", ex); throw; } } Message oneCompressedMessage = new Message(outputStream.ToArray(), compressionCodec); return(oneCompressedMessage); } } default: throw new UnknownCodecException(String.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}", compressionCodec)); } }
public FetchResponsePartitionData(short error = ErrorMapping.NoError, long hw = -1, MessageSet messages = null) { this.Error = error; this.Hw = hw; this.Messages = messages; }
public static Message Compress(IEnumerable <Message> messages, CompressionCodecs compressionCodec, int partition) { switch (compressionCodec) { case CompressionCodecs.DefaultCompressionCodec: case CompressionCodecs.GZIPCompressionCodec: using (var outputStream = new MemoryStream()) { using (var gZipStream = new GZipStream(outputStream, CompressionMode.Compress)) { //if (Logger.IsDebugEnabled) { Logger.DebugFormat( "Allocating BufferedMessageSet of size = {0}", MessageSet.GetMessageSetSize(messages)); } var bufferedMessageSet = new BufferedMessageSet(messages, partition); using (var inputStream = new MemoryStream(bufferedMessageSet.SetSize)) { bufferedMessageSet.WriteTo(inputStream); inputStream.Position = 0; try { gZipStream.Write(inputStream.ToArray(), 0, inputStream.ToArray().Length); gZipStream.Close(); } catch (IOException ex) { Logger.ErrorFormat("Error while writing to the GZIP stream {0}", ex.FormatException()); throw; } } var oneCompressedMessage = new Message(outputStream.ToArray(), compressionCodec) { PartitionId = partition }; return(oneCompressedMessage); } } case CompressionCodecs.SnappyCompressionCodec: Logger.DebugFormat( "Allocating BufferedMessageSet of size = {0}", MessageSet.GetMessageSetSize(messages)); var messageSet = new BufferedMessageSet(messages, partition); using (var inputStream = new MemoryStream(messageSet.SetSize)) { messageSet.WriteTo(inputStream); inputStream.Position = 0; try { return(new Message(SnappyHelper.Compress(inputStream.GetBuffer()), compressionCodec) { PartitionId = partition }); } catch (Exception ex) { Logger.ErrorFormat("Error while writing to the Snappy stream {0}", ex.FormatException()); throw; } } default: throw new UnknownCodecException(string.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}", compressionCodec)); } }