internal void TestWriteToWithMessageSet(MessageSet set)
{
// do the write twice to ensure the message set is restored to its orginal state
for (var i = 0; i < 1; i++)
{
var stream = ByteBuffer.Allocate(1024);
var written = set.WriteTo(stream, 0, 1024);
stream.SetLength(written);
Assert.Equal(set.SizeInBytes, written);
stream.Position = 0;
var newSet = new ByteBufferMessageSet(stream);
Assert.Equal(TestUtil.IteratorToArray(set.Iterator()), TestUtil.IteratorToArray(newSet.Iterator()));
}
}
public static Message Compress(IEnumerable <Message> messages, CompressionCodecs compressionCodec)
{
switch (compressionCodec)
{
case CompressionCodecs.DefaultCompressionCodec:
case CompressionCodecs.GZIPCompressionCodec:
using (MemoryStream outputStream = new MemoryStream())
{
using (GZipStream gZipStream = new GZipStream(outputStream, CompressionMode.Compress))
{
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat(
CultureInfo.CurrentCulture,
"Allocating BufferedMessageSet of size = {0}",
MessageSet.GetMessageSetSize(messages));
}
var bufferedMessageSet = new BufferedMessageSet(messages);
using (MemoryStream inputStream = new MemoryStream(bufferedMessageSet.SetSize))
{
bufferedMessageSet.WriteTo(inputStream);
inputStream.Position = 0;
try
{
gZipStream.Write(inputStream.ToArray(), 0, inputStream.ToArray().Length);
gZipStream.Close();
}
catch (IOException ex)
{
Logger.Error("Error while writing to the GZIP stream", ex);
throw;
}
}
Message oneCompressedMessage = new Message(outputStream.ToArray(), compressionCodec);
return(oneCompressedMessage);
}
}
default:
throw new UnknownCodecException(String.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}", compressionCodec));
}
}
public FetchResponsePartitionData(short error = ErrorMapping.NoError, long hw = -1, MessageSet messages = null)
{
this.Error = error;
this.Hw = hw;
this.Messages = messages;
}
public static Message Compress(IEnumerable <Message> messages, CompressionCodecs compressionCodec, int partition)
{
switch (compressionCodec)
{
case CompressionCodecs.DefaultCompressionCodec:
case CompressionCodecs.GZIPCompressionCodec:
using (var outputStream = new MemoryStream())
{
using (var gZipStream = new GZipStream(outputStream, CompressionMode.Compress))
{
//if (Logger.IsDebugEnabled)
{
Logger.DebugFormat(
"Allocating BufferedMessageSet of size = {0}",
MessageSet.GetMessageSetSize(messages));
}
var bufferedMessageSet = new BufferedMessageSet(messages, partition);
using (var inputStream = new MemoryStream(bufferedMessageSet.SetSize))
{
bufferedMessageSet.WriteTo(inputStream);
inputStream.Position = 0;
try
{
gZipStream.Write(inputStream.ToArray(), 0, inputStream.ToArray().Length);
gZipStream.Close();
}
catch (IOException ex)
{
Logger.ErrorFormat("Error while writing to the GZIP stream {0}",
ex.FormatException());
throw;
}
}
var oneCompressedMessage = new Message(outputStream.ToArray(), compressionCodec)
{
PartitionId = partition
};
return(oneCompressedMessage);
}
}
case CompressionCodecs.SnappyCompressionCodec:
Logger.DebugFormat(
"Allocating BufferedMessageSet of size = {0}",
MessageSet.GetMessageSetSize(messages));
var messageSet = new BufferedMessageSet(messages, partition);
using (var inputStream = new MemoryStream(messageSet.SetSize))
{
messageSet.WriteTo(inputStream);
inputStream.Position = 0;
try
{
return(new Message(SnappyHelper.Compress(inputStream.GetBuffer()), compressionCodec)
{
PartitionId = partition
});
}
catch (Exception ex)
{
Logger.ErrorFormat("Error while writing to the Snappy stream {0}", ex.FormatException());
throw;
}
}
default:
throw new UnknownCodecException(string.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}",
compressionCodec));
}
}
