public void BufferedMessageSetWriteToValidSequence()
{
byte[] messageBytes = new byte[] { 1, 2, 3, 4, 5 };
Message msg1 = new Message(messageBytes);
Message msg2 = new Message(messageBytes);
MessageSet messageSet = new BufferedMessageSet(new List<Message>() { msg1, msg2 });
MemoryStream ms = new MemoryStream();
messageSet.WriteTo(ms);
////first message
byte[] messageLength = new byte[MessageLengthPartLength];
Array.Copy(ms.ToArray(), MessageLengthPartOffset, messageLength, 0, MessageLengthPartLength);
if (BitConverter.IsLittleEndian)
{
Array.Reverse(messageLength);
}
Assert.AreEqual(MagicNumberPartLength + AttributesPartLength + ChecksumPartLength + messageBytes.Length, BitConverter.ToInt32(messageLength, 0));
Assert.AreEqual(1, ms.ToArray()[MagicNumberPartOffset]); // default magic number should be 1
byte[] checksumPart = new byte[ChecksumPartLength];
Array.Copy(ms.ToArray(), ChecksumPartOffset, checksumPart, 0, ChecksumPartLength);
Assert.AreEqual(Crc32Hasher.Compute(messageBytes), checksumPart);
byte[] dataPart = new byte[messageBytes.Length];
Array.Copy(ms.ToArray(), DataPartOffset, dataPart, 0, messageBytes.Length);
Assert.AreEqual(messageBytes, dataPart);
////second message
int secondMessageOffset = MessageLengthPartLength + MagicNumberPartLength + AttributesPartLength + ChecksumPartLength +
messageBytes.Length;
messageLength = new byte[MessageLengthPartLength];
Array.Copy(ms.ToArray(), secondMessageOffset + MessageLengthPartOffset, messageLength, 0, MessageLengthPartLength);
if (BitConverter.IsLittleEndian)
{
Array.Reverse(messageLength);
}
Assert.AreEqual(MagicNumberPartLength + AttributesPartLength + ChecksumPartLength + messageBytes.Length, BitConverter.ToInt32(messageLength, 0));
Assert.AreEqual(1, ms.ToArray()[secondMessageOffset + MagicNumberPartOffset]); // default magic number should be 1
checksumPart = new byte[ChecksumPartLength];
Array.Copy(ms.ToArray(), secondMessageOffset + ChecksumPartOffset, checksumPart, 0, ChecksumPartLength);
Assert.AreEqual(Crc32Hasher.Compute(messageBytes), checksumPart);
dataPart = new byte[messageBytes.Length];
Array.Copy(ms.ToArray(), secondMessageOffset + DataPartOffset, dataPart, 0, messageBytes.Length);
Assert.AreEqual(messageBytes, dataPart);
}
public void ShouldAbleToWriteMessageSetWithExtraBytes()
{
var stream = new MemoryStream();
var writer = new KafkaBinaryWriter(stream);
var msg1 = new Message(new byte[101]) {Offset = 0};
var msg2 = new Message(new byte[102]) {Offset = 1};
var set = new BufferedMessageSet(new List<Message>() {msg1, msg2}, 0);
set.WriteTo(writer);
writer.Write(new byte[10]); // less than offset and size
var size = (int) stream.Position;
stream.Seek(0, SeekOrigin.Begin);
var reader = new KafkaBinaryReader(stream);
var newSet = BufferedMessageSet.ParseFrom(reader, size, 0);
var messages = newSet.Messages.ToList();
messages.Count().Should().Be(2);
messages[0].Payload.Count().Should().Be(101);
messages[1].Payload.Count().Should().Be(102);
}
public static Message Compress(IEnumerable <Message> messages, CompressionCodecs compressionCodec)
{
switch (compressionCodec)
{
case CompressionCodecs.DefaultCompressionCodec:
case CompressionCodecs.GZIPCompressionCodec:
using (MemoryStream outputStream = new MemoryStream())
{
using (GZipStream gZipStream = new GZipStream(outputStream, CompressionMode.Compress))
{
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat(
CultureInfo.CurrentCulture,
"Allocating BufferedMessageSet of size = {0}",
MessageSet.GetMessageSetSize(messages));
}
var bufferedMessageSet = new BufferedMessageSet(messages);
using (MemoryStream inputStream = new MemoryStream(bufferedMessageSet.SetSize))
{
bufferedMessageSet.WriteTo(inputStream);
inputStream.Position = 0;
try
{
gZipStream.Write(inputStream.ToArray(), 0, inputStream.ToArray().Length);
gZipStream.Close();
}
catch (IOException ex)
{
Logger.Error("Error while writing to the GZIP stream", ex);
throw;
}
}
Message oneCompressedMessage = new Message(outputStream.ToArray(), compressionCodec);
return(oneCompressedMessage);
}
}
default:
throw new UnknownCodecException(String.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}", compressionCodec));
}
}
public static Message Compress(IEnumerable<Message> messages, CompressionCodecs compressionCodec, int partition)
{
switch (compressionCodec)
{
case CompressionCodecs.DefaultCompressionCodec:
case CompressionCodecs.GZIPCompressionCodec:
using (var outputStream = new MemoryStream())
{
using (var gZipStream = new GZipStream(outputStream, CompressionMode.Compress))
{
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat(
"Allocating BufferedMessageSet of size = {0}",
MessageSet.GetMessageSetSize(messages));
}
var bufferedMessageSet = new BufferedMessageSet(messages, partition);
using (var inputStream = new MemoryStream(bufferedMessageSet.SetSize))
{
bufferedMessageSet.WriteTo(inputStream);
inputStream.Position = 0;
try
{
gZipStream.Write(inputStream.ToArray(), 0, inputStream.ToArray().Length);
gZipStream.Close();
}
catch (IOException ex)
{
Logger.ErrorFormat("Error while writing to the GZIP stream {0}", ex.FormatException());
throw;
}
}
Message oneCompressedMessage = new Message(outputStream.ToArray(), compressionCodec)
{
PartitionId = partition
};
return oneCompressedMessage;
}
}
case CompressionCodecs.SnappyCompressionCodec:
Logger.DebugFormat(
"Allocating BufferedMessageSet of size = {0}",
MessageSet.GetMessageSetSize(messages));
var messageSet = new BufferedMessageSet(messages, partition);
using (var inputStream = new MemoryStream(messageSet.SetSize))
{
messageSet.WriteTo(inputStream);
inputStream.Position = 0;
try
{
return new Message(SnappyHelper.Compress(inputStream.GetBuffer()), compressionCodec)
{
PartitionId = partition
};
}
catch (Exception ex)
{
Logger.ErrorFormat("Error while writing to the Snappy stream {0}", ex.FormatException());
throw;
}
}
default:
throw new UnknownCodecException(String.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}", compressionCodec));
}
}
public static Message Compress(IEnumerable <Message> messages, CompressionCodecs compressionCodec, int partition)
{
switch (compressionCodec)
{
case CompressionCodecs.DefaultCompressionCodec:
case CompressionCodecs.GZIPCompressionCodec:
using (var outputStream = new MemoryStream())
{
using (var gZipStream = new GZipStream(outputStream, CompressionMode.Compress))
{
//if (Logger.IsDebugEnabled)
{
Logger.DebugFormat(
"Allocating BufferedMessageSet of size = {0}",
MessageSet.GetMessageSetSize(messages));
}
var bufferedMessageSet = new BufferedMessageSet(messages, partition);
using (var inputStream = new MemoryStream(bufferedMessageSet.SetSize))
{
bufferedMessageSet.WriteTo(inputStream);
inputStream.Position = 0;
try
{
gZipStream.Write(inputStream.ToArray(), 0, inputStream.ToArray().Length);
gZipStream.Close();
}
catch (IOException ex)
{
Logger.ErrorFormat("Error while writing to the GZIP stream {0}",
ex.FormatException());
throw;
}
}
var oneCompressedMessage = new Message(outputStream.ToArray(), compressionCodec)
{
PartitionId = partition
};
return(oneCompressedMessage);
}
}
case CompressionCodecs.SnappyCompressionCodec:
Logger.DebugFormat(
"Allocating BufferedMessageSet of size = {0}",
MessageSet.GetMessageSetSize(messages));
var messageSet = new BufferedMessageSet(messages, partition);
using (var inputStream = new MemoryStream(messageSet.SetSize))
{
messageSet.WriteTo(inputStream);
inputStream.Position = 0;
try
{
return(new Message(SnappyHelper.Compress(inputStream.GetBuffer()), compressionCodec)
{
PartitionId = partition
});
}
catch (Exception ex)
{
Logger.ErrorFormat("Error while writing to the Snappy stream {0}", ex.FormatException());
throw;
}
}
default:
throw new UnknownCodecException(string.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}",
compressionCodec));
}
}
