internal static OffsetResponse DeserializeOffsetResponse(byte[] buff, int count)
{
var stream = new MemoryStream(buff, 0, count);
stream.Position += 4; // skip body
var response = new OffsetResponse();
var len = BigEndianConverter.ReadInt32(stream);
// TODO: make sure connection is closed
if (len != 1)
{
throw new BrokerException("Invalid message format");
}
response.TopicName = ReadString(stream);
len = BigEndianConverter.ReadInt32(stream);
response.Partitions = new OffsetResponse.PartitionOffsetData[len];
for (int i = 0; i < len; i++)
{
var part = response.Partitions[i] = new OffsetResponse.PartitionOffsetData();
part.Partition = BigEndianConverter.ReadInt32(stream);
part.ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream);
var len2 = BigEndianConverter.ReadInt32(stream);
part.Offsets = new long[len2];
for (int j = 0; j < len2; j++)
{
part.Offsets[j] = BigEndianConverter.ReadInt64(stream);
}
}
return(response);
}
public void Deserialize(ReusableMemoryStream stream, object extra)
{
Partition = BigEndianConverter.ReadInt32(stream);
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream);
HighWatermarkOffset = BigEndianConverter.ReadInt64(stream);
Messages = DeserializeMessageSet(stream, extra as Deserializers);
}
public void TestSerializeMessageSet()
{
using (var serialized = new ReusableMemoryStream(null))
{
var set = new PartitionData
{
Partition = 42,
CompressionCodec = CompressionCodec.None,
Messages = new[]
{
new Message {
Key = Key, Value = Value
},
new Message {
Key = Key, Value = Value
}
}
};
set.Serialize(serialized, SerializationConfig.ByteArraySerializers);
serialized.Position = 0;
Assert.AreEqual(42, BigEndianConverter.ReadInt32(serialized)); // Partition
Assert.AreEqual(serialized.Length - 4 - 4, BigEndianConverter.ReadInt32(serialized)); // MessageSet size
Assert.AreEqual(0, BigEndianConverter.ReadInt64(serialized)); // First message offset
int firstMsgSize = BigEndianConverter.ReadInt32(serialized);
serialized.Position += firstMsgSize;
Assert.AreEqual(0, BigEndianConverter.ReadInt64(serialized)); // First message offset
int secondMsgSize = BigEndianConverter.ReadInt32(serialized);
serialized.Position += secondMsgSize;
Assert.AreEqual(serialized.Length, serialized.Position);
}
}
public void Deserialize(ReusableMemoryStream stream, object _, Basics.ApiVersion __)
{
Partition = BigEndianConverter.ReadInt32(stream);
Offset = BigEndianConverter.ReadInt64(stream);
Metadata = Basics.DeserializeString(stream);
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream);
}
/// <summary>
/// FetchResponse => [TopicName [Partition ErrorCode HighwaterMarkOffset MessageSetSize MessageSet]]
/// TopicName => string
/// Partition => int32
/// ErrorCode => int16
/// HighwaterMarkOffset => int64
/// MessageSetSize => int32
/// </summary>
/// <param name="buff"></param>
/// <returns></returns>
public static FetchResponse DeserializeFetchResponse(byte[] buff)
{
var stream = new MemoryStream(buff);
stream.Position += 4; // skip body
var response = new FetchResponse();
var len = BigEndianConverter.ReadInt32(stream);
response.Topics = new FetchResponse.TopicFetchData[len];
for (int t = 0; t < len; t++)
{
var topic = new FetchResponse.TopicFetchData();
response.Topics[t] = topic;
topic.Topic = ReadString(stream);
var len2 = BigEndianConverter.ReadInt32(stream);
topic.Partitions = new FetchResponse.PartitionFetchData[len2];
for (int i = 0; i < len2; i++)
{
topic.Partitions[i] = new FetchResponse.PartitionFetchData
{
Partition = BigEndianConverter.ReadInt32(stream),
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream),
HighWatermarkOffset = BigEndianConverter.ReadInt64(stream),
Messages = ReadMessageSet(stream).ToArray()
}
}
;
}
return(response);
}
internal CompressionCodec Compression; // Only used in test for serializing
#region Deserialization
public void Deserialize(ReusableMemoryStream stream, object extra, Basics.ApiVersion version)
{
Partition = BigEndianConverter.ReadInt32(stream);
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream);
HighWatermarkOffset = BigEndianConverter.ReadInt64(stream);
if (version >= Basics.ApiVersion.V4)
{
LastStableOffset = BigEndianConverter.ReadInt64(stream);
AbortedTransactions = Basics.DeserializeArray <AbortedTransaction>(stream);
}
if (version >= Basics.ApiVersion.V5)
{
LogStartOffset = BigEndianConverter.ReadInt64(stream);
}
try
{
Messages = version >= Basics.ApiVersion.V3
? DeserializeRecordBatch(stream, extra as Deserializers)
: DeserializeMessageSet(stream, extra as Deserializers);
}
catch (ProtocolException pEx)
{
pEx.Partition = Partition;
throw;
}
}
+ 4; // batchLength
/// <remarks>
/// From the official protocol documentation available at
/// https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-FetchAPI
/// "As an optimization the server is allowed to return a partial message at the end of the message set. Clients should handle this case."
/// If the end of the RecordBatch exceeds the length of the whole response (= endOfAllBatches), we should discard this RecordBatch.
/// </remarks>
public static RecordBatch Deserialize(ReusableMemoryStream input, Deserializers deserializers, long endOfAllBatches)
{
var recordBatch = new RecordBatch();
if (input.Position + BytesNecessaryToGetLength > endOfAllBatches)
{
throw new ProtocolException(
$"Trying to read a batch record at {input.Position} and the end of all batches is {endOfAllBatches}."
+ $" There is not enough bytes remaining to even read the first fields...");
}
recordBatch.BaseOffset = BigEndianConverter.ReadInt64(input);
var batchLength = BigEndianConverter.ReadInt32(input);
var endOfBatch = input.Position + batchLength;
if (endOfAllBatches < endOfBatch)
{
// Partial message, CRCs won't match, return here so the CRC check doesn't throw
return(null);
}
recordBatch.PartitionLeaderEpoch = BigEndianConverter.ReadInt32(input);
var magic = input.ReadByte();
// Current magic value is 2
if ((uint)magic != 2)
{
throw new UnsupportedMagicByteVersion((byte)magic, "2");
}
var crc = (uint)BigEndianConverter.ReadInt32(input);
var afterCrcPosition = input.Position; // The crc is calculated starting from this position
Crc32.CheckCrcCastagnoli((int)crc, input, afterCrcPosition, endOfBatch - afterCrcPosition);
var attributes = BigEndianConverter.ReadInt16(input);
recordBatch.CompressionCodec = (CompressionCodec)(attributes & CompressionCodecMask);
recordBatch.IsTransactional = (attributes & TransactionalFlagMask) != 0;
recordBatch.IsControl = (attributes & ControlFlagMask) != 0;
recordBatch.TimestampType = (attributes & TimestampTypeMask) > 0
? TimestampType.LogAppendTime
: TimestampType.CreateTime;
var lastOffsetDelta = BigEndianConverter.ReadInt32(input);
var firstTimestamp = BigEndianConverter.ReadInt64(input);
var maxTimestamp = BigEndianConverter.ReadInt64(input);
recordBatch.ProducerId = BigEndianConverter.ReadInt64(input);
recordBatch.ProducerEpoch = BigEndianConverter.ReadInt16(input);
recordBatch.BaseSequence = BigEndianConverter.ReadInt32(input);
var numberOfRecords = BigEndianConverter.ReadInt32(input);
recordBatch.Records = DeserializeRecords(recordBatch, input, numberOfRecords, endOfBatch, firstTimestamp, deserializers);
return(recordBatch);
}
public void Deserialize(ReusableMemoryStream stream, object _, Basics.ApiVersion version)
{
Partition = BigEndianConverter.ReadInt32(stream);
Time = BigEndianConverter.ReadInt64(stream);
if (version == Basics.ApiVersion.V0)
{
MaxNumberOfOffsets = BigEndianConverter.ReadInt32(stream);
}
}
private static ProducerResponse.PartitionResponse DeserializePartitionResponse(MemoryStream stream)
{
return(new ProducerResponse.PartitionResponse
{
Partition = BigEndianConverter.ReadInt32(stream),
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream),
Offset = BigEndianConverter.ReadInt64(stream)
});
}
public void Deserialize(ReusableMemoryStream stream, object _, Basics.ApiVersion version)
{
Partition = BigEndianConverter.ReadInt32(stream);
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream);
Offset = BigEndianConverter.ReadInt64(stream);
if (version >= Basics.ApiVersion.V2)
{
Timestamp = BigEndianConverter.ReadInt64(stream);
}
}
// Used only in tests
public void Deserialize(ReusableMemoryStream stream, object _, Basics.ApiVersion version)
{
Partition = BigEndianConverter.ReadInt32(stream);
FetchOffset = BigEndianConverter.ReadInt64(stream);
MaxBytes = BigEndianConverter.ReadInt32(stream);
if (version >= Basics.ApiVersion.V5)
{
LogStartOffset = BigEndianConverter.ReadInt64(stream);
}
}
//N.B., MessageSets are not preceded by an int32 like other array elements in the protocol.
//
//MessageSet => [Offset MessageSize Message]
// Offset => int64
// MessageSize => int32
//
//Message => Crc MagicByte Attributes Key Value
// Crc => int32
// MagicByte => int8
// Attributes => int8
// Key => bytes
// Value => bytes
private static IEnumerable <Message> ReadMessageSet(MemoryStream stream)
{
// "As an optimization the server is allowed to return a partial message at the end of the message set.
// Clients should handle this case"
var messageSetSize = BigEndianConverter.ReadInt32(stream);
var remainingMessageSetBytes = messageSetSize;
while (remainingMessageSetBytes > 0)
{
// we need at least be able to read offset and messageSize
if (remainingMessageSetBytes < +8 + 4)
{
// not enough bytes left. This is a partial message. Skip to the end of the message set.
stream.Position += remainingMessageSetBytes;
yield break;
}
var offset = BigEndianConverter.ReadInt64(stream);
var messageSize = BigEndianConverter.ReadInt32(stream);
// we took 12 bytes there, check again that we have a full message.
remainingMessageSetBytes -= 8 + 4;
if (remainingMessageSetBytes < messageSize)
{
// not enough bytes left. This is a partial message. Skip to the end of the message set.
stream.Position += remainingMessageSetBytes;
yield break;
}
// Message
var crc = BigEndianConverter.ReadInt32(stream);
var crcPos = stream.Position;
var magic = stream.ReadByte();
var attributes = stream.ReadByte();
var msg = new Message();
msg.Key = ReadByteArray(stream);
msg.Value = ReadByteArray(stream);
msg.Offset = offset;
var pos = stream.Position;
var computedCrcArray = Crc32.Compute(stream, crcPos, pos - crcPos);
var computedCrc = BigEndianConverter.ToInt32(computedCrcArray);
if (computedCrc != crc)
{
throw new BrokerException(string.Format("Corrupt message: Crc does not match. Caclulated {0} but got {1}", computedCrc, crc));
}
yield return(msg);
// subtract messageSize of that message from remaining bytes
remainingMessageSetBytes -= messageSize;
}
}
public void Deserialize(ReusableMemoryStream stream, object extra, Basics.ApiVersion version)
{
Partition = BigEndianConverter.ReadInt32(stream);
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream);
HighWatermarkOffset = BigEndianConverter.ReadInt64(stream);
try
{
Messages = DeserializeMessageSet(stream, extra as Deserializers);
}
catch (ProtocolException pEx)
{
pEx.Partition = Partition;
throw;
}
}
public void Test000_BigEndianConverter()
{
const byte b0 = 123;
const byte b1 = 98;
const byte b2 = 0;
const byte b3 = 188;
const byte b4 = 23;
const byte b5 = 89;
const byte b6 = 101;
const byte b7 = 7;
using (var s = new ReusableMemoryStream(null))
{
var n1 = (short)((b0 << 8) | b1);
BigEndianConverter.Write(s, n1);
Assert.AreEqual(b0, s[0]);
Assert.AreEqual(b1, s[1]);
s.Position = 0;
Assert.AreEqual(n1, BigEndianConverter.ReadInt16(s));
var n2 = (int)((b0 << 24) | (b1 << 16) | (b2 << 8) | b3);
s.Position = 0;
BigEndianConverter.Write(s, n2);
Assert.AreEqual(b0, s[0]);
Assert.AreEqual(b1, s[1]);
Assert.AreEqual(b2, s[2]);
Assert.AreEqual(b3, s[3]);
s.Position = 0;
Assert.AreEqual(n2, BigEndianConverter.ReadInt32(s));
var n3 = (((long)b0 << 56) |
((long)b1 << 48 | ((long)b2 << 40) | ((long)b3 << 32) | ((long)b4 << 24) |
((long)b5 << 16) | ((long)b6 << 8) | b7));
s.Position = 0;
BigEndianConverter.Write(s, n3);
Assert.AreEqual(b0, s[0]);
Assert.AreEqual(b1, s[1]);
Assert.AreEqual(b2, s[2]);
Assert.AreEqual(b3, s[3]);
Assert.AreEqual(b4, s[4]);
Assert.AreEqual(b5, s[5]);
Assert.AreEqual(b6, s[6]);
Assert.AreEqual(b7, s[7]);
s.Position = 0;
Assert.AreEqual(n3, BigEndianConverter.ReadInt64(s));
}
}
public void Deserialize(ReusableMemoryStream stream, object noextra, Basics.ApiVersion version)
{
Partition = BigEndianConverter.ReadInt32(stream);
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream);
Timestamp = -1;
if (version > Basics.ApiVersion.V0)
{
Timestamp = BigEndianConverter.ReadInt64(stream);
Offsets = new long[1];
Offsets[0] = BigEndianConverter.ReadInt64(stream);
}
else
{
Offsets = Basics.DeserializeArray(stream, BigEndianConverter.ReadInt64);
}
}
public void TestSerializeMessageSetCompressed(CompressionCodec codec, byte attr)
{
using (var serialized = Pool.Reserve())
{
var set = new PartitionData
{
Partition = 42,
CompressionCodec = codec,
Messages = new[]
{
new Message {
Key = Key, Value = Value
},
new Message {
Key = Key, Value = Value
}
}
};
set.Serialize(serialized, SerializationConfig.ByteArraySerializers);
serialized.Position = 0;
Assert.AreEqual(42, BigEndianConverter.ReadInt32(serialized)); // Partition
Assert.AreEqual(serialized.Length - 4 - 4, BigEndianConverter.ReadInt32(serialized)); // MessageSet size
Assert.AreEqual(0, BigEndianConverter.ReadInt64(serialized)); // First message offset
int firstMsgSize = BigEndianConverter.ReadInt32(serialized);
serialized.Position += firstMsgSize;
Assert.AreEqual(serialized.Length, serialized.Position);
int msgPos = 4 + 4 + 8 + 4; // partition, msgset size, offset, msg size
int valuePos = msgPos + 4 + 1 + 1 + 4 + 4; // + crc, magic, attr, key size, value size
serialized.Position = msgPos;
serialized.Position += 5;
Assert.AreEqual(attr, serialized.ReadByte()); // attributes
Assert.AreEqual(-1, BigEndianConverter.ReadInt32(serialized)); // No key => size = -1
Assert.AreEqual(serialized.Length - valuePos, BigEndianConverter.ReadInt32(serialized)); // check rest of message length match
}
}
public void Deserialize(ReusableMemoryStream stream, object extra, Basics.ApiVersion version)
{
ProducerId = BigEndianConverter.ReadInt64(stream);
FirstOffset = BigEndianConverter.ReadInt64(stream);
}
// Deserialize a message set to a sequence of messages.
// This handles the "partial message allowed at end of message set" from Kafka brokers
// and compressed message sets (the method recursively calls itself in this case and
// flatten the result). The returned enumeration must be enumerated for deserialization
// effectiveley occuring.
private static IEnumerable <ResponseMessage> LazyDeserializeMessageSet(ReusableMemoryStream stream, int messageSetSize, Deserializers deserializers)
{
var remainingMessageSetBytes = messageSetSize;
while (remainingMessageSetBytes > 0)
{
const int offsetSize = 8;
const int msgsizeSize = 4;
if (remainingMessageSetBytes < offsetSize + msgsizeSize)
{
// This is a partial message => skip to the end of the message set.
// TODO: unit test this
stream.Position += remainingMessageSetBytes;
yield break;
}
var offset = BigEndianConverter.ReadInt64(stream);
var messageSize = BigEndianConverter.ReadInt32(stream);
remainingMessageSetBytes -= offsetSize + msgsizeSize;
if (remainingMessageSetBytes < messageSize)
{
// This is a partial message => skip to the end of the message set.
stream.Position += remainingMessageSetBytes;
yield break;
}
// Message body
var crc = BigEndianConverter.ReadInt32(stream);
var crcStartPos = stream.Position; // crc is computed from this position
var magic = stream.ReadByte();
if (magic != 0)
{
throw new UnsupportedMagicByteVersion((byte)magic);
}
var attributes = stream.ReadByte();
// Check for compression
var codec = (CompressionCodec)(attributes & 3); // Lowest 2 bits
if (codec == CompressionCodec.None)
{
var msg = new ResponseMessage
{
Offset = offset,
Message = new Message
{
Key = Basics.DeserializeByteArray(stream, deserializers.Item1),
Value = Basics.DeserializeByteArray(stream, deserializers.Item2)
}
};
CheckCrc(crc, stream, crcStartPos);
yield return(msg);
}
else
{
// Key is null, read/check/skip
if (BigEndianConverter.ReadInt32(stream) != -1)
{
throw new InvalidDataException("Compressed messages key should be null");
}
// Uncompress
var compressedLength = BigEndianConverter.ReadInt32(stream);
var dataPos = stream.Position;
stream.Position += compressedLength;
CheckCrc(crc, stream, crcStartPos);
using (var uncompressedStream = stream.Pool.Reserve())
{
Uncompress(uncompressedStream, stream.GetBuffer(), (int)dataPos, compressedLength, codec);
// Deserialize recursively
foreach (var m in LazyDeserializeMessageSet(uncompressedStream, (int)uncompressedStream.Length, deserializers))
{
// Flatten
yield return(m);
}
}
}
remainingMessageSetBytes -= messageSize;
}
}
// Used only in tests
public void Deserialize(ReusableMemoryStream stream, object noextra)
{
Partition = BigEndianConverter.ReadInt32(stream);
FetchOffset = BigEndianConverter.ReadInt64(stream);
MaxBytes = BigEndianConverter.ReadInt32(stream);
}
// Deserialize a message set to a sequence of messages.
// This handles the "partial message allowed at end of message set" from Kafka brokers
// and compressed message sets (the method recursively calls itself in this case and
// flatten the result). The returned enumeration must be enumerated for deserialization
// effectiveley occuring.
//
// A message set can contain a mix of v0 and v1 messages.
// In the case of compressed messages, offsets are returned differently by brokers.
// Messages inside compressed message v0 will have absolute offsets assigned.
// Messages inside compressed message v1 will have relative offset assigned, starting
// from 0. The wrapping compressed message itself is assigned the absolute offset of the last
// message in the set. That means in this case we can only assign offsets after having decompressing
// all messages. Lazy deserialization won't be so lazy anymore...
private static IEnumerable <ResponseMessage> LazyDeserializeMessageSet(ReusableMemoryStream stream, int messageSetSize, Deserializers deserializers)
{
var remainingMessageSetBytes = messageSetSize;
while (remainingMessageSetBytes > 0)
{
const int offsetSize = 8;
const int msgsizeSize = 4;
if (remainingMessageSetBytes < offsetSize + msgsizeSize)
{
// This is a partial message => skip to the end of the message set.
// TODO: unit test this
stream.Position += remainingMessageSetBytes;
yield break;
}
var offset = BigEndianConverter.ReadInt64(stream);
var messageSize = BigEndianConverter.ReadInt32(stream);
remainingMessageSetBytes -= offsetSize + msgsizeSize;
if (remainingMessageSetBytes < messageSize)
{
// This is a partial message => skip to the end of the message set.
stream.Position += remainingMessageSetBytes;
yield break;
}
// Message body
var crc = BigEndianConverter.ReadInt32(stream);
var crcStartPos = stream.Position; // crc is computed from this position
var magic = stream.ReadByte();
if ((uint)magic > 1)
{
throw new UnsupportedMagicByteVersion((byte)magic);
}
var attributes = stream.ReadByte();
long timestamp = 0;
if (magic == 1)
{
timestamp = BigEndianConverter.ReadInt64(stream);
}
// Check for compression
var codec = (CompressionCodec)(attributes & 3); // Lowest 2 bits
if (codec == CompressionCodec.None)
{
var msg = new ResponseMessage
{
Offset = offset,
Message = new Message
{
Key = Basics.DeserializeByteArray(stream, deserializers.Item1),
Value = Basics.DeserializeByteArray(stream, deserializers.Item2),
TimeStamp = timestamp
}
};
CheckCrc(crc, stream, crcStartPos);
yield return(msg);
}
else
{
// Key is null, read/check/skip
if (BigEndianConverter.ReadInt32(stream) != -1)
{
throw new InvalidDataException("Compressed messages key should be null");
}
// Uncompress
var compressedLength = BigEndianConverter.ReadInt32(stream);
var dataPos = stream.Position;
stream.Position += compressedLength;
CheckCrc(crc, stream, crcStartPos);
using (var uncompressedStream = stream.Pool.Reserve())
{
Uncompress(uncompressedStream, stream.GetBuffer(), (int)dataPos, compressedLength, codec);
// Deserialize recursively
if (magic == 0) // v0 message
{
foreach (var m in
LazyDeserializeMessageSet(uncompressedStream, (int)uncompressedStream.Length,
deserializers))
{
// Flatten
yield return(m);
}
}
else // v1 message, we have to assign the absolute offsets
{
var innerMsgs = ResponseMessageListPool.Reserve();
// We need to deserialize all messages first, because the wrapper offset is the
// offset of the last messe in the set, so wee need to know how many messages there are
// before assigning offsets.
innerMsgs.AddRange(LazyDeserializeMessageSet(uncompressedStream,
(int)uncompressedStream.Length, deserializers));
var baseOffset = offset - innerMsgs.Count + 1;
foreach (var msg in innerMsgs)
{
yield return
(new ResponseMessage
{
Offset = msg.Offset + baseOffset,
Message = msg.Message
});
}
ResponseMessageListPool.Release(innerMsgs);
}
}
}
remainingMessageSetBytes -= messageSize;
}
}
public void Deserialize(ReusableMemoryStream stream, object noextra = null)
{
Partition = BigEndianConverter.ReadInt32(stream);
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream);
Offset = BigEndianConverter.ReadInt64(stream);
}
//N.B., MessageSets are not preceded by an int32 like other array elements in the protocol.
//
//MessageSet => [Offset MessageSize Message]
// Offset => int64
// MessageSize => int32
//
//Message => Crc MagicByte Attributes Key Value
// Crc => int32
// MagicByte => int8
// Attributes => int8
// Key => bytes
// Value => bytes
private static IEnumerable <Message> ReadMessageSet(Stream stream, int messageSetSize)
{
// "As an optimization the server is allowed to return a partial message at the end of the message set.
// Clients should handle this case"
var remainingMessageSetBytes = messageSetSize;
while (remainingMessageSetBytes > 0)
{
// we need at least be able to read offset and messageSize
if (remainingMessageSetBytes < 8 + 4)
{
// not enough bytes left. This is a partial message. Skip to the end of the message set.
stream.Position += remainingMessageSetBytes;
yield break;
}
var offset = BigEndianConverter.ReadInt64(stream);
var messageSize = BigEndianConverter.ReadInt32(stream);
// we took 12 bytes there, check again that we have a full message.
remainingMessageSetBytes -= 8 + 4;
if (remainingMessageSetBytes < messageSize)
{
// not enough bytes left. This is a partial message. Skip to the end of the message set.
stream.Position += remainingMessageSetBytes;
yield break;
}
// Message
var crc = (uint)BigEndianConverter.ReadInt32(stream);
byte magic = (byte)stream.ReadByte();
if (magic != 0)
{
throw new BrokerException("Invalid kafks message magic"); // TODO: use special exception for data corruption
}
var attributes = (byte)stream.ReadByte();
var compression = ParseCompression(attributes);
var key = ReadByteArray(stream);
var value = ReadByteArray(stream);
if (compression == CompressionType.None)
{
var msg = new Message();
msg.Key = key;
msg.Value = value;
msg.Offset = offset;
var computedCrc = Crc32.Update(magic);
computedCrc = Crc32.Update(attributes, computedCrc);
if (key == null)
{
computedCrc = Crc32.Update(_minusOne32, computedCrc);
}
else
{
computedCrc = Crc32.Update(key.Length, computedCrc);
computedCrc = Crc32.Update(key, computedCrc);
}
if (value == null)
{
computedCrc = Crc32.Update(_minusOne32);
}
else
{
computedCrc = Crc32.Update(value.Length, computedCrc);
computedCrc = Crc32.Update(value, computedCrc);
}
computedCrc = Crc32.GetHash(computedCrc);
if (computedCrc != crc)
{
throw new BrokerException(string.Format("Corrupt message: Crc does not match. Caclulated {0} but got {1}", computedCrc, crc));
}
yield return(msg);
}
else if (compression == CompressionType.Gzip)
{
var decompressedStream = new MemoryStream();
new GZipStream(new MemoryStream(value), CompressionMode.Decompress).CopyTo(decompressedStream);
decompressedStream.Seek(0, SeekOrigin.Begin);
// Recursion
var innerMessages = ReadMessageSet(decompressedStream, (int)decompressedStream.Length);
foreach (var innerMessage in innerMessages)
{
yield return(innerMessage);
}
}
else if (compression == CompressionType.Lz4)
{
using (var lz4Stream = new Lz4KafkaStream(new MemoryStream(value), CompressionStreamMode.Decompress))
{
var decompressed = new MemoryStream();
lz4Stream.CopyTo(decompressed);
decompressed.Seek(0, SeekOrigin.Begin);
var decompressedMessages = ReadMessageSet(decompressed, (int)decompressed.Length);
foreach (var msg in decompressedMessages)
{
yield return(msg);
}
}
}
else if (compression == CompressionType.Snappy)
{
if (_snappyCompressedBuffer == null)
{
KafkaSnappyStream.AllocateBuffers(out _snappyUncompressedBuffer, out _snappyCompressedBuffer);
}
using (var snappyStream = new KafkaSnappyStream(new MemoryStream(value), CompressionStreamMode.Decompress, _snappyUncompressedBuffer, _snappyCompressedBuffer))
{
var decompressed = new MemoryStream();
snappyStream.CopyTo(decompressed);
decompressed.Seek(0, SeekOrigin.Begin);
var decompressedMessages = ReadMessageSet(decompressed, (int)decompressed.Length);
foreach (var msg in decompressedMessages)
{
yield return(msg);
}
}
}
else
{
throw new BrokerException(string.Format("Unknown compression type: {0}", attributes & 3));
}
// subtract messageSize of that message from remaining bytes
remainingMessageSetBytes -= messageSize;
}
}
public void Deserialize(ReusableMemoryStream stream, object noextra)
{
Partition = BigEndianConverter.ReadInt32(stream);
Time = BigEndianConverter.ReadInt64(stream);
MaxNumberOfOffsets = BigEndianConverter.ReadInt32(stream);
}
