+ 4; // batchLength
/// <remarks>
/// From the official protocol documentation available at
/// https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-FetchAPI
/// "As an optimization the server is allowed to return a partial message at the end of the message set. Clients should handle this case."
/// If the end of the RecordBatch exceeds the length of the whole response (= endOfAllBatches), we should discard this RecordBatch.
/// </remarks>
public static RecordBatch Deserialize(ReusableMemoryStream input, Deserializers deserializers, long endOfAllBatches)
{
var recordBatch = new RecordBatch();
if (input.Position + BytesNecessaryToGetLength > endOfAllBatches)
{
throw new ProtocolException(
$"Trying to read a batch record at {input.Position} and the end of all batches is {endOfAllBatches}."
+ $" There is not enough bytes remaining to even read the first fields...");
}
recordBatch.BaseOffset = BigEndianConverter.ReadInt64(input);
var batchLength = BigEndianConverter.ReadInt32(input);
var endOfBatch = input.Position + batchLength;
if (endOfAllBatches < endOfBatch)
{
// Partial message, CRCs won't match, return here so the CRC check doesn't throw
return(null);
}
recordBatch.PartitionLeaderEpoch = BigEndianConverter.ReadInt32(input);
var magic = input.ReadByte();
// Current magic value is 2
if ((uint)magic != 2)
{
throw new UnsupportedMagicByteVersion((byte)magic, "2");
}
var crc = (uint)BigEndianConverter.ReadInt32(input);
var afterCrcPosition = input.Position; // The crc is calculated starting from this position
Crc32.CheckCrcCastagnoli((int)crc, input, afterCrcPosition, endOfBatch - afterCrcPosition);
var attributes = BigEndianConverter.ReadInt16(input);
recordBatch.CompressionCodec = (CompressionCodec)(attributes & CompressionCodecMask);
recordBatch.IsTransactional = (attributes & TransactionalFlagMask) != 0;
recordBatch.IsControl = (attributes & ControlFlagMask) != 0;
recordBatch.TimestampType = (attributes & TimestampTypeMask) > 0
? TimestampType.LogAppendTime
: TimestampType.CreateTime;
var lastOffsetDelta = BigEndianConverter.ReadInt32(input);
var firstTimestamp = BigEndianConverter.ReadInt64(input);
var maxTimestamp = BigEndianConverter.ReadInt64(input);
recordBatch.ProducerId = BigEndianConverter.ReadInt64(input);
recordBatch.ProducerEpoch = BigEndianConverter.ReadInt16(input);
recordBatch.BaseSequence = BigEndianConverter.ReadInt32(input);
var numberOfRecords = BigEndianConverter.ReadInt32(input);
recordBatch.Records = DeserializeRecords(recordBatch, input, numberOfRecords, endOfBatch, firstTimestamp, deserializers);
return(recordBatch);
}
internal static List <ResponseMessage> DeserializeRecordBatch(ReusableMemoryStream stream,
Deserializers deserializers)
{
var list = ResponseMessageListPool.Reserve();
// First of all, we need to check if it is really a recordBatch (i.e. magic byte = 2)
// For that, we need to fast-forward to the magic byte. Depending on its value, we go back
// and deserialize following the corresponding format. (Kafka is amazing).
var startOfBatchOffset = stream.Position;
stream.Position += HeaderSize;
var magic = stream.ReadByte();
stream.Position = startOfBatchOffset;
if (magic < 2)
{
// We were mistaken, this is not a recordBatch but a message set!
return(DeserializeMessageSet(stream, deserializers));
}
// Now we know what we received is a proper recordBatch
var size = BigEndianConverter.ReadInt32(stream);
var endOfAllBatches = stream.Position + size;
if (stream.Length < endOfAllBatches)
{
throw new ProtocolException($"Fetch response advertise record batches of total size {size},"
+ $" but the stream only contains {stream.Length - stream.Position} byte remaining");
}
while (stream.Position < endOfAllBatches)
{
var recordBatch = RecordBatch.Deserialize(stream, deserializers, endOfAllBatches);
if (recordBatch == null)
{
// We received a partial record batch, discard it
stream.Position = endOfAllBatches;
break;
}
list.AddRange(recordBatch.Records.Select(record => new ResponseMessage
{
Message = new Message {
Key = record.Key, Value = record.Value, TimeStamp = record.Timestamp
},
Offset = record.Offset
}));
}
return(list);
}
public IEnumerable <Record> DeserializeRecordsUncompressed(ReusableMemoryStream input, int numberOfRecords, long endOfBatch,
long firstTimeStamp, Deserializers deserializers)
{
for (var i = 0; i < numberOfRecords; i++)
{
var length = VarIntConverter.ReadInt32(input);
if (input.Length - input.Position < length)
{
throw new ProtocolException(
$"Record said it was of length {length}, but actually only {input.Length - input.Position} bytes remain");
}
var attributes = input.ReadByte(); // ignored for now
var timeStampDelta = VarIntConverter.ReadInt64(input);
var offsetDelta = VarIntConverter.ReadInt32(input);
var keyLength = VarIntConverter.ReadInt32(input);
var key = keyLength == -1 ? null : deserializers.Item1.Deserialize(input, keyLength);
var valueLength = VarIntConverter.ReadInt32(input);
var value = valueLength == -1 ? null : deserializers.Item1.Deserialize(input, valueLength);
var headersCount = VarIntConverter.ReadInt32(input);
var headers = new List <KafkaRecordHeader>(headersCount);
for (var j = 0; j < headersCount; j++)
{
headers.Add(Record.DeserializeHeader(input));
}
yield return(new Record
{
Headers = headers,
Key = key,
Timestamp = firstTimeStamp + timeStampDelta,
Value = value,
Offset = BaseOffset + offsetDelta
});
}
}
// Deserialize a message set to a sequence of messages.
// This handles the "partial message allowed at end of message set" from Kafka brokers
// and compressed message sets (the method recursively calls itself in this case and
// flatten the result). The returned enumeration must be enumerated for deserialization
// effectiveley occuring.
private static IEnumerable <ResponseMessage> LazyDeserializeMessageSet(ReusableMemoryStream stream, int messageSetSize, Deserializers deserializers)
{
var remainingMessageSetBytes = messageSetSize;
while (remainingMessageSetBytes > 0)
{
const int offsetSize = 8;
const int msgsizeSize = 4;
if (remainingMessageSetBytes < offsetSize + msgsizeSize)
{
// This is a partial message => skip to the end of the message set.
// TODO: unit test this
stream.Position += remainingMessageSetBytes;
yield break;
}
var offset = BigEndianConverter.ReadInt64(stream);
var messageSize = BigEndianConverter.ReadInt32(stream);
remainingMessageSetBytes -= offsetSize + msgsizeSize;
if (remainingMessageSetBytes < messageSize)
{
// This is a partial message => skip to the end of the message set.
stream.Position += remainingMessageSetBytes;
yield break;
}
// Message body
var crc = BigEndianConverter.ReadInt32(stream);
var crcStartPos = stream.Position; // crc is computed from this position
var magic = stream.ReadByte();
if (magic != 0)
{
throw new UnsupportedMagicByteVersion((byte)magic);
}
var attributes = stream.ReadByte();
// Check for compression
var codec = (CompressionCodec)(attributes & 3); // Lowest 2 bits
if (codec == CompressionCodec.None)
{
var msg = new ResponseMessage
{
Offset = offset,
Message = new Message
{
Key = Basics.DeserializeByteArray(stream, deserializers.Item1),
Value = Basics.DeserializeByteArray(stream, deserializers.Item2)
}
};
CheckCrc(crc, stream, crcStartPos);
yield return(msg);
}
else
{
// Key is null, read/check/skip
if (BigEndianConverter.ReadInt32(stream) != -1)
{
throw new InvalidDataException("Compressed messages key should be null");
}
// Uncompress
var compressedLength = BigEndianConverter.ReadInt32(stream);
var dataPos = stream.Position;
stream.Position += compressedLength;
CheckCrc(crc, stream, crcStartPos);
using (var uncompressedStream = stream.Pool.Reserve())
{
Uncompress(uncompressedStream, stream.GetBuffer(), (int)dataPos, compressedLength, codec);
// Deserialize recursively
foreach (var m in LazyDeserializeMessageSet(uncompressedStream, (int)uncompressedStream.Length, deserializers))
{
// Flatten
yield return(m);
}
}
}
remainingMessageSetBytes -= messageSize;
}
}
// Deserialize a message set to a sequence of messages.
// This handles the "partial message allowed at end of message set" from Kafka brokers
// and compressed message sets (the method recursively calls itself in this case and
// flatten the result). The returned enumeration must be enumerated for deserialization
// effectiveley occuring.
//
// A message set can contain a mix of v0 and v1 messages.
// In the case of compressed messages, offsets are returned differently by brokers.
// Messages inside compressed message v0 will have absolute offsets assigned.
// Messages inside compressed message v1 will have relative offset assigned, starting
// from 0. The wrapping compressed message itself is assigned the absolute offset of the last
// message in the set. That means in this case we can only assign offsets after having decompressing
// all messages. Lazy deserialization won't be so lazy anymore...
private static IEnumerable <ResponseMessage> LazyDeserializeMessageSet(ReusableMemoryStream stream, int messageSetSize, Deserializers deserializers)
{
var remainingMessageSetBytes = messageSetSize;
while (remainingMessageSetBytes > 0)
{
const int offsetSize = 8;
const int msgsizeSize = 4;
if (remainingMessageSetBytes < offsetSize + msgsizeSize)
{
// This is a partial message => skip to the end of the message set.
// TODO: unit test this
stream.Position += remainingMessageSetBytes;
yield break;
}
var offset = BigEndianConverter.ReadInt64(stream);
var messageSize = BigEndianConverter.ReadInt32(stream);
remainingMessageSetBytes -= offsetSize + msgsizeSize;
if (remainingMessageSetBytes < messageSize)
{
// This is a partial message => skip to the end of the message set.
stream.Position += remainingMessageSetBytes;
yield break;
}
// Message body
var crc = BigEndianConverter.ReadInt32(stream);
var crcStartPos = stream.Position; // crc is computed from this position
var magic = stream.ReadByte();
if ((uint)magic > 1)
{
throw new UnsupportedMagicByteVersion((byte)magic);
}
var attributes = stream.ReadByte();
long timestamp = 0;
if (magic == 1)
{
timestamp = BigEndianConverter.ReadInt64(stream);
}
// Check for compression
var codec = (CompressionCodec)(attributes & 3); // Lowest 2 bits
if (codec == CompressionCodec.None)
{
var msg = new ResponseMessage
{
Offset = offset,
Message = new Message
{
Key = Basics.DeserializeByteArray(stream, deserializers.Item1),
Value = Basics.DeserializeByteArray(stream, deserializers.Item2),
TimeStamp = timestamp
}
};
CheckCrc(crc, stream, crcStartPos);
yield return(msg);
}
else
{
// Key is null, read/check/skip
if (BigEndianConverter.ReadInt32(stream) != -1)
{
throw new InvalidDataException("Compressed messages key should be null");
}
// Uncompress
var compressedLength = BigEndianConverter.ReadInt32(stream);
var dataPos = stream.Position;
stream.Position += compressedLength;
CheckCrc(crc, stream, crcStartPos);
using (var uncompressedStream = stream.Pool.Reserve())
{
Uncompress(uncompressedStream, stream.GetBuffer(), (int)dataPos, compressedLength, codec);
// Deserialize recursively
if (magic == 0) // v0 message
{
foreach (var m in
LazyDeserializeMessageSet(uncompressedStream, (int)uncompressedStream.Length,
deserializers))
{
// Flatten
yield return(m);
}
}
else // v1 message, we have to assign the absolute offsets
{
var innerMsgs = ResponseMessageListPool.Reserve();
// We need to deserialize all messages first, because the wrapper offset is the
// offset of the last messe in the set, so wee need to know how many messages there are
// before assigning offsets.
innerMsgs.AddRange(LazyDeserializeMessageSet(uncompressedStream,
(int)uncompressedStream.Length, deserializers));
var baseOffset = offset - innerMsgs.Count + 1;
foreach (var msg in innerMsgs)
{
yield return
(new ResponseMessage
{
Offset = msg.Offset + baseOffset,
Message = msg.Message
});
}
ResponseMessageListPool.Release(innerMsgs);
}
}
}
remainingMessageSetBytes -= messageSize;
}
}