public void Deserialize(ReusableMemoryStream stream, object extra, Basics.ApiVersion version)
{
TopicName = Basics.DeserializeString(stream);
var count = BigEndianConverter.ReadInt32(stream);
var array = new TPartitionData[count];
object pdExtra = null;
if (extra != null)
{
var config = extra as SerializationConfig;
pdExtra = config.GetDeserializersForTopic(TopicName);
}
try
{
for (int i = 0; i < count; ++i)
{
array[i] = new TPartitionData();
array[i].Deserialize(stream, pdExtra, version);
}
PartitionsData = array;
}
catch (ProtocolException pEx)
{
pEx.Topic = TopicName;
throw;
}
}
public void Deserialize(ReusableMemoryStream stream, object _, Basics.ApiVersion __)
{
MemberId = Basics.DeserializeString(stream);
BigEndianConverter.ReadInt32(stream);
Metadata = new ConsumerGroupProtocolMetadata();
Metadata.Deserialize(stream, null, Basics.ApiVersion.Ignored);
}
private static PartitionMeta DeserializePartitionMeta(MemoryStream stream)
{
var ret = new PartitionMeta();
ret.ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream);
ret.Id = BigEndianConverter.ReadInt32(stream);
ret.Leader = BigEndianConverter.ReadInt32(stream);
var count = BigEndianConverter.ReadInt32(stream);
ret.Replicas = new int[count];
for (int i = 0; i < count; i++)
{
ret.Replicas[i] = BigEndianConverter.ReadInt32(stream);
}
count = BigEndianConverter.ReadInt32(stream);
ret.Isr = new int[count];
for (int i = 0; i < count; i++)
{
ret.Isr[i] = BigEndianConverter.ReadInt32(stream);
}
return(ret);
}
/// <summary>
/// FetchResponse => [TopicName [Partition ErrorCode HighwaterMarkOffset MessageSetSize MessageSet]]
/// TopicName => string
/// Partition => int32
/// ErrorCode => int16
/// HighwaterMarkOffset => int64
/// MessageSetSize => int32
/// </summary>
/// <param name="buff"></param>
/// <returns></returns>
public static FetchResponse DeserializeFetchResponse(byte[] buff)
{
var stream = new MemoryStream(buff);
stream.Position += 4; // skip body
var response = new FetchResponse();
var len = BigEndianConverter.ReadInt32(stream);
response.Topics = new FetchResponse.TopicFetchData[len];
for (int t = 0; t < len; t++)
{
var topic = new FetchResponse.TopicFetchData();
response.Topics[t] = topic;
topic.Topic = ReadString(stream);
var len2 = BigEndianConverter.ReadInt32(stream);
topic.Partitions = new FetchResponse.PartitionFetchData[len2];
for (int i = 0; i < len2; i++)
{
topic.Partitions[i] = new FetchResponse.PartitionFetchData
{
Partition = BigEndianConverter.ReadInt32(stream),
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream),
HighWatermarkOffset = BigEndianConverter.ReadInt64(stream),
Messages = ReadMessageSet(stream).ToArray()
}
}
;
}
return(response);
}
public void Deserialize(ReusableMemoryStream stream, object extra)
{
Partition = BigEndianConverter.ReadInt32(stream);
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream);
HighWatermarkOffset = BigEndianConverter.ReadInt64(stream);
Messages = DeserializeMessageSet(stream, extra as Deserializers);
}
bool ReadBlock()
{
if (!StreamUtils.ReadAll(_base, _compressedBuffer, 4))
{
return(false);
}
var blockSize = BigEndianConverter.ReadInt32(_compressedBuffer);
if (blockSize > _compressedBuffer.Length)
{
_compressedBuffer = new byte[blockSize];
}
if (!StreamUtils.ReadAll(_base, _compressedBuffer, blockSize))
{
throw new InvalidDataException("Unexpected end of snappy data block");
}
// Reallocate compressed buffer if needed
var uncompressedLen = Snappy.SnappyCodec.GetUncompressedLength(_compressedBuffer, 0, blockSize);
if (uncompressedLen > _uncompressedBuffer.Length)
{
_uncompressedBuffer = new byte[uncompressedLen];
}
var read = SnappyCodec.Uncompress(_compressedBuffer, 0, blockSize, _uncompressedBuffer, 0);
_bufferLen = read;
_bufferPtr = 0;
return(true);
}
internal static List <ResponseMessage> DeserializeMessageSet(ReusableMemoryStream stream, Deserializers deserializers)
{
var list = ResponseMessageListPool.Reserve();
list.AddRange(LazyDeserializeMessageSet(stream, BigEndianConverter.ReadInt32(stream), deserializers));
return(list);
}
internal CompressionCodec Compression; // Only used in test for serializing
#region Deserialization
public void Deserialize(ReusableMemoryStream stream, object extra, Basics.ApiVersion version)
{
Partition = BigEndianConverter.ReadInt32(stream);
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream);
HighWatermarkOffset = BigEndianConverter.ReadInt64(stream);
if (version >= Basics.ApiVersion.V4)
{
LastStableOffset = BigEndianConverter.ReadInt64(stream);
AbortedTransactions = Basics.DeserializeArray <AbortedTransaction>(stream);
}
if (version >= Basics.ApiVersion.V5)
{
LogStartOffset = BigEndianConverter.ReadInt64(stream);
}
try
{
Messages = version >= Basics.ApiVersion.V3
? DeserializeRecordBatch(stream, extra as Deserializers)
: DeserializeMessageSet(stream, extra as Deserializers);
}
catch (ProtocolException pEx)
{
pEx.Partition = Partition;
throw;
}
}
public void Deserialize(ReusableMemoryStream stream, object _, Basics.ApiVersion __)
{
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream);
CoordinatorId = BigEndianConverter.ReadInt32(stream);
CoordinatorHost = Basics.DeserializeString(stream);
CoordinatorPort = BigEndianConverter.ReadInt32(stream);
}
public void TestSerializeMetadataRequest()
{
var meta = new TopicRequest
{
Topics = new[] { "poulpe", "banana" }
};
using (var serialized = meta.Serialize(new ReusableMemoryStream(null), 61, ClientId, null))
{
CheckHeader(Basics.ApiKey.MetadataRequest, 0, 61, TheClientId, serialized);
Assert.AreEqual(2, BigEndianConverter.ReadInt32(serialized));
Assert.AreEqual("poulpe", Basics.DeserializeString(serialized));
Assert.AreEqual("banana", Basics.DeserializeString(serialized));
}
meta = new TopicRequest
{
Topics = null
};
using (var serialized = meta.Serialize(new ReusableMemoryStream(null), 61, ClientId, null))
{
CheckHeader(Basics.ApiKey.MetadataRequest, 0, 61, TheClientId, serialized);
Assert.AreEqual(0, BigEndianConverter.ReadInt32(serialized));
Assert.AreEqual(serialized.Length, serialized.Position);
}
}
public void TestSerializeMessageSet()
{
using (var serialized = new ReusableMemoryStream(null))
{
var set = new PartitionData
{
Partition = 42,
CompressionCodec = CompressionCodec.None,
Messages = new[]
{
new Message {
Key = Key, Value = Value
},
new Message {
Key = Key, Value = Value
}
}
};
set.Serialize(serialized, SerializationConfig.ByteArraySerializers);
serialized.Position = 0;
Assert.AreEqual(42, BigEndianConverter.ReadInt32(serialized)); // Partition
Assert.AreEqual(serialized.Length - 4 - 4, BigEndianConverter.ReadInt32(serialized)); // MessageSet size
Assert.AreEqual(0, BigEndianConverter.ReadInt64(serialized)); // First message offset
int firstMsgSize = BigEndianConverter.ReadInt32(serialized);
serialized.Position += firstMsgSize;
Assert.AreEqual(0, BigEndianConverter.ReadInt64(serialized)); // First message offset
int secondMsgSize = BigEndianConverter.ReadInt32(serialized);
serialized.Position += secondMsgSize;
Assert.AreEqual(serialized.Length, serialized.Position);
}
}
private void TestSerializeOneMessageCommon(Message message)
{
using (var serialized = new ReusableMemoryStream(null))
{
message.Serialize(serialized, CompressionCodec.None, new Tuple <ISerializer, ISerializer>(null, null));
Assert.AreEqual(FullMessageSize, serialized.Length);
Assert.AreEqual(0, serialized.GetBuffer()[4]); // magic byte is 0
Assert.AreEqual(0, serialized.GetBuffer()[5]); // attributes is 0
serialized.Position = 6;
Assert.AreEqual(TheKey.Length, BigEndianConverter.ReadInt32(serialized));
CompareBuffers(Key, serialized);
Assert.AreEqual(TheValue.Length, BigEndianConverter.ReadInt32(serialized));
CompareBuffers(Value, serialized);
}
using (var serialized = new ReusableMemoryStream(null))
{
var msg = new Message {
Value = Value
};
msg.Serialize(serialized, CompressionCodec.None, new Tuple <ISerializer, ISerializer>(null, null));
Assert.AreEqual(NullKeyMessageSize, serialized.Length);
Assert.AreEqual(0, serialized.GetBuffer()[4]); // magic byte is 0
Assert.AreEqual(0, serialized.GetBuffer()[5]); // attributes is 0
serialized.Position = 6;
Assert.AreEqual(-1, BigEndianConverter.ReadInt32(serialized));
Assert.AreEqual(TheValue.Length, BigEndianConverter.ReadInt32(serialized));
CompareBuffers(Value, serialized);
}
}
internal static OffsetResponse DeserializeOffsetResponse(byte[] buff, int count)
{
var stream = new MemoryStream(buff, 0, count);
stream.Position += 4; // skip body
var response = new OffsetResponse();
var len = BigEndianConverter.ReadInt32(stream);
// TODO: make sure connection is closed
if (len != 1)
{
throw new BrokerException("Invalid message format");
}
response.TopicName = ReadString(stream);
len = BigEndianConverter.ReadInt32(stream);
response.Partitions = new OffsetResponse.PartitionOffsetData[len];
for (int i = 0; i < len; i++)
{
var part = response.Partitions[i] = new OffsetResponse.PartitionOffsetData();
part.Partition = BigEndianConverter.ReadInt32(stream);
part.ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream);
var len2 = BigEndianConverter.ReadInt32(stream);
part.Offsets = new long[len2];
for (int j = 0; j < len2; j++)
{
part.Offsets[j] = BigEndianConverter.ReadInt64(stream);
}
}
return(response);
}
public void Deserialize(ReusableMemoryStream stream, object _, Basics.ApiVersion __)
{
Partition = BigEndianConverter.ReadInt32(stream);
Offset = BigEndianConverter.ReadInt64(stream);
Metadata = Basics.DeserializeString(stream);
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream);
}
public void Deserialize(ReusableMemoryStream stream, object extra, Basics.ApiVersion version)
{
ProducePartitionResponse.Deserialize(stream, extra, version);
if (version > Basics.ApiVersion.V0)
{
ThrottleTime = BigEndianConverter.ReadInt32(stream);
}
}
private static BrokerMeta DeserializeBrokerMeta(MemoryStream stream)
{
return(new BrokerMeta {
NodeId = BigEndianConverter.ReadInt32(stream),
Host = ReadString(stream),
Port = BigEndianConverter.ReadInt32(stream)
});
}
public void Deserialize(ReusableMemoryStream stream, object _, Basics.ApiVersion __)
{
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream);
Id = BigEndianConverter.ReadInt32(stream);
Leader = BigEndianConverter.ReadInt32(stream);
Replicas = Basics.DeserializeArray(stream, BigEndianConverter.ReadInt32);
Isr = Basics.DeserializeArray(stream, BigEndianConverter.ReadInt32);
}
public void Deserialize(ReusableMemoryStream stream, object _, Basics.ApiVersion __)
{
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream);
GenerationId = BigEndianConverter.ReadInt32(stream);
GroupProtocol = Basics.DeserializeString(stream);
LeaderId = Basics.DeserializeString(stream);
MemberId = Basics.DeserializeString(stream);
GroupMembers = Basics.DeserializeArray <GroupMember>(stream);
}
private static ProducerResponse.PartitionResponse DeserializePartitionResponse(MemoryStream stream)
{
return(new ProducerResponse.PartitionResponse
{
Partition = BigEndianConverter.ReadInt32(stream),
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream),
Offset = BigEndianConverter.ReadInt64(stream)
});
}
+ 4; // batchLength
/// <remarks>
/// From the official protocol documentation available at
/// https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-FetchAPI
/// "As an optimization the server is allowed to return a partial message at the end of the message set. Clients should handle this case."
/// If the end of the RecordBatch exceeds the length of the whole response (= endOfAllBatches), we should discard this RecordBatch.
/// </remarks>
public static RecordBatch Deserialize(ReusableMemoryStream input, Deserializers deserializers, long endOfAllBatches)
{
var recordBatch = new RecordBatch();
if (input.Position + BytesNecessaryToGetLength > endOfAllBatches)
{
throw new ProtocolException(
$"Trying to read a batch record at {input.Position} and the end of all batches is {endOfAllBatches}."
+ $" There is not enough bytes remaining to even read the first fields...");
}
recordBatch.BaseOffset = BigEndianConverter.ReadInt64(input);
var batchLength = BigEndianConverter.ReadInt32(input);
var endOfBatch = input.Position + batchLength;
if (endOfAllBatches < endOfBatch)
{
// Partial message, CRCs won't match, return here so the CRC check doesn't throw
return(null);
}
recordBatch.PartitionLeaderEpoch = BigEndianConverter.ReadInt32(input);
var magic = input.ReadByte();
// Current magic value is 2
if ((uint)magic != 2)
{
throw new UnsupportedMagicByteVersion((byte)magic, "2");
}
var crc = (uint)BigEndianConverter.ReadInt32(input);
var afterCrcPosition = input.Position; // The crc is calculated starting from this position
Crc32.CheckCrcCastagnoli((int)crc, input, afterCrcPosition, endOfBatch - afterCrcPosition);
var attributes = BigEndianConverter.ReadInt16(input);
recordBatch.CompressionCodec = (CompressionCodec)(attributes & CompressionCodecMask);
recordBatch.IsTransactional = (attributes & TransactionalFlagMask) != 0;
recordBatch.IsControl = (attributes & ControlFlagMask) != 0;
recordBatch.TimestampType = (attributes & TimestampTypeMask) > 0
? TimestampType.LogAppendTime
: TimestampType.CreateTime;
var lastOffsetDelta = BigEndianConverter.ReadInt32(input);
var firstTimestamp = BigEndianConverter.ReadInt64(input);
var maxTimestamp = BigEndianConverter.ReadInt64(input);
recordBatch.ProducerId = BigEndianConverter.ReadInt64(input);
recordBatch.ProducerEpoch = BigEndianConverter.ReadInt16(input);
recordBatch.BaseSequence = BigEndianConverter.ReadInt32(input);
var numberOfRecords = BigEndianConverter.ReadInt32(input);
recordBatch.Records = DeserializeRecords(recordBatch, input, numberOfRecords, endOfBatch, firstTimestamp, deserializers);
return(recordBatch);
}
public void Deserialize(ReusableMemoryStream stream, object _, Basics.ApiVersion version)
{
Partition = BigEndianConverter.ReadInt32(stream);
Time = BigEndianConverter.ReadInt64(stream);
if (version == Basics.ApiVersion.V0)
{
MaxNumberOfOffsets = BigEndianConverter.ReadInt32(stream);
}
}
// Used only in tests
public void Deserialize(ReusableMemoryStream stream, object _, Basics.ApiVersion version)
{
Partition = BigEndianConverter.ReadInt32(stream);
FetchOffset = BigEndianConverter.ReadInt64(stream);
MaxBytes = BigEndianConverter.ReadInt32(stream);
if (version >= Basics.ApiVersion.V5)
{
LogStartOffset = BigEndianConverter.ReadInt64(stream);
}
}
public static object DeserializeByteArray(ReusableMemoryStream stream, IDeserializer deserializer)
{
var len = BigEndianConverter.ReadInt32(stream);
if (len == -1)
{
return(null);
}
return(deserializer.Deserialize(stream, len));
}
public void Deserialize(ReusableMemoryStream stream, object _, Basics.ApiVersion version)
{
Partition = BigEndianConverter.ReadInt32(stream);
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream);
Offset = BigEndianConverter.ReadInt64(stream);
if (version >= Basics.ApiVersion.V2)
{
Timestamp = BigEndianConverter.ReadInt64(stream);
}
}
public static TData[] DeserializeArray <TData>(ReusableMemoryStream stream, Func <ReusableMemoryStream, TData> dataDeserializer)
{
var count = BigEndianConverter.ReadInt32(stream);
var array = new TData[count];
for (int i = 0; i < count; ++i)
{
array[i] = dataDeserializer(stream);
}
return(array);
}
public void Deserialize(ReusableMemoryStream stream, object _, Basics.ApiVersion __)
{
ErrorCode = (ErrorCode)BigEndianConverter.ReadInt16(stream);
MemberAssignment = new ConsumerGroupMemberAssignment
{
PartitionAssignments = Enumerable.Empty <TopicData <PartitionAssignment> >()
};
if (BigEndianConverter.ReadInt32(stream) > 0)
{
MemberAssignment.Deserialize(stream, null, Basics.ApiVersion.Ignored);
}
}
public static TData[] DeserializeArrayExtra <TData>(ReusableMemoryStream stream, object extra) where TData : IMemoryStreamSerializable, new()
{
var count = BigEndianConverter.ReadInt32(stream);
var array = new TData[count];
for (int i = 0; i < count; ++i)
{
array[i] = new TData();
array[i].Deserialize(stream, extra);
}
return(array);
}
//N.B., MessageSets are not preceded by an int32 like other array elements in the protocol.
//
//MessageSet => [Offset MessageSize Message]
// Offset => int64
// MessageSize => int32
//
//Message => Crc MagicByte Attributes Key Value
// Crc => int32
// MagicByte => int8
// Attributes => int8
// Key => bytes
// Value => bytes
private static IEnumerable <Message> ReadMessageSet(MemoryStream stream)
{
// "As an optimization the server is allowed to return a partial message at the end of the message set.
// Clients should handle this case"
var messageSetSize = BigEndianConverter.ReadInt32(stream);
var remainingMessageSetBytes = messageSetSize;
while (remainingMessageSetBytes > 0)
{
// we need at least be able to read offset and messageSize
if (remainingMessageSetBytes < +8 + 4)
{
// not enough bytes left. This is a partial message. Skip to the end of the message set.
stream.Position += remainingMessageSetBytes;
yield break;
}
var offset = BigEndianConverter.ReadInt64(stream);
var messageSize = BigEndianConverter.ReadInt32(stream);
// we took 12 bytes there, check again that we have a full message.
remainingMessageSetBytes -= 8 + 4;
if (remainingMessageSetBytes < messageSize)
{
// not enough bytes left. This is a partial message. Skip to the end of the message set.
stream.Position += remainingMessageSetBytes;
yield break;
}
// Message
var crc = BigEndianConverter.ReadInt32(stream);
var crcPos = stream.Position;
var magic = stream.ReadByte();
var attributes = stream.ReadByte();
var msg = new Message();
msg.Key = ReadByteArray(stream);
msg.Value = ReadByteArray(stream);
msg.Offset = offset;
var pos = stream.Position;
var computedCrcArray = Crc32.Compute(stream, crcPos, pos - crcPos);
var computedCrc = BigEndianConverter.ToInt32(computedCrcArray);
if (computedCrc != crc)
{
throw new BrokerException(string.Format("Corrupt message: Crc does not match. Caclulated {0} but got {1}", computedCrc, crc));
}
yield return(msg);
// subtract messageSize of that message from remaining bytes
remainingMessageSetBytes -= messageSize;
}
}
private static byte[] ReadByteArray(Stream stream)
{
var len = BigEndianConverter.ReadInt32(stream);
if (len == -1)
{
return(null);
}
var buff = new byte[len];
stream.Read(buff, 0, len);
return(buff);
}
private static void CheckHeader(
Basics.ApiKey apiKey,
short apiVersion,
int correlationId,
string clientId,
ReusableMemoryStream stream)
{
Assert.AreEqual(stream.Length - 4, BigEndianConverter.ReadInt32(stream)); // Size
Assert.AreEqual((short)apiKey, BigEndianConverter.ReadInt16(stream));
Assert.AreEqual(apiVersion, BigEndianConverter.ReadInt16(stream));
Assert.AreEqual(correlationId, BigEndianConverter.ReadInt32(stream));
Assert.AreEqual(clientId, Basics.DeserializeString(stream));
}
