/// <inheritdoc />
public override IMemberMetadata DecodeMetadata(IKafkaReader reader)
{
var version = reader.ReadInt16();
var topicNames = new string[reader.ReadInt32()];
for (var t = 0; t < topicNames.Length; t++)
{
topicNames[t] = reader.ReadString();
}
var userData = reader.ReadBytes();
return(new ConsumerProtocolMetadata(version, topicNames, userData));
}
/// <summary>
/// Decode messages from a payload and assign it a given kafka offset.
/// </summary>
/// <param name="reader">The reader</param>
/// <param name="messageSize">The size of the message, for Crc Hash calculation</param>
/// <param name="offset">The offset represting the log entry from kafka of this message.</param>
/// <param name="partitionId">The partition being read</param>
/// <returns>Enumerable representing stream of messages decoded from byte[].</returns>
/// <remarks>The return type is an Enumerable as the message could be a compressed message set.</remarks>
public static IImmutableList <Message> ReadMessage(this IKafkaReader reader, int messageSize, long offset, int partitionId = 0)
{
var crc = BitConverter.ToUInt32(reader.RawRead(4), 0);
var crcHash = BitConverter.ToUInt32(reader.CrcHash(messageSize - 4), 0);
if (crc != crcHash)
{
throw new CrcValidationException("Buffer did not match CRC validation.")
{
Crc = crc, CalculatedCrc = crcHash
}
}
;
var messageVersion = reader.ReadByte();
var attribute = reader.ReadByte();
DateTime?timestamp = null;
if (messageVersion >= 1)
{
var milliseconds = reader.ReadInt64();
if (milliseconds >= 0)
{
timestamp = milliseconds.FromUnixEpochMilliseconds();
}
}
var key = reader.ReadBytes();
var codec = (MessageCodec)(Message.AttributeMask & attribute);
switch (codec)
{
case MessageCodec.CodecNone: {
var value = reader.ReadBytes();
return(ImmutableList <Message> .Empty.Add(new Message(value, attribute, offset, partitionId, messageVersion, key, timestamp)));
}
case MessageCodec.CodecGzip: {
var messageLength = reader.ReadInt32();
var messageStream = new LimitedReadableStream(reader.BaseStream, messageLength);
using (var gzipReader = new BigEndianBinaryReader(messageStream.Unzip())) {
return(gzipReader.ReadMessages(partitionId));
}
}
default:
throw new NotSupportedException($"Codec type of {codec} is not supported.");
}
}
/// <inheritdoc />
public IMemberAssignment DecodeAssignment(IKafkaReader reader)
{
var expectedLength = reader.ReadInt32();
if (!reader.HasBytes(expectedLength))
{
throw new BufferUnderRunException($"{ProtocolType} Assignment size of {expectedLength} is not fully available.");
}
if (expectedLength == 0)
{
return(null);
}
return(DecodeAssignment(reader, expectedLength));
}
/// <inheritdoc />
public IMemberMetadata DecodeMetadata(string assignmentStrategy, IKafkaReader reader)
{
var expectedLength = reader.ReadInt32();
if (!reader.HasBytes(expectedLength))
{
throw new BufferUnderRunException($"{ProtocolType} Metadata size of {expectedLength} is not fully available.");
}
if (expectedLength == 0)
{
return(null);
}
return(DecodeMetadata(assignmentStrategy, reader, expectedLength));
}
/// <inheritdoc />
public override IMemberAssignment DecodeAssignment(IKafkaReader reader)
{
var version = reader.ReadInt16();
var topics = new List <TopicPartition>();
var topicCount = reader.ReadInt32();
for (var t = 0; t < topicCount; t++)
{
var topicName = reader.ReadString();
var partitionCount = reader.ReadInt32();
for (var p = 0; p < partitionCount; p++)
{
var partitionId = reader.ReadInt32();
topics.Add(new TopicPartition(topicName, partitionId));
}
}
return(new ConsumerMemberAssignment(version, topics));
}
/// <inheritdoc />
protected override ConsumerMemberAssignment DecodeAssignment(IKafkaReader reader, int expectedLength)
{
var version = reader.ReadInt16();
var topics = new List <TopicPartition>();
var topicCount = reader.ReadInt32();
for (var t = 0; t < topicCount; t++)
{
var topicName = reader.ReadString();
var partitionCount = reader.ReadInt32();
for (var p = 0; p < partitionCount; p++)
{
var partitionId = reader.ReadInt32();
topics.Add(new TopicPartition(topicName, partitionId));
}
}
var userData = reader.ReadBytes();
return(new ConsumerMemberAssignment(topics, userData, version));
}
/// <summary>
/// Decode a byte[] that represents a collection of messages.
/// </summary>
/// <param name="reader">The reader</param>
/// <param name="partitionId">The partitionId messages are being read from.</param>
/// <returns>Enumerable representing stream of messages decoded from byte[]</returns>
public static IImmutableList <Message> ReadMessages(this IKafkaReader reader, int partitionId = 0)
{
var expectedLength = reader.ReadInt32();
if (!reader.Available(expectedLength))
{
throw new BufferUnderRunException($"Message set size of {expectedLength} is not fully available.");
}
var messages = ImmutableList <Message> .Empty;
var finalPosition = reader.Position + expectedLength;
while (reader.Position < finalPosition)
{
// this checks that we have at least the minimum amount of data to retrieve a header
if (reader.Available(MessageHeaderSize) == false)
{
break;
}
var offset = reader.ReadInt64();
var messageSize = reader.ReadInt32();
// if the stream does not have enough left in the payload, we got only a partial message
if (reader.Available(messageSize) == false)
{
throw new BufferUnderRunException($"Message header size of {MessageHeaderSize} is not fully available.");
}
try {
messages = messages.AddRange(reader.ReadMessage(messageSize, offset, partitionId));
} catch (EndOfStreamException ex) {
throw new BufferUnderRunException($"Message size of {messageSize} is not available.", ex);
}
}
return(messages);
}
/// <summary>
/// Decode messages from a payload and assign it a given kafka offset.
/// </summary>
/// <param name="reader">The reader</param>
/// <param name="messageSize">The size of the message, for Crc Hash calculation</param>
/// <param name="offset">The offset represting the log entry from kafka of this message.</param>
/// <returns>Enumerable representing stream of messages decoded from byte[].</returns>
/// <remarks>The return type is an Enumerable as the message could be a compressed message set.</remarks>
public static IImmutableList <Message> ReadMessage(this IKafkaReader reader, int messageSize, long offset)
{
var crc = reader.ReadUInt32();
var crcHash = reader.ReadCrc(messageSize - 4);
if (crc != crcHash)
{
throw new CrcValidationException(crc, crcHash);
}
var messageVersion = reader.ReadByte();
var attribute = reader.ReadByte();
DateTimeOffset?timestamp = null;
if (messageVersion >= 1)
{
var milliseconds = reader.ReadInt64();
if (milliseconds >= 0)
{
timestamp = DateTimeOffset.FromUnixTimeMilliseconds(milliseconds);
}
}
var key = reader.ReadBytes();
var value = reader.ReadBytes();
var codec = (MessageCodec)(Message.CodecMask & attribute);
if (codec == MessageCodec.None)
{
return(ImmutableList <Message> .Empty.Add(new Message(value, key, attribute, offset, messageVersion, timestamp)));
}
var uncompressedBytes = value.ToUncompressed(codec);
using (var messageSetReader = new KafkaReader(uncompressedBytes)) {
return(messageSetReader.ReadMessages(codec));
}
}
protected abstract TAssignment DecodeAssignment(IKafkaReader reader, int expectedLength);
protected abstract TMetadata DecodeMetadata(string assignmentStrategy, IKafkaReader reader, int expectedLength);
protected override ByteTypeAssignment DecodeAssignment(IKafkaReader reader, int expectedLength)
{
return(new ByteTypeAssignment(reader.ReadBytes()));
}
protected override ByteTypeMetadata DecodeMetadata(string assignmentStrategy, IKafkaReader reader, int expectedLength)
{
return(new ByteTypeMetadata(assignmentStrategy, reader.ReadBytes()));
}
public static ErrorResponseCode ReadErrorCode(this IKafkaReader reader)
{
return((ErrorResponseCode)reader.ReadInt16());
}
/// <inheritdoc />
protected override ConsumerProtocolMetadata DecodeMetadata(string assignmentStrategy, IKafkaReader reader, int expectedLength)
{
var version = reader.ReadInt16();
var topicNames = new string[reader.ReadInt32()];
for (var t = 0; t < topicNames.Length; t++)
{
topicNames[t] = reader.ReadString();
}
var userData = reader.ReadBytes();
return(new ConsumerProtocolMetadata(topicNames, assignmentStrategy, userData, version));
}
/// <inheritdoc /> public abstract IMemberAssignment DecodeAssignment(IKafkaReader reader);
/// <inheritdoc /> public abstract IMemberMetadata DecodeMetadata(IKafkaReader reader);
