private IEnumerable <ProduceResponse> DecodeProduceResponse(byte[] data)
{
var stream = new ReadByteStream(data);
var correlationId = stream.ReadInt();
var topicCount = stream.ReadInt();
for (int i = 0; i < topicCount; i++)
{
var topic = stream.ReadInt16String();
var partitionCount = stream.ReadInt();
for (int j = 0; j < partitionCount; j++)
{
var response = new ProduceResponse()
{
Topic = topic,
PartitionId = stream.ReadInt(),
Error = stream.ReadInt16(),
Offset = stream.ReadLong()
};
yield return(response);
}
}
}
public static Partition FromStream(ReadByteStream stream)
{
var partition = new Partition {
ErrorCode = stream.ReadInt16(),
PartitionId = stream.ReadInt(),
LeaderId = stream.ReadInt(),
Replicas = new List <int>(),
Isrs = new List <int>()
};
var numReplicas = stream.ReadInt();
for (int i = 0; i < numReplicas; i++)
{
partition.Replicas.Add(stream.ReadInt());
}
var numIsr = stream.ReadInt();
for (int i = 0; i < numIsr; i++)
{
partition.Isrs.Add(stream.ReadInt());
}
return(partition);
}
private IEnumerable <OffsetResponse> DecodeOffsetResponse(byte[] data)
{
var stream = new ReadByteStream(data);
var correlationId = stream.ReadInt();
var topicCount = stream.ReadInt();
for (int i = 0; i < topicCount; i++)
{
var topic = stream.ReadInt16String();
var partitionCount = stream.ReadInt();
for (int j = 0; j < partitionCount; j++)
{
var response = new OffsetResponse()
{
Topic = topic,
PartitionId = stream.ReadInt(),
Error = stream.ReadInt16(),
Offsets = new List <long>()
};
var offsetCount = stream.ReadInt();
for (int k = 0; k < offsetCount; k++)
{
response.Offsets.Add(stream.ReadLong());
}
yield return(response);
}
}
}
/// <summary>
/// Decode a byte[] that represents a collection of messages.
/// </summary>
/// <param name="messageSet">The byte[] encode as a message set from kafka.</param>
/// <returns>Enumerable representing stream of messages decoded from byte[]</returns>
public static IEnumerable <Message> DecodeMessageSet(byte[] messageSet)
{
var stream = new ReadByteStream(messageSet);
while (stream.HasData)
{
//this checks that we have at least the minimum amount of data to retrieve a header
if (stream.Available(MessageHeaderSize) == false)
{
yield break;
}
var offset = stream.ReadLong();
var messageSize = stream.ReadInt();
//if messagessize is greater than payload, our max buffer is insufficient.
if ((stream.Payload.Length - MessageHeaderSize) < messageSize)
{
throw new BufferUnderRunException(MessageHeaderSize, messageSize);
}
//if the stream does not have enough left in the payload, we got only a partial message
if (stream.Available(messageSize) == false)
{
yield break;
}
foreach (var message in DecodeMessage(offset, stream.ReadBytesFromStream(messageSize)))
{
yield return(message);
}
}
}
/// <summary>
/// Decode a byte[] that represents a collection of messages.
/// </summary>
/// <param name="messageSet">The byte[] encode as a message set from kafka.</param>
/// <returns>Enumerable representing stream of messages decoded from byte[]</returns>
public static IEnumerable <Message> DecodeMessageSet(byte[] messageSet)
{
var stream = new ReadByteStream(messageSet);
while (stream.HasData)
{
//if the message set hits against our max bytes wall on the fetch we will have a 1/2 completed message downloaded.
//the decode should guard against this situation
if (stream.Available(MinimumMessageSize) == false)
{
yield break;
}
var offset = stream.ReadLong();
var messageSize = stream.ReadInt();
if (stream.Available(messageSize) == false)
{
yield break;
}
foreach (var message in DecodeMessage(offset, stream.ReadBytesFromStream(messageSize)))
{
yield return(message);
}
}
}
private IEnumerable <FetchResponse> DecodeFetchResponse(byte[] data)
{
var stream = new ReadByteStream(data);
var correlationId = stream.ReadInt();
var topicCount = stream.ReadInt();
for (int i = 0; i < topicCount; i++)
{
var topic = stream.ReadInt16String();
var partitionCount = stream.ReadInt();
for (int j = 0; j < partitionCount; j++)
{
var partitionId = stream.ReadInt();
var response = new FetchResponse
{
Topic = topic,
PartitionId = partitionId,
Error = stream.ReadInt16(),
HighWaterMark = stream.ReadLong()
};
//note: dont use initializer here as it breaks stream position.
response.Messages = Message.DecodeMessageSet(stream.ReadIntPrefixedBytes())
.Select(x => { x.Meta.PartitionId = partitionId; return(x); });
yield return(response);
}
}
}
public static Broker FromStream(ReadByteStream stream)
{
return(new Broker
{
BrokerId = stream.ReadInt(),
Host = stream.ReadInt16String(),
Port = stream.ReadInt()
});
}
/// <summary>
/// Decode a byte[] that represents a collection of messages.
/// </summary>
/// <param name="messageSet">The byte[] encode as a message set from kafka.</param>
/// <returns>Enumerable representing stream of messages decoded from byte[]</returns>
public static IEnumerable <Message> DecodeMessageSet(byte[] messageSet)
{
var stream = new ReadByteStream(messageSet);
while (stream.HasData)
{
var offset = stream.ReadLong();
foreach (var message in DecodeMessage(offset, stream.ReadIntPrefixedBytes()))
{
yield return(message);
}
}
}
private IEnumerable <ConsumerMetadataResponse> DecodeConsumerMetadataResponse(byte[] data)
{
var stream = new ReadByteStream(data);
var correlationId = stream.ReadInt();
var response = new ConsumerMetadataResponse
{
Error = stream.ReadInt16(),
CoordinatorId = stream.ReadInt(),
CoordinatorHost = stream.ReadInt16String(),
CoordinatorPort = stream.ReadInt()
};
yield return(response);
}
public static Topic FromStream(ReadByteStream stream)
{
var topic = new Topic
{
ErrorCode = stream.ReadInt16(),
Name = stream.ReadInt16String(),
Partitions = new List <Partition>()
};
var numPartitions = stream.ReadInt();
for (int i = 0; i < numPartitions; i++)
{
topic.Partitions.Add(Partition.FromStream(stream));
}
return(topic);
}
/// <summary>
/// Decode messages from a payload and assign it a given kafka offset.
/// </summary>
/// <param name="offset">The offset represting the log entry from kafka of this message.</param>
/// <param name="payload">The byte[] encode as a message from kafka.</param>
/// <returns>Enumerable representing stream of messages decoded from byte[].</returns>
/// <remarks>The return type is an Enumerable as the message could be a compressed message set.</remarks>
public static IEnumerable <Message> DecodeMessage(long offset, byte[] payload)
{
var crc = payload.Take(4).ToArray();
var stream = new ReadByteStream(payload.Skip(4));
var hash = Crc32Provider.ComputeHash(stream.Payload);
if (crc.SequenceEqual(hash) == false)
{
throw new FailCrcCheckException("Payload did not match CRC validation.");
}
var message = new Message
{
Meta = new MessageMetadata {
Offset = offset
},
MagicNumber = stream.ReadByte(),
Attribute = stream.ReadByte(),
Key = stream.ReadIntPrefixedBytes()
};
var codec = (MessageCodec)(ProtocolConstants.AttributeCodeMask & message.Attribute);
switch (codec)
{
case MessageCodec.CodecNone:
message.Value = stream.ReadIntPrefixedBytes();
yield return(message);
break;
case MessageCodec.CodecGzip:
var gZipData = stream.ReadIntPrefixedBytes();
foreach (var m in DecodeMessageSet(Compression.Unzip(gZipData)))
{
yield return(m);
}
break;
default:
throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", codec));
}
}
/// <summary>
/// Decode a byte[] that represents a collection of messages.
/// </summary>
/// <param name="messageSet">The byte[] encode as a message set from kafka.</param>
/// <returns>Enumerable representing stream of messages decoded from byte[]</returns>
public static IEnumerable <Message> DecodeMessageSet(byte[] messageSet)
{
var stream = new ReadByteStream(messageSet);
while (stream.HasData)
{
var offset = stream.ReadLong();
var size = stream.ReadInt();
var payload = stream.ReadBytesFromStream(size);
if (payload.Length < size)
{
throw new InsufficientDataException(payload.Length, size);
}
foreach (var message in DecodeMessage(offset, payload))
{
yield return(message);
}
}
}
/// <summary>
/// Decode the metadata response from kafka server.
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
private MetadataResponse DecodeMetadataResponse(byte[] data)
{
var stream = new ReadByteStream(data);
var response = new MetadataResponse();
response.CorrelationId = stream.ReadInt();
var brokerCount = stream.ReadInt();
for (var i = 0; i < brokerCount; i++)
{
response.Brokers.Add(Broker.FromStream(stream));
}
var topicCount = stream.ReadInt();
for (var i = 0; i < topicCount; i++)
{
response.Topics.Add(Topic.FromStream(stream));
}
return(response);
}
