private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, ProduceResponse response)
{
if (response == null)
{
return(false);
}
var groupedTopics = response.Topics.GroupBy(t => t.TopicName).ToList();
writer.Write(groupedTopics.Count);
foreach (var topic in groupedTopics)
{
var partitions = topic.ToList();
writer.Write(topic.Key)
.Write(partitions.Count);
foreach (var partition in partitions)
{
writer.Write(partition.PartitionId)
.Write(partition.ErrorCode)
.Write(partition.Offset);
if (context.ApiVersion >= 2)
{
writer.Write(partition.Timestamp.ToUnixEpochMilliseconds() ?? -1L);
}
}
}
if (context.ApiVersion >= 1)
{
writer.Write((int?)response.ThrottleTime?.TotalMilliseconds ?? 0);
}
return(true);
}
protected override void EncodeBody(IKafkaWriter writer, IRequestContext context)
{
var totalCompressedBytes = 0;
var groupedPayloads = (from p in topics
group p by new {
p.topic, p.partition_id, p.Codec
} into tpc
select tpc).ToList();
writer.Write(acks)
.Write((int)timeout.TotalMilliseconds)
.Write(groupedPayloads.Count);
foreach (var groupedPayload in groupedPayloads)
{
var payloads = groupedPayload.ToList();
writer.Write(groupedPayload.Key.topic)
.Write(payloads.Count)
.Write(groupedPayload.Key.partition_id);
var compressedBytes = writer.Write(payloads.SelectMany(x => x.Messages), groupedPayload.Key.Codec);
Interlocked.Add(ref totalCompressedBytes, compressedBytes);
}
if (context.OnProduceRequestMessages != null)
{
var segment = writer.ToSegment();
context.OnProduceRequestMessages(topics.Sum(_ => _.Messages.Count), segment.Count, totalCompressedBytes);
}
}
public static int Write(this IKafkaWriter writer, IEnumerable <Message> messages, MessageCodec codec)
{
if (codec == MessageCodec.None)
{
using (writer.MarkForLength()) {
writer.Write(messages);
}
return(0);
}
using (var messageWriter = new KafkaWriter()) {
messageWriter.Write(messages);
var messageSet = messageWriter.ToSegment(false);
using (writer.MarkForLength()) { // messageset
writer.Write(0L); // offset
using (writer.MarkForLength()) { // message
using (writer.MarkForCrc()) {
writer.Write((byte)0) // message version
.Write((byte)codec) // attribute
.Write(-1); // key -- null, so -1 length
using (writer.MarkForLength()) { // value
var initialPosition = writer.Position;
writer.WriteCompressed(messageSet, codec);
var compressedMessageLength = writer.Position - initialPosition;
return(messageSet.Count - compressedMessageLength);
}
}
}
}
}
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, OffsetFetchResponse response)
{
if (response == null)
{
return(false);
}
var groupedTopics = response.responses.GroupBy(t => t.topic).ToList();
writer.Write(groupedTopics.Count);
foreach (var topic in groupedTopics)
{
var partitions = topic.ToList();
writer.Write(topic.Key)
.Write(partitions.Count); // partitionsPerTopic
foreach (var partition in partitions)
{
writer.Write(partition.partition_id)
.Write(partition.offset)
.Write(partition.metadata)
.Write(partition.error_code);
}
}
return(true);
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, ProduceResponse response)
{
if (response == null)
{
return(false);
}
var groupedTopics = response.responses.GroupBy(t => t.topic).ToList();
writer.Write(groupedTopics.Count);
foreach (var topic in groupedTopics)
{
var partitions = topic.ToList();
writer.Write(topic.Key)
.Write(partitions.Count);
foreach (var partition in partitions)
{
writer.Write(partition.partition_id)
.Write(partition.error_code)
.Write(partition.base_offset);
if (context.ApiVersion >= 2)
{
writer.Write(partition.timestamp?.ToUnixTimeMilliseconds() ?? -1L);
}
}
}
if (context.ApiVersion >= 1)
{
writer.Write((int?)response.throttle_time_ms?.TotalMilliseconds ?? 0);
}
return(true);
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, OffsetsResponse response)
{
if (response == null)
{
return(false);
}
var groupedTopics = response.responses.GroupBy(t => t.topic).ToList();
writer.Write(groupedTopics.Count);
foreach (var topic in groupedTopics)
{
var partitions = topic.ToList();
writer.Write(topic.Key)
.Write(partitions.Count);
foreach (var partition in partitions)
{
writer.Write(partition.partition_id)
.Write(partition.error_code);
if (context.ApiVersion == 0)
{
writer.Write(1)
.Write(partition.offset);
}
else
{
writer.Write(partition.timestamp?.ToUnixTimeMilliseconds() ?? 0L)
.Write(partition.offset);
}
}
}
return(true);
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, DescribeGroupsResponse response)
{
if (response == null)
{
return(false);
}
writer.Write(response.Groups.Count);
foreach (var group in response.Groups)
{
writer.Write(group.ErrorCode)
.Write(group.GroupId)
.Write(group.State)
.Write(group.ProtocolType)
.Write(group.Protocol);
var encoder = context.GetEncoder(group.ProtocolType);
writer.Write(group.Members.Count);
foreach (var member in group.Members)
{
writer.Write(member.MemberId)
.Write(member.ClientId)
.Write(member.ClientHost)
.Write(member.MemberMetadata, encoder)
.Write(member.MemberAssignment, encoder);
}
}
return(true);
}
protected override void EncodeBody(IKafkaWriter writer, IRequestContext context)
{
var topicGroups = topics.GroupBy(x => x.topic).ToList();
writer.Write(-1) // replica_id -- see above for rationale
.Write(topicGroups.Count);
foreach (var topicGroup in topicGroups)
{
var partitions = topicGroup.GroupBy(x => x.partition_id).ToList();
writer.Write(topicGroup.Key)
.Write(partitions.Count);
foreach (var partition in partitions)
{
foreach (var offset in partition)
{
writer.Write(partition.Key)
.Write(offset.timestamp);
if (context.ApiVersion == 0)
{
writer.Write(offset.max_num_offsets);
}
}
}
}
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, OffsetResponse response)
{
if (response == null)
{
return(false);
}
var groupedTopics = response.Topics.GroupBy(t => t.TopicName).ToList();
writer.Write(groupedTopics.Count);
foreach (var topic in groupedTopics)
{
var partitions = topic.ToList();
writer.Write(topic.Key)
.Write(partitions.Count);
foreach (var partition in partitions)
{
writer.Write(partition.PartitionId)
.Write(partition.ErrorCode);
if (context.ApiVersion == 0)
{
writer.Write(1)
.Write(partition.Offset);
}
else
{
writer.Write(partition.Timestamp.ToUnixEpochMilliseconds() ?? 0L)
.Write(partition.Offset);
}
}
}
return(true);
}
protected override void EncodeBody(IKafkaWriter writer, IRequestContext context)
{
var topicGroups = topics.GroupBy(x => x.topic).ToList();
writer.Write(-1) // replica_id -- see above
.Write((int)Math.Min(int.MaxValue, max_wait_time.TotalMilliseconds))
.Write(min_bytes);
if (context.ApiVersion >= 3)
{
writer.Write(max_bytes);
}
writer.Write(topicGroups.Count);
foreach (var topicGroup in topicGroups)
{
var partitions = topicGroup.GroupBy(x => x.partition_id).ToList();
writer.Write(topicGroup.Key)
.Write(partitions.Count);
foreach (var partition in partitions)
{
foreach (var fetch in partition)
{
writer.Write(partition.Key)
.Write(fetch.fetch_offset)
.Write(fetch.max_bytes);
}
}
}
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, FetchResponse response)
{
if (response == null)
{
return(false);
}
if (context.ApiVersion >= 1)
{
writer.Write((int?)response.ThrottleTime?.TotalMilliseconds ?? 0);
}
var groupedTopics = response.Topics.GroupBy(t => t.TopicName).ToList();
writer.Write(groupedTopics.Count);
foreach (var topic in groupedTopics)
{
var partitions = topic.ToList();
writer.Write(topic.Key)
.Write(partitions.Count); // partitionsPerTopic
foreach (var partition in partitions)
{
writer.Write(partition.PartitionId)
.Write(partition.ErrorCode)
.Write(partition.HighWaterMark)
.Write(partition.Messages);
}
}
return(true);
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, DescribeGroupsResponse response)
{
if (response == null)
{
return(false);
}
writer.Write(response.groups.Count);
foreach (var group in response.groups)
{
writer.Write(group.error_code)
.Write(group.group_id)
.Write(group.state)
.Write(group.protocol_type)
.Write(group.protocol);
var encoder = context.GetEncoder(group.protocol_type);
writer.Write(group.members.Count);
foreach (var member in group.members)
{
writer.Write(member.member_id)
.Write(member.client_id)
.Write(member.client_host)
.Write(member.member_metadata, encoder)
.Write(member.member_assignment, encoder);
}
}
return(true);
}
/// <summary>
/// Write raw compressed bytes (no length prefix)
/// </summary>
public static void WriteCompressed(this IKafkaWriter writer, ArraySegment <byte> bytes, MessageCodec codec)
{
switch (codec)
{
case MessageCodec.Gzip:
using (var gzip = new GZipStream(writer.Stream, ZipLevel, true)) {
gzip.Write(bytes.Array, bytes.Offset, bytes.Count);
gzip.Flush();
}
break;
case MessageCodec.Snappy:
#if DOTNETSTANDARD
var buffer = new byte[Snappy.SnappyCodec.GetMaxCompressedLength(bytes.Count)];
var size = Snappy.SnappyCodec.Compress(bytes.Array, bytes.Offset, bytes.Count, buffer, 0);
writer.Write(new ArraySegment <byte>(buffer, 0, size), false);
break;
#else
throw new NotSupportedException("Snappy codec is only supported on .net core");
#endif
default:
throw new NotSupportedException($"Codec type of {codec} is not supported.");
}
}
public static IKafkaWriter Write(this IKafkaWriter writer, IImmutableList <int> values)
{
writer.Write(values.Count);
foreach (var value in values)
{
writer.Write(value);
}
return(writer);
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, HeartbeatResponse response)
{
if (response == null)
{
return(false);
}
writer.Write(response.error_code);
return(true);
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, LeaveGroupResponse response)
{
if (response == null)
{
return(false);
}
writer.Write(response.ErrorCode);
writer.Write(response.ErrorCode);
return(true);
}
public static IKafkaWriter Write(this IKafkaWriter writer, IEnumerable <Message> messages)
{
foreach (var message in messages)
{
writer.Write(0L);
using (writer.MarkForLength()) {
message.WriteTo(writer);
}
}
return(writer);
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, MetadataResponse response)
{
if (response == null)
{
return(false);
}
writer.Write(response.brokers.Count);
foreach (var broker in response.brokers)
{
writer.Write(broker.Id)
.Write(broker.Host)
.Write(broker.Port);
if (context.ApiVersion >= 1)
{
writer.Write(broker.Rack);
}
}
if (context.ApiVersion >= 2)
{
writer.Write(response.cluster_id);
}
if (context.ApiVersion >= 1)
{
writer.Write(response.controller_id.GetValueOrDefault());
}
var groupedTopics = response.topic_metadata.GroupBy(t => new { TopicName = t.topic, ErrorCode = t.topic_error_code, IsInternal = t.is_internal }).ToList();
writer.Write(groupedTopics.Count);
foreach (var topic in groupedTopics)
{
var partitions = topic.SelectMany(_ => _.partition_metadata).ToList();
writer.Write(topic.Key.ErrorCode)
.Write(topic.Key.TopicName);
if (context.ApiVersion >= 1)
{
writer.Write(topic.Key.IsInternal.GetValueOrDefault());
}
writer.Write(partitions.Count); // partitionsPerTopic
foreach (var partition in partitions)
{
writer.Write(partition.partition_error_code)
.Write(partition.partition_id)
.Write(partition.leader)
.Write(partition.replicas)
.Write(partition.isr);
}
}
return(true);
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, MetadataResponse response)
{
if (response == null)
{
return(false);
}
writer.Write(response.Brokers.Count);
foreach (var broker in response.Brokers)
{
writer.Write(broker.BrokerId)
.Write(broker.Host)
.Write(broker.Port);
if (context.ApiVersion >= 1)
{
writer.Write(broker.Rack);
}
}
if (context.ApiVersion >= 2)
{
writer.Write(response.ClusterId);
}
if (context.ApiVersion >= 1)
{
writer.Write(response.ControllerId.GetValueOrDefault());
}
var groupedTopics = response.Topics.GroupBy(t => new { t.TopicName, t.ErrorCode, t.IsInternal }).ToList();
writer.Write(groupedTopics.Count);
foreach (var topic in groupedTopics)
{
var partitions = topic.SelectMany(_ => _.Partitions).ToList();
writer.Write(topic.Key.ErrorCode)
.Write(topic.Key.TopicName);
if (context.ApiVersion >= 1)
{
writer.Write(topic.Key.IsInternal.GetValueOrDefault());
}
writer.Write(partitions.Count); // partitionsPerTopic
foreach (var partition in partitions)
{
writer.Write(partition.ErrorCode)
.Write(partition.PartitionId)
.Write(partition.LeaderId)
.Write(partition.Replicas)
.Write(partition.Isrs);
}
}
return(true);
}
/// <summary>
/// Encodes a collection of messages, in order.
/// </summary>
/// <param name="writer">The writer</param>
/// <param name="messages">The collection of messages to encode together.</param>
/// <param name="includeLength">Whether to include the length at the start</param>
public static IKafkaWriter Write(this IKafkaWriter writer, IEnumerable <Message> messages, bool includeLength = true)
{
using (includeLength ? writer.MarkForLength() : Disposable.None) {
var offset = 0L;
foreach (var message in messages)
{
writer.Write(offset) // TODO: should this be incremented? offset++?
.Write(message);
}
}
return(writer);
}
public void WriteTo(IKafkaWriter writer)
{
using (writer.MarkForCrc()) {
writer.Write(MessageVersion)
.Write((byte)0);
if (MessageVersion >= 1)
{
writer.Write(Timestamp.GetValueOrDefault(DateTimeOffset.UtcNow).ToUnixTimeMilliseconds());
}
writer.Write(Key)
.Write(Value);
}
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, SyncGroupResponse response)
{
if (response == null)
{
return(false);
}
var encoder = context.GetEncoder();
writer.Write(response.ErrorCode)
.Write(response.MemberAssignment, encoder);
return(true);
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, GroupCoordinatorResponse response)
{
if (response == null)
{
return(false);
}
writer.Write(response.error_code)
.Write(response.Id)
.Write(response.Host)
.Write(response.Port);
return(true);
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, SyncGroupResponse response)
{
if (response == null)
{
return(false);
}
var encoder = context.GetEncoder(context.ProtocolType);
writer.Write(response.error_code)
.Write(response.member_assignment, encoder);
return(true);
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, SaslHandshakeResponse response)
{
if (response == null)
{
return(false);
}
writer.Write(response.error_code)
.Write(response.enabled_mechanisms.Count);
foreach (var mechanism in response.enabled_mechanisms)
{
writer.Write(mechanism);
}
return(true);
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, DeleteTopicsResponse response)
{
if (response == null)
{
return(false);
}
writer.Write(response.Topics.Count);
foreach (var topic in response.Topics)
{
writer.Write(topic.TopicName)
.Write(topic.ErrorCode);
}
return(true);
}
protected override void EncodeBody(IKafkaWriter writer, IRequestContext context)
{
writer.Write(group_id)
.Write(generation_id)
.Write(member_id)
.Write(group_assignments.Count);
var encoder = context.GetEncoder(context.ProtocolType);
foreach (var assignment in group_assignments)
{
writer.Write(assignment.member_id)
.Write(assignment.member_assignment, encoder);
}
}
/// <summary>
/// Encodes a message object
/// </summary>
/// <param name="writer">The writer</param>
/// <param name="message">Message data to encode.</param>
/// <param name="includeLength">Whether to include the length at the start</param>
/// <returns>Encoded byte[] representation of the message object.</returns>
/// <remarks>
/// Format:
/// Crc (Int32), MagicByte (Byte), Attribute (Byte), Key (Byte[]), Value (Byte[])
/// </remarks>
public static IKafkaWriter Write(this IKafkaWriter writer, Message message, bool includeLength = true)
{
using (includeLength ? writer.MarkForLength() : Disposable.None) {
using (writer.MarkForCrc()) {
writer.Write(message.MessageVersion)
.Write(message.Attribute);
if (message.MessageVersion >= 1)
{
writer.Write(message.Timestamp.GetValueOrDefault(DateTime.UtcNow).ToUnixEpochMilliseconds());
}
writer.Write(message.Key)
.Write(message.Value);
}
}
return(writer);
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, ListGroupsResponse response)
{
if (response == null)
{
return(false);
}
writer.Write(response.error_code)
.Write(response.groups.Count);
foreach (var group in response.groups)
{
writer.Write(group.group_id)
.Write(group.protocol_type);
}
return(true);
}
public static IKafkaWriter Write(this IKafkaWriter writer, IEnumerable <string> values, bool includeLength = false)
{
if (includeLength)
{
var valuesList = values.ToList();
writer.Write(valuesList.Count);
writer.Write(valuesList); // NOTE: !includeLength passed next time
return(writer);
}
foreach (var item in values)
{
writer.Write(item);
}
return(writer);
}
