public static int Write(this IKafkaWriter writer, IEnumerable <Message> messages, MessageCodec codec) { if (codec == MessageCodec.None) { using (writer.MarkForLength()) { writer.Write(messages); } return(0); } using (var messageWriter = new KafkaWriter()) { messageWriter.Write(messages); var messageSet = messageWriter.ToSegment(false); using (writer.MarkForLength()) { // messageset writer.Write(0L); // offset using (writer.MarkForLength()) { // message using (writer.MarkForCrc()) { writer.Write((byte)0) // message version .Write((byte)codec) // attribute .Write(-1); // key -- null, so -1 length using (writer.MarkForLength()) { // value var initialPosition = writer.Position; writer.WriteCompressed(messageSet, codec); var compressedMessageLength = writer.Position - initialPosition; return(messageSet.Count - compressedMessageLength); } } } } } }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, DescribeGroupsResponse response) { if (response == null) { return(false); } writer.Write(response.groups.Count); foreach (var group in response.groups) { writer.Write(group.error_code) .Write(group.group_id) .Write(group.state) .Write(group.protocol_type) .Write(group.protocol); var encoder = context.GetEncoder(group.protocol_type); writer.Write(group.members.Count); foreach (var member in group.members) { writer.Write(member.member_id) .Write(member.client_id) .Write(member.client_host) .Write(member.member_metadata, encoder) .Write(member.member_assignment, encoder); } } return(true); }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, OffsetFetchResponse response) { if (response == null) { return(false); } var groupedTopics = response.responses.GroupBy(t => t.topic).ToList(); writer.Write(groupedTopics.Count); foreach (var topic in groupedTopics) { var partitions = topic.ToList(); writer.Write(topic.Key) .Write(partitions.Count); // partitionsPerTopic foreach (var partition in partitions) { writer.Write(partition.partition_id) .Write(partition.offset) .Write(partition.metadata) .Write(partition.error_code); } } return(true); }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, ProduceResponse response) { if (response == null) { return(false); } var groupedTopics = response.responses.GroupBy(t => t.topic).ToList(); writer.Write(groupedTopics.Count); foreach (var topic in groupedTopics) { var partitions = topic.ToList(); writer.Write(topic.Key) .Write(partitions.Count); foreach (var partition in partitions) { writer.Write(partition.partition_id) .Write(partition.error_code) .Write(partition.base_offset); if (context.ApiVersion >= 2) { writer.Write(partition.timestamp?.ToUnixTimeMilliseconds() ?? -1L); } } } if (context.ApiVersion >= 1) { writer.Write((int?)response.throttle_time_ms?.TotalMilliseconds ?? 0); } return(true); }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, OffsetsResponse response) { if (response == null) { return(false); } var groupedTopics = response.responses.GroupBy(t => t.topic).ToList(); writer.Write(groupedTopics.Count); foreach (var topic in groupedTopics) { var partitions = topic.ToList(); writer.Write(topic.Key) .Write(partitions.Count); foreach (var partition in partitions) { writer.Write(partition.partition_id) .Write(partition.error_code); if (context.ApiVersion == 0) { writer.Write(1) .Write(partition.offset); } else { writer.Write(partition.timestamp?.ToUnixTimeMilliseconds() ?? 0L) .Write(partition.offset); } } } return(true); }
protected override void EncodeBody(IKafkaWriter writer, IRequestContext context) { var topicGroups = topics.GroupBy(x => x.topic).ToList(); writer.Write(-1) // replica_id -- see above .Write((int)Math.Min(int.MaxValue, max_wait_time.TotalMilliseconds)) .Write(min_bytes); if (context.ApiVersion >= 3) { writer.Write(max_bytes); } writer.Write(topicGroups.Count); foreach (var topicGroup in topicGroups) { var partitions = topicGroup.GroupBy(x => x.partition_id).ToList(); writer.Write(topicGroup.Key) .Write(partitions.Count); foreach (var partition in partitions) { foreach (var fetch in partition) { writer.Write(partition.Key) .Write(fetch.fetch_offset) .Write(fetch.max_bytes); } } } }
protected override void EncodeBody(IKafkaWriter writer, IRequestContext context) { var topicGroups = topics.GroupBy(x => x.topic).ToList(); writer.Write(-1) // replica_id -- see above for rationale .Write(topicGroups.Count); foreach (var topicGroup in topicGroups) { var partitions = topicGroup.GroupBy(x => x.partition_id).ToList(); writer.Write(topicGroup.Key) .Write(partitions.Count); foreach (var partition in partitions) { foreach (var offset in partition) { writer.Write(partition.Key) .Write(offset.timestamp); if (context.ApiVersion == 0) { writer.Write(offset.max_num_offsets); } } } } }
protected override void EncodeBody(IKafkaWriter writer, IRequestContext context) { var totalCompressedBytes = 0; var groupedPayloads = (from p in topics group p by new { p.topic, p.partition_id, p.Codec } into tpc select tpc).ToList(); writer.Write(acks) .Write((int)timeout.TotalMilliseconds) .Write(groupedPayloads.Count); foreach (var groupedPayload in groupedPayloads) { var payloads = groupedPayload.ToList(); writer.Write(groupedPayload.Key.topic) .Write(payloads.Count) .Write(groupedPayload.Key.partition_id); var compressedBytes = writer.Write(payloads.SelectMany(x => x.Messages), groupedPayload.Key.Codec); Interlocked.Add(ref totalCompressedBytes, compressedBytes); } if (context.OnProduceRequestMessages != null) { var segment = writer.ToSegment(); context.OnProduceRequestMessages(topics.Sum(_ => _.Messages.Count), segment.Count, totalCompressedBytes); } }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, DescribeGroupsResponse response) { if (response == null) { return(false); } writer.Write(response.Groups.Count); foreach (var group in response.Groups) { writer.Write(group.ErrorCode) .Write(group.GroupId) .Write(group.State) .Write(group.ProtocolType) .Write(group.Protocol); var encoder = context.GetEncoder(group.ProtocolType); writer.Write(group.Members.Count); foreach (var member in group.Members) { writer.Write(member.MemberId) .Write(member.ClientId) .Write(member.ClientHost) .Write(member.MemberMetadata, encoder) .Write(member.MemberAssignment, encoder); } } return(true); }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, OffsetResponse response) { if (response == null) { return(false); } var groupedTopics = response.Topics.GroupBy(t => t.TopicName).ToList(); writer.Write(groupedTopics.Count); foreach (var topic in groupedTopics) { var partitions = topic.ToList(); writer.Write(topic.Key) .Write(partitions.Count); foreach (var partition in partitions) { writer.Write(partition.PartitionId) .Write(partition.ErrorCode); if (context.ApiVersion == 0) { writer.Write(1) .Write(partition.Offset); } else { writer.Write(partition.Timestamp.ToUnixEpochMilliseconds() ?? 0L) .Write(partition.Offset); } } } return(true); }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, FetchResponse response) { if (response == null) { return(false); } if (context.ApiVersion >= 1) { writer.Write((int?)response.ThrottleTime?.TotalMilliseconds ?? 0); } var groupedTopics = response.Topics.GroupBy(t => t.TopicName).ToList(); writer.Write(groupedTopics.Count); foreach (var topic in groupedTopics) { var partitions = topic.ToList(); writer.Write(topic.Key) .Write(partitions.Count); // partitionsPerTopic foreach (var partition in partitions) { writer.Write(partition.PartitionId) .Write(partition.ErrorCode) .Write(partition.HighWaterMark) .Write(partition.Messages); } } return(true); }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, ProduceResponse response) { if (response == null) { return(false); } var groupedTopics = response.Topics.GroupBy(t => t.TopicName).ToList(); writer.Write(groupedTopics.Count); foreach (var topic in groupedTopics) { var partitions = topic.ToList(); writer.Write(topic.Key) .Write(partitions.Count); foreach (var partition in partitions) { writer.Write(partition.PartitionId) .Write(partition.ErrorCode) .Write(partition.Offset); if (context.ApiVersion >= 2) { writer.Write(partition.Timestamp.ToUnixEpochMilliseconds() ?? -1L); } } } if (context.ApiVersion >= 1) { writer.Write((int?)response.ThrottleTime?.TotalMilliseconds ?? 0); } return(true); }
public static IKafkaWriter Write(this IKafkaWriter writer, IImmutableList <int> values) { writer.Write(values.Count); foreach (var value in values) { writer.Write(value); } return(writer); }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, LeaveGroupResponse response) { if (response == null) { return(false); } writer.Write(response.error_code); writer.Write(response.error_code); return(true); }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, MetadataResponse response) { if (response == null) { return(false); } writer.Write(response.Brokers.Count); foreach (var broker in response.Brokers) { writer.Write(broker.BrokerId) .Write(broker.Host) .Write(broker.Port); if (context.ApiVersion >= 1) { writer.Write(broker.Rack); } } if (context.ApiVersion >= 2) { writer.Write(response.ClusterId); } if (context.ApiVersion >= 1) { writer.Write(response.ControllerId.GetValueOrDefault()); } var groupedTopics = response.Topics.GroupBy(t => new { t.TopicName, t.ErrorCode, t.IsInternal }).ToList(); writer.Write(groupedTopics.Count); foreach (var topic in groupedTopics) { var partitions = topic.SelectMany(_ => _.Partitions).ToList(); writer.Write(topic.Key.ErrorCode) .Write(topic.Key.TopicName); if (context.ApiVersion >= 1) { writer.Write(topic.Key.IsInternal.GetValueOrDefault()); } writer.Write(partitions.Count); // partitionsPerTopic foreach (var partition in partitions) { writer.Write(partition.ErrorCode) .Write(partition.PartitionId) .Write(partition.LeaderId) .Write(partition.Replicas) .Write(partition.Isrs); } } return(true); }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, MetadataResponse response) { if (response == null) { return(false); } writer.Write(response.brokers.Count); foreach (var broker in response.brokers) { writer.Write(broker.Id) .Write(broker.Host) .Write(broker.Port); if (context.ApiVersion >= 1) { writer.Write(broker.Rack); } } if (context.ApiVersion >= 2) { writer.Write(response.cluster_id); } if (context.ApiVersion >= 1) { writer.Write(response.controller_id.GetValueOrDefault()); } var groupedTopics = response.topic_metadata.GroupBy(t => new { TopicName = t.topic, ErrorCode = t.topic_error_code, IsInternal = t.is_internal }).ToList(); writer.Write(groupedTopics.Count); foreach (var topic in groupedTopics) { var partitions = topic.SelectMany(_ => _.partition_metadata).ToList(); writer.Write(topic.Key.ErrorCode) .Write(topic.Key.TopicName); if (context.ApiVersion >= 1) { writer.Write(topic.Key.IsInternal.GetValueOrDefault()); } writer.Write(partitions.Count); // partitionsPerTopic foreach (var partition in partitions) { writer.Write(partition.partition_error_code) .Write(partition.partition_id) .Write(partition.leader) .Write(partition.replicas) .Write(partition.isr); } } return(true); }
public void WriteTo(IKafkaWriter writer) { using (writer.MarkForCrc()) { writer.Write(MessageVersion) .Write((byte)0); if (MessageVersion >= 1) { writer.Write(Timestamp.GetValueOrDefault(DateTimeOffset.UtcNow).ToUnixTimeMilliseconds()); } writer.Write(Key) .Write(Value); } }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, DeleteTopicsResponse response) { if (response == null) { return(false); } writer.Write(response.Topics.Count); foreach (var topic in response.Topics) { writer.Write(topic.TopicName) .Write(topic.ErrorCode); } return(true); }
protected override void EncodeBody(IKafkaWriter writer, IRequestContext context) { writer.Write(group_id) .Write(generation_id) .Write(member_id) .Write(group_assignments.Count); var encoder = context.GetEncoder(context.ProtocolType); foreach (var assignment in group_assignments) { writer.Write(assignment.member_id) .Write(assignment.member_assignment, encoder); } }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, SaslHandshakeResponse response) { if (response == null) { return(false); } writer.Write(response.error_code) .Write(response.enabled_mechanisms.Count); foreach (var mechanism in response.enabled_mechanisms) { writer.Write(mechanism); } return(true); }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, ListGroupsResponse response) { if (response == null) { return(false); } writer.Write(response.error_code) .Write(response.groups.Count); foreach (var group in response.groups) { writer.Write(group.group_id) .Write(group.protocol_type); } return(true); }
public static IKafkaWriter Write(this IKafkaWriter writer, IEnumerable <string> values, bool includeLength = false) { if (includeLength) { var valuesList = values.ToList(); writer.Write(valuesList.Count); writer.Write(valuesList); // NOTE: !includeLength passed next time return(writer); } foreach (var item in values) { writer.Write(item); } return(writer); }
/// <summary> /// Write raw compressed bytes (no length prefix) /// </summary> public static void WriteCompressed(this IKafkaWriter writer, ArraySegment <byte> bytes, MessageCodec codec) { switch (codec) { case MessageCodec.Gzip: using (var gzip = new GZipStream(writer.Stream, ZipLevel, true)) { gzip.Write(bytes.Array, bytes.Offset, bytes.Count); gzip.Flush(); } break; case MessageCodec.Snappy: #if DOTNETSTANDARD var buffer = new byte[Snappy.SnappyCodec.GetMaxCompressedLength(bytes.Count)]; var size = Snappy.SnappyCodec.Compress(bytes.Array, bytes.Offset, bytes.Count, buffer, 0); writer.Write(new ArraySegment <byte>(buffer, 0, size), false); break; #else throw new NotSupportedException("Snappy codec is only supported on .net core"); #endif default: throw new NotSupportedException($"Codec type of {codec} is not supported."); } }
/// <summary> /// Encodes a message object /// </summary> /// <param name="writer">The writer</param> /// <param name="message">Message data to encode.</param> /// <param name="includeLength">Whether to include the length at the start</param> /// <returns>Encoded byte[] representation of the message object.</returns> /// <remarks> /// Format: /// Crc (Int32), MagicByte (Byte), Attribute (Byte), Key (Byte[]), Value (Byte[]) /// </remarks> public static IKafkaWriter Write(this IKafkaWriter writer, Message message, bool includeLength = true) { using (includeLength ? writer.MarkForLength() : Disposable.None) { using (writer.MarkForCrc()) { writer.Write(message.MessageVersion) .Write(message.Attribute); if (message.MessageVersion >= 1) { writer.Write(message.Timestamp.GetValueOrDefault(DateTime.UtcNow).ToUnixEpochMilliseconds()); } writer.Write(message.Key) .Write(message.Value); } } return(writer); }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, ApiVersionsResponse response) { if (response == null) { return(false); } writer.Write(response.ErrorCode) .Write(response.SupportedVersions.Count); foreach (var versionSupport in response.SupportedVersions) { writer.Write((short)versionSupport.ApiKey) .Write(versionSupport.MinVersion) .Write(versionSupport.MaxVersion); } return(true); }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, ApiVersionsResponse response) { if (response == null) { return(false); } writer.Write(response.error_code) .Write(response.api_versions.Count); foreach (var versionSupport in response.api_versions) { writer.Write((short)versionSupport.api_key) .Write(versionSupport.min_version) .Write(versionSupport.max_version); } return(true); }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, HeartbeatResponse response) { if (response == null) { return(false); } writer.Write(response.ErrorCode); return(true); }
/// <inheritdoc /> protected override void EncodeAssignment(IKafkaWriter writer, ConsumerMemberAssignment value) { var topicGroups = value.PartitionAssignments.GroupBy(x => x.TopicName).ToList(); writer.Write(value.Version) .Write(topicGroups.Count); foreach (var topicGroup in topicGroups) { var partitions = topicGroup.ToList(); writer.Write(topicGroup.Key) .Write(partitions.Count); foreach (var partition in partitions) { writer.Write(partition.PartitionId); } } }
public static IKafkaWriter Write(this IKafkaWriter writer, IEnumerable <Message> messages) { foreach (var message in messages) { writer.Write(0L); using (writer.MarkForLength()) { message.WriteTo(writer); } } return(writer); }
protected override void EncodeBody(IKafkaWriter writer, IRequestContext context) { writer.Write(group_id) .Write((int)session_timeout.TotalMilliseconds); if (context.ApiVersion >= 1) { writer.Write((int)rebalance_timeout.TotalMilliseconds); } writer.Write(member_id) .Write(protocol_type) .Write(group_protocols.Count); var encoder = context.GetEncoder(protocol_type); foreach (var protocol in group_protocols) { writer.Write(protocol.protocol_name) .Write(protocol.protocol_metadata, encoder); } }