public static PartitionMetadata ParseFrom(KafkaBinaryReader reader, Dictionary <int, Broker> brokers) { var errorCode = reader.ReadInt16(); var partitionId = reader.ReadInt32(); var leaderId = reader.ReadInt32(); Broker leader = null; if (leaderId != -1) { leader = brokers[leaderId]; } // list of all replicas var numReplicas = reader.ReadInt32(); var replicas = new List <Broker>(); for (var i = 0; i < numReplicas; ++i) { replicas.Add(brokers[reader.ReadInt32()]); } // list of in-sync replicas var numIsr = reader.ReadInt32(); var isrs = new List <Broker>(); for (var i = 0; i < numIsr; ++i) { isrs.Add(brokers[reader.ReadInt32()]); } return(new PartitionMetadata(partitionId, leader, replicas, isrs)); }
internal static IList <BufferedMessageSet> ParseMultiFrom(KafkaBinaryReader reader, int size, int count) { var result = new List <BufferedMessageSet>(); if (size == 0) { return(result); } int readed = 0; short errorCode = reader.ReadInt16(); readed += 2; if (errorCode != KafkaException.NoError) { throw new KafkaException(errorCode); } for (int i = 0; i < count; i++) { int partSize = reader.ReadInt32(); readed += 4; var item = ParseFrom(reader, partSize); readed += partSize; result.Add(item); } if (size != readed) { throw new KafkaException(KafkaException.InvalidRetchSizeCode); } return(result); }
internal static IList <BufferedMessageSet> ParseMultiFrom(KafkaBinaryReader reader, int size, int count, List <long> initialOffsets) { var result = new List <BufferedMessageSet>(); if (size == 0) { return(result); } int readed = 0; short errorCode = reader.ReadInt16(); readed += 2; if (errorCode != KafkaException.NoError) { throw new KafkaException(errorCode); } for (int i = 0; i < count; i++) { int partSize = reader.ReadInt32(); readed += 4; var item = ParseFrom(reader, partSize, initialOffsets[i]); readed += partSize; result.Add(item); var totalSetSize = item.SetSize + 2;// 2 is the size of int16 that is the error info if (totalSetSize != partSize) { break; } } return(result); }
public ProducerResponse ParseFrom(KafkaBinaryReader reader) { var size = reader.ReadInt32(); var correlationId = reader.ReadInt32(); var topicCount = reader.ReadInt32(); var statuses = new Dictionary <TopicAndPartition, ProducerResponseStatus>(); for (var i = 0; i < topicCount; ++i) { var topic = reader.ReadShortString(); var partitionCount = reader.ReadInt32(); for (var p = 0; p < partitionCount; ++p) { var partitionId = reader.ReadInt32(); var error = reader.ReadInt16(); var offset = reader.ReadInt64(); var topicAndPartition = new TopicAndPartition(topic, partitionId); statuses.Add(topicAndPartition, new ProducerResponseStatus { Error = ErrorMapper.ToError(error), Offset = offset }); } } return(new ProducerResponse(correlationId, statuses)); }
public static LeaveGroupResponse ParseFrom(KafkaBinaryReader reader) { var size = reader.ReadInt32(); var correlationid = reader.ReadInt32(); var error = reader.ReadInt16(); return(new LeaveGroupResponse(error)); }
internal static BufferedMessageSet ParseFrom(KafkaBinaryReader reader, int size, long initialOffset) { if (size == 0) { return(new BufferedMessageSet(Enumerable.Empty <Message>(), initialOffset)); } short errorCode = reader.ReadInt16(); if (errorCode != ErrorMapping.NoError) { if (errorCode == ErrorMapping.OffsetOutOfRangeCode) { return(new BufferedMessageSet(Enumerable.Empty <Message>(), errorCode, initialOffset)); } throw new KafkaException(errorCode); } int readed = 2; if (readed == size) { return(new BufferedMessageSet(Enumerable.Empty <Message>(), initialOffset)); } var messages = new List <Message>(); do { int msgSize = reader.ReadInt32(); readed += 4; int sizeNotUsed = size - readed; if (msgSize > sizeNotUsed || msgSize < 0) { if (messages.Count == 0 || msgSize < 0) { string errorMessage = String.Format( "Invalid message size. Read size = {0}, the remaining data size = {1} (possible causes (1) a single message larger than the fetch size; (2) log corruption)", msgSize, sizeNotUsed); throw new InvalidMessageSizeException(errorMessage); } return(new BufferedMessageSet(messages, initialOffset)); } Message msg = Message.ParseFrom(reader, msgSize); readed += msgSize; messages.Add(msg); }while (readed < size); if (size != readed) { throw new KafkaException(ErrorMapping.InvalidFetchSizeCode); } return(new BufferedMessageSet(messages, initialOffset)); }
internal static PartitionData ParseFrom(KafkaBinaryReader reader) { var partition = reader.ReadInt32(); var error = reader.ReadInt16(); var highWatermark = reader.ReadInt64(); var messageSetSize = reader.ReadInt32(); var bufferedMessageSet = BufferedMessageSet.ParseFrom(reader, messageSetSize, partition); return(new PartitionData(partition, ErrorMapper.ToError(error), bufferedMessageSet)); }
public static string ReadShortString(KafkaBinaryReader reader, string encoding) { var size = reader.ReadInt16(); if (size < 0) { return null; } var bytes = reader.ReadBytes(size); Encoding encoder = Encoding.GetEncoding(encoding); return encoder.GetString(bytes); }
public static GroupCoordinatorResponse ParseFrom(KafkaBinaryReader reader) { var size = reader.ReadInt32(); var correlationid = reader.ReadInt32(); var error = reader.ReadInt16(); var coordinatorid = reader.ReadInt32(); var coordinatorhost = reader.ReadShortString(); var coordinatorport = reader.ReadInt32(); return(new GroupCoordinatorResponse(error, coordinatorid, coordinatorhost, coordinatorport)); }
internal static TopicMetadata ParseFrom(KafkaBinaryReader reader, Dictionary <int, Broker> brokers) { var errorCode = reader.ReadInt16(); var topic = BitWorks.ReadShortString(reader, AbstractRequest.DefaultEncoding); var numPartitions = reader.ReadInt32(); var partitionsMetadata = new List <PartitionMetadata>(); for (var i = 0; i < numPartitions; i++) { partitionsMetadata.Add(PartitionMetadata.ParseFrom(reader, brokers)); } return(new TopicMetadata(topic, partitionsMetadata, ErrorMapper.ToError(errorCode))); }
public void GetBytesValid() { const string topicName = "topic"; var requestInfo = new Dictionary <string, List <PartitionOffsetRequestInfo> >(); requestInfo[topicName] = new List <PartitionOffsetRequestInfo>() { new PartitionOffsetRequestInfo(0, OffsetRequest.LatestTime, 10) }; var request = new OffsetRequest(requestInfo); // format = len(request) + requesttype + version + correlation id + client id + replica id + request info count + request infos int count = 2 + 2 + 4 + 2 + 4 + 4 + 4 + BitWorks.GetShortStringLength("topic", AbstractRequest.DefaultEncoding) + 4 + 4 + 8 + 4; var ms = new MemoryStream(); request.WriteTo(ms); byte[] bytes = ms.ToArray(); Assert.IsNotNull(bytes); Assert.AreEqual(count, bytes.Length); var reader = new KafkaBinaryReader(ms); reader.ReadInt32().Should().Be(count - 4); // length reader.ReadInt16().Should().Be((short)RequestTypes.Offsets); // request type reader.ReadInt16().Should().Be(0); // version reader.ReadInt32().Should().Be(0); // correlation id string.IsNullOrEmpty(reader.ReadShortString()).Should().BeTrue(); // client id reader.ReadInt32().Should().Be(-1); // replica id reader.ReadInt32().Should().Be(1); // request info count reader.ReadShortString().Should().Be("topic"); reader.ReadInt32().Should().Be(1); // info count reader.ReadInt32().Should().Be(0); // partition id reader.ReadInt64().Should().Be(OffsetRequest.LatestTime); // time reader.ReadInt32().Should().Be(10); // max offset }
public static PartitionOffsetsResponse ReadFrom(KafkaBinaryReader reader) { var partitionId = reader.ReadInt32(); var error = reader.ReadInt16(); var numOffsets = reader.ReadInt32(); var offsets = new List <long>(); for (var o = 0; o < numOffsets; ++o) { offsets.Add(reader.ReadInt64()); } return(new PartitionOffsetsResponse(partitionId, (ErrorMapping)Enum.Parse(typeof(ErrorMapping), error.ToString(CultureInfo.InvariantCulture)), offsets)); }
internal static BufferedMessageSet ParseFrom(KafkaBinaryReader reader, int size) { if (size == 0) { return(new BufferedMessageSet(Enumerable.Empty <Message>())); } short errorCode = reader.ReadInt16(); if (errorCode != KafkaException.NoError) { throw new KafkaException(errorCode); } int readed = 2; if (readed == size) { return(new BufferedMessageSet(Enumerable.Empty <Message>())); } var messages = new List <Message>(); do { int msgSize = reader.ReadInt32(); readed += 4; Message msg = Message.ParseFrom(reader, msgSize); readed += msgSize; messages.Add(msg); }while (readed < size); if (size != readed) { throw new KafkaException(KafkaException.InvalidRetchSizeCode); } return(new BufferedMessageSet(messages)); }
internal static TopicMetadata ParseFrom(KafkaBinaryReader reader, Dictionary<int, Broker> brokers) { var errorCode = reader.ReadInt16(); var topic = BitWorks.ReadShortString(reader, AbstractRequest.DefaultEncoding); var numPartitions = reader.ReadInt32(); var partitionsMetadata = new List<PartitionMetadata>(); for (int i = 0; i < numPartitions; i++) { partitionsMetadata.Add(PartitionMetadata.ParseFrom(reader, brokers)); } return new TopicMetadata(topic, partitionsMetadata, ErrorMapper.ToError(errorCode)); }