Reads data from underlying stream using big endian bytes order for primitive types and UTF-8 encoding for strings.
Inheritance: System.IO.BinaryReader
 public void ShouldParseResponse()
 {
     var stream = new MemoryStream();
     var writer = new KafkaBinaryWriter(stream);
     writer.Write(1);
     writer.Write(123); // correlation id
     writer.Write(1); // topic count
     writer.WriteShortString("topic");
     writer.Write(1); // partition count
     writer.Write(999); // partition id
     writer.Write((short)ErrorMapping.NoError);
     writer.Write(3); // number of offsets
     writer.Write(111L);
     writer.Write(222L);
     writer.Write(333L);
     stream.Seek(0, SeekOrigin.Begin);
     var reader = new KafkaBinaryReader(stream);
     var response = new OffsetResponse.Parser().ParseFrom(reader);
     response.CorrelationId.Should().Be(123);
     response.ResponseMap.Count.Should().Be(1);
     var partitions = response.ResponseMap["topic"];
     partitions.Count.Should().Be(1);
     var info = partitions.First();
     info.Error.Should().Be(ErrorMapping.NoError);
     info.Offsets.Count.Should().Be(3);
     info.Offsets.SequenceEqual(new List<long>() {111L, 222L, 333L}).Should().BeTrue();
     info.PartitionId.Should().Be(999);
 }
        public void ShouldAbleToParseFetchResponse()
        {
            var stream = new MemoryStream();
            var writer = new KafkaBinaryWriter(stream);
            writer.Write(1);
            writer.Write(123); // correlation id
            writer.Write(1); // data count
            writer.WriteShortString("topic1");
            writer.Write(1); // partition count
            writer.Write(111); //partition id
            writer.Write((short)ErrorMapping.NoError);

            writer.Write(1011L); // hw
            var messageStream = new MemoryStream();
            var messageWriter = new KafkaBinaryWriter(messageStream);
            new BufferedMessageSet(new List<Message>() { new Message(new byte[100]) }, 0).WriteTo(messageWriter);
            writer.Write((int)messageStream.Length);
            writer.Write(messageStream.GetBuffer(), 0, (int)messageStream.Length);
            stream.Seek(0, SeekOrigin.Begin);
            var reader = new KafkaBinaryReader(stream);
            var response = new FetchResponse.Parser().ParseFrom(reader);
            var set = response.MessageSet("topic1", 111);
            set.Should().NotBeNull();
            var messages = set.Messages.ToList();
            messages.Count().Should().Be(1);
            messages.First().Payload.Length.Should().Be(100);
        }
        public void GetBytesValid()
        {
            const string topicName = "topic";
            var requestInfo = new Dictionary<string, List<PartitionOffsetRequestInfo>>();
            requestInfo[topicName] = new List<PartitionOffsetRequestInfo>() { new PartitionOffsetRequestInfo(0, OffsetRequest.LatestTime, 10) };
            var request = new OffsetRequest(requestInfo);

            // format = len(request) + requesttype + version + correlation id + client id + replica id + request info count + request infos
            int count = 2 + 2 + 4 + 2 + 4 + 4 + 4 +
                        BitWorks.GetShortStringLength("topic", AbstractRequest.DefaultEncoding) + 4 + 4 + 8 + 4;
            var ms = new MemoryStream();
            request.WriteTo(ms);
            byte[] bytes = ms.ToArray();
            Assert.IsNotNull(bytes);
            Assert.AreEqual(count, bytes.Length);

            var reader = new KafkaBinaryReader(ms);
            reader.ReadInt32().Should().Be(count - 4); // length
            reader.ReadInt16().Should().Be((short)RequestTypes.Offsets); // request type
            reader.ReadInt16().Should().Be(0); // version
            reader.ReadInt32().Should().Be(0); // correlation id
            string.IsNullOrEmpty(reader.ReadShortString()).Should().BeTrue(); // client id
            reader.ReadInt32().Should().Be(-1); // replica id
            reader.ReadInt32().Should().Be(1); // request info count
            reader.ReadShortString().Should().Be("topic");
            reader.ReadInt32().Should().Be(1); // info count
            reader.ReadInt32().Should().Be(0); // partition id
            reader.ReadInt64().Should().Be(OffsetRequest.LatestTime); // time
            reader.ReadInt32().Should().Be(10); // max offset
        }
Exemplo n.º 4
0
 public static string ReadShortString(KafkaBinaryReader reader, string encoding)
 {
     var size = reader.ReadInt16();
     if (size < 0)
     {
         return null;
     }
     var bytes = reader.ReadBytes(size);
     Encoding encoder = Encoding.GetEncoding(encoding);
     return encoder.GetString(bytes);
 }
 public void ShouldAbleToParseRequest()
 {
     var stream = new MemoryStream();
     var writer = new KafkaBinaryWriter(stream);
     writer.Write(1);
     writer.Write(100); // correlation id
     writer.Write(2); // broker count
     writer.Write(0); // broker id
     writer.WriteShortString("host1");
     writer.Write(9092); // port
     writer.Write(1); // broker id
     writer.WriteShortString("host2");
     writer.Write(9093); // port
     writer.Write(1); // topic count
     writer.Write((short)ErrorMapping.NoError);
     writer.WriteShortString("topic1");
     writer.Write(1); // partitions
     writer.Write((short)ErrorMapping.NoError);
     writer.Write(111); // partition id
     writer.Write(0); // leader broker id
     writer.Write(1); // num replicas
     writer.Write(1); // replica broker id
     writer.Write(1); // in sync replicas
     writer.Write(1); // in sync replica broker id
     stream.Seek(0, SeekOrigin.Begin);
     var reader = new KafkaBinaryReader(stream);
     var response = new TopicMetadataRequest.Parser().ParseFrom(reader);
     var enumerator = response.GetEnumerator();
     enumerator.MoveNext().Should().BeTrue();
     enumerator.Current.Topic.Should().Be("topic1");
     enumerator.Current.Error.Should().Be(ErrorMapping.NoError);
     var partitionEnumerator = enumerator.Current.PartitionsMetadata.GetEnumerator();
     partitionEnumerator.MoveNext().Should().BeTrue();
     partitionEnumerator.Current.PartitionId.Should().Be(111);
     var leader = partitionEnumerator.Current.Leader;
     leader.Id.Should().Be(0);
     leader.Host.Should().Be("host1");
     leader.Port.Should().Be(9092);
     var replicas = partitionEnumerator.Current.Replicas.ToList();
     replicas.Count.Should().Be(1);
     replicas.First().Id.Should().Be(1);
     replicas.First().Host.Should().Be("host2");
     replicas.First().Port.Should().Be(9093);
     var isrs = partitionEnumerator.Current.Isr.ToList();
     isrs.Count.Should().Be(1);
     isrs.First().Id.Should().Be(1);
     isrs.First().Host.Should().Be("host2");
     isrs.First().Port.Should().Be(9093);
 }
 public void ShouldAbleToWriteMessageSetWithExtraBytes()
 {
     var stream = new MemoryStream();
     var writer = new KafkaBinaryWriter(stream);
     var msg1 = new Message(new byte[101]) {Offset = 0};
     var msg2 = new Message(new byte[102]) {Offset = 1};
     var set = new BufferedMessageSet(new List<Message>() {msg1, msg2}, 0);
     set.WriteTo(writer);
     writer.Write(new byte[10]); // less than offset and size
     var size = (int) stream.Position;
     stream.Seek(0, SeekOrigin.Begin);
     var reader = new KafkaBinaryReader(stream);
     var newSet = BufferedMessageSet.ParseFrom(reader, size, 0);
     var messages = newSet.Messages.ToList();
     messages.Count().Should().Be(2);
     messages[0].Payload.Count().Should().Be(101);
     messages[1].Payload.Count().Should().Be(102);
 }
 public void ShouldAbleToParseResponse()
 {
     var stream = new MemoryStream();
     var writer = new KafkaBinaryWriter(stream);
     writer.Write(1);
     writer.Write(123); // correlation id
     writer.Write(1); // topic count
     writer.WriteShortString("topic");
     writer.Write(1); // partition count
     writer.Write(999); // partition id
     writer.Write((short)ErrorMapping.NoError); // error
     writer.Write(111L); // offset
     stream.Seek(0, SeekOrigin.Begin);
     var reader = new KafkaBinaryReader(stream);
     var response = new ProducerResponse.Parser().ParseFrom(reader);
     response.CorrelationId.Should().Be(123);
     response.Statuses.Count.Should().Be(1);
     var info = response.Statuses[new TopicAndPartition("topic", 999)];
     info.Error.Should().Be(ErrorMapping.NoError);
     info.Offset.Should().Be(111L);
 }
Exemplo n.º 8
0
        /**
        * A message. The format of an N byte message is the following:
        *
        * 1. 4 byte CRC32 of the message
        * 2. 1 byte "magic" identifier to allow format changes, value is 2 currently
        * 3. 1 byte "attributes" identifier to allow annotations on the message independent of the version (e.g. compression enabled, type of codec used)
        * 4. 4 byte key length, containing length K
        * 5. K byte key
        * 6. 4 byte payload length, containing length V
        * 7. V byte payload
        *
        */
        internal static Message ParseFrom(KafkaBinaryReader reader, long offset, int size, int partitionID)
        {
            Message result;
            int readed = 0;
            uint checksum = reader.ReadUInt32();
            readed += 4;
            byte magic = reader.ReadByte();
            readed++;

            byte[] payload;
            if (magic == 2 || magic == 0) // some producers (CLI) send magic 0 while others have value of 2
            {
                byte attributes = reader.ReadByte();
                readed++;
                var keyLength = reader.ReadInt32();
                readed += 4;
                byte[] key = null;
                if (keyLength != -1)
                {
                    key = reader.ReadBytes(keyLength);
                    readed += keyLength;
                }
                var payloadSize = reader.ReadInt32();
                readed += 4;
                payload = reader.ReadBytes(payloadSize);
                readed += payloadSize;
                result = new Message(payload, key, Messages.CompressionCodec.GetCompressionCodec(attributes & CompressionCodeMask))
                {
                    Offset = offset,
                    PartitionId = partitionID
                };
            }
            else
            {
                payload = reader.ReadBytes(size - DefaultHeaderSize);
                readed += size - DefaultHeaderSize;
                result = new Message(payload) { Offset = offset, PartitionId = partitionID };
            }

            if (size != readed)
            {
                throw new KafkaException(ErrorMapping.InvalidFetchSizeCode);
            }

            return result;
        }
Exemplo n.º 9
0
 /// <summary>
 /// Try to show the payload as decoded to UTF-8.
 /// </summary>
 /// <returns>The decoded payload as string.</returns>
 public override string ToString()
 {
     using (var reader = new KafkaBinaryReader(this.MessageBuffer))
     {
         return ParseFrom(reader, this.Size);
     }
 }
Exemplo n.º 10
0
 public override string ToString()
 {
     using (var reader = new KafkaBinaryReader(this.RequestBuffer))
     {
         return ParseFrom(reader, this.TotalSize);
     }
 }
Exemplo n.º 11
0
        public static string ParseFrom(KafkaBinaryReader reader, int count, bool skipReqInfo = false)
        {
            Guard.Assert<ArgumentNullException>(() => reader != null);
            var sb = new StringBuilder();

            if (!skipReqInfo)
            {
                sb.Append("Request size: ");
                sb.Append(reader.ReadInt32());
                sb.Append(", RequestId: ");
                short reqId = reader.ReadInt16();
                sb.Append(reqId);
                sb.Append("(");
                sb.Append((RequestTypes)reqId);
                sb.Append(")");
            }

            sb.Append(", Topic: ");
            string topic = reader.ReadTopic(DefaultEncoding);
            sb.Append(topic);
            sb.Append(", Partition: ");
            sb.Append(reader.ReadInt32());
            sb.Append(", Set size: ");
            sb.Append(reader.ReadInt32());
            int size = count - DefaultHeaderSize - GetTopicLength(topic);
            sb.Append(", Set {");
            sb.Append(BufferedMessageSet.ParseFrom(reader, size));
            sb.Append("}");
            return sb.ToString();
        }
        public static BufferedMessageSet Decompress(Message message, int partition)
        {
            switch (message.CompressionCodec)
            {
                case CompressionCodecs.DefaultCompressionCodec:
                case CompressionCodecs.GZIPCompressionCodec:
                    byte[] inputBytes = message.Payload;
                    using (var outputStream = new MemoryStream())
                    {
                        using (var inputStream = new MemoryStream(inputBytes))
                        {
                            using (var gzipInputStream = new GZipStream(inputStream, CompressionMode.Decompress))
                            {
                                try
                                {
                                    gzipInputStream.CopyTo(outputStream);
                                    gzipInputStream.Close();
                                }
                                catch (IOException ex)
                                {
                                    Logger.InfoFormat("Error while reading from the GZIP input stream: {0}", ex.FormatException());
                                    throw;
                                }
                            }
                        }

                        outputStream.Position = 0;
                        using (var reader = new KafkaBinaryReader(outputStream))
                        {
                            return BufferedMessageSet.ParseFrom(reader, (int)outputStream.Length, partition);
                        }
                    }

                case CompressionCodecs.SnappyCompressionCodec:
                    try
                    {
                        using (var stream = new MemoryStream(SnappyHelper.Decompress(message.Payload)))
                        {
                            using (var reader = new KafkaBinaryReader(stream))
                            {
                                return BufferedMessageSet.ParseFrom(reader, (int)stream.Length, partition);
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        Logger.ErrorFormat("Error while reading from the Snappy input stream  {0}", ex.FormatException());
                        throw;
                    }

                default:
                    throw new UnknownCodecException(String.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}", message.CompressionCodec));
            }
        }
Exemplo n.º 13
0
 internal static Broker ParseFrom(KafkaBinaryReader reader)
 {
     var id = reader.ReadInt32();
     var host = BitWorks.ReadShortString(reader, AbstractRequest.DefaultEncoding);
     var port = reader.ReadInt32();
     return new Broker(id, host, port);
 }
Exemplo n.º 14
0
        /// <summary>
        /// Creates string representation of message
        /// </summary>
        /// <param name="reader">
        /// The reader.
        /// </param>
        /// <param name="count">
        /// The count.
        /// </param>
        /// <returns>
        /// String representation of message
        /// </returns>
        public static string ParseFrom(KafkaBinaryReader reader, int count)
        {
            Guard.Assert<ArgumentNullException>(() => reader != null);
            var sb = new StringBuilder();
            int payloadSize = count - DefaultHeaderSize;
            sb.Append("Magic: ");
            sb.Append(reader.ReadByte());
            sb.Append(", Checksum: ");
            for (int i = 0; i < 4; i++)
            {
                sb.Append("[");
                sb.Append(reader.ReadByte());
                sb.Append("]");
            }

            sb.Append(", topic: ");
            var encodedPayload = reader.ReadBytes(payloadSize);
            try
            {
                sb.Append(Encoding.UTF8.GetString(encodedPayload));
            }
            catch (Exception)
            {
                sb.Append("n/a");
            }

            return sb.ToString();
        }
        public static PartitionMetadata ParseFrom(KafkaBinaryReader reader, Dictionary<int, Broker> brokers)
        {
            var errorCode = reader.ReadInt16();
            var partitionId = reader.ReadInt32();
            var leaderId = reader.ReadInt32();
            Broker leader = null;
            if (leaderId != -1)
            {
                leader = brokers[leaderId];
            }

            // list of all replicas
            var numReplicas = reader.ReadInt32();
            var replicas = new List<Broker>();
            for (int i = 0; i < numReplicas; ++i)
            {
                replicas.Add(brokers[reader.ReadInt32()]);
            }

            // list of in-sync replicas
            var numIsr = reader.ReadInt32();
            var isrs = new List<Broker>();
            for (int i = 0; i < numIsr; ++i)
            {
                isrs.Add(brokers[reader.ReadInt32()]);
            }

            return new PartitionMetadata(partitionId, leader, replicas, isrs);
        }
Exemplo n.º 16
0
 public override string ToString()
 {
     using (var reader = new KafkaBinaryReader(this.RequestBuffer))
     {
         return ParseFrom(reader, (int)this.RequestBuffer.Length);
     }
 }
Exemplo n.º 17
0
        /// <summary>
        /// Helper method to get string representation of set
        /// </summary>
        /// <param name="reader">
        /// The reader.
        /// </param>
        /// <param name="count">
        /// The count.
        /// </param>
        /// <returns>
        /// String representation of set
        /// </returns>
        internal static string ParseFrom(KafkaBinaryReader reader, int count)
        {
            Guard.Assert<ArgumentNullException>(() => reader != null);
            var sb = new StringBuilder();
            int i = 1;
            while (reader.BaseStream.Position != reader.BaseStream.Length)
            {
                sb.Append("Message ");
                sb.Append(i);
                sb.Append(" {Length: ");
                int msgSize = reader.ReadInt32();
                sb.Append(msgSize);
                sb.Append(", ");
                sb.Append(Message.ParseFrom(reader, msgSize));
                sb.AppendLine("} ");
                i++;
            }

            return sb.ToString();
        }
 internal static PartitionData ParseFrom(KafkaBinaryReader reader)
 {
     var partition = reader.ReadInt32();
     var error = reader.ReadInt16();
     var highWatermark = reader.ReadInt64();
     var messageSetSize = reader.ReadInt32();
     var bufferedMessageSet = BufferedMessageSet.ParseFrom(reader, messageSetSize, partition);
     return new PartitionData(partition, ErrorMapper.ToError(error), bufferedMessageSet);
 }
Exemplo n.º 19
0
        public static string ParseFrom(KafkaBinaryReader reader, int count)
        {
            Guard.Assert<ArgumentNullException>(() => reader != null);

            var sb = new StringBuilder();
            sb.Append("Request size: ");
            sb.Append(reader.ReadInt32());
            sb.Append(", RequestId: ");
            short reqId = reader.ReadInt16();
            sb.Append(reqId);
            sb.Append("(");
            sb.Append((RequestTypes)reqId);
            sb.Append("), Single Requests: {");
            int i = 1;
            while (reader.BaseStream.Position != reader.BaseStream.Length)
            {
                sb.Append("Request ");
                sb.Append(i);
                sb.Append(" {");
                int msgSize = 0;
                sb.Append(ProducerRequest.ParseFrom(reader, msgSize));
                sb.AppendLine("} ");
                i++;
            }

            return sb.ToString();
        }
 public void ShouldParseEmptyMessageSet()
 {
     var stream = new MemoryStream();
     var reader = new KafkaBinaryReader(stream);
     var newSet = BufferedMessageSet.ParseFrom(reader, 0, 0);
     var messages = newSet.Messages.ToList();
     messages.Count().Should().Be(0);
 }
Exemplo n.º 21
0
 internal static TopicData ParseFrom(KafkaBinaryReader reader)
 {
     var topic = reader.ReadShortString();
     var partitionCount = reader.ReadInt32();
     var partitions = new PartitionData[partitionCount];
     for (int i = 0; i < partitionCount; i++)
     {
         partitions[i] = Producers.PartitionData.ParseFrom(reader);
     }
     return new TopicData(topic, partitions.OrderBy(x => x.Partition));
 }
Exemplo n.º 22
0
        internal static Message ParseFrom(KafkaBinaryReader reader, int size)
        {
            Message result;
            int readed = 0;
            byte magic = reader.ReadByte();
            readed++;
            byte[] checksum;
            byte[] payload;
            if (magic == 1)
            {
                byte attributes = reader.ReadByte();
                readed++;
                checksum = reader.ReadBytes(4);
                readed += 4;
                payload = reader.ReadBytes(size - (DefaultHeaderSize + 1));
                readed += size - (DefaultHeaderSize + 1);
                result = new Message(payload, checksum, Messages.CompressionCodec.GetCompressionCodec(attributes & CompressionCodeMask));
            }
            else
            {
                checksum = reader.ReadBytes(4);
                readed += 4;
                payload = reader.ReadBytes(size - DefaultHeaderSize);
                readed += size - DefaultHeaderSize;
                result = new Message(payload, checksum);
            }

            if (size != readed)
            {
                throw new KafkaException(KafkaException.InvalidRetchSizeCode);
            }

            return result;
        }