Exemple #1
0
        /// <inheritdoc />
        public override IMemberAssignment DecodeAssignment(IKafkaReader reader)
        {
            var version = reader.ReadInt16();

            var topics     = new List <TopicPartition>();
            var topicCount = reader.ReadInt32();

            for (var t = 0; t < topicCount; t++)
            {
                var topicName = reader.ReadString();

                var partitionCount = reader.ReadInt32();
                for (var p = 0; p < partitionCount; p++)
                {
                    var partitionId = reader.ReadInt32();
                    topics.Add(new TopicPartition(topicName, partitionId));
                }
            }
            return(new ConsumerMemberAssignment(version, topics));
        }
Exemple #2
0
        /// <inheritdoc />
        public override IMemberMetadata DecodeMetadata(IKafkaReader reader)
        {
            var version    = reader.ReadInt16();
            var topicNames = new string[reader.ReadInt32()];

            for (var t = 0; t < topicNames.Length; t++)
            {
                topicNames[t] = reader.ReadString();
            }
            var userData = reader.ReadBytes();

            return(new ConsumerProtocolMetadata(version, topicNames, userData));
        }
Exemple #3
0
        /// <inheritdoc />
        protected override ConsumerMemberAssignment DecodeAssignment(IKafkaReader reader, int expectedLength)
        {
            var version = reader.ReadInt16();

            var topics     = new List <TopicPartition>();
            var topicCount = reader.ReadInt32();

            for (var t = 0; t < topicCount; t++)
            {
                var topicName = reader.ReadString();

                var partitionCount = reader.ReadInt32();
                for (var p = 0; p < partitionCount; p++)
                {
                    var partitionId = reader.ReadInt32();
                    topics.Add(new TopicPartition(topicName, partitionId));
                }
            }
            var userData = reader.ReadBytes();

            return(new ConsumerMemberAssignment(topics, userData, version));
        }
Exemple #4
0
        /// <inheritdoc />
        protected override ConsumerProtocolMetadata DecodeMetadata(string assignmentStrategy, IKafkaReader reader, int expectedLength)
        {
            var version    = reader.ReadInt16();
            var topicNames = new string[reader.ReadInt32()];

            for (var t = 0; t < topicNames.Length; t++)
            {
                topicNames[t] = reader.ReadString();
            }
            var userData = reader.ReadBytes();

            return(new ConsumerProtocolMetadata(topicNames, assignmentStrategy, userData, version));
        }
Exemple #5
0
        /// <summary>
        /// Decode a byte[] that represents a collection of messages.
        /// </summary>
        /// <param name="reader">The reader</param>
        /// <param name="partitionId">The partitionId messages are being read from.</param>
        /// <returns>Enumerable representing stream of messages decoded from byte[]</returns>
        public static IImmutableList <Message> ReadMessages(this IKafkaReader reader, int partitionId = 0)
        {
            var expectedLength = reader.ReadInt32();

            if (!reader.Available(expectedLength))
            {
                throw new BufferUnderRunException($"Message set size of {expectedLength} is not fully available.");
            }

            var messages      = ImmutableList <Message> .Empty;
            var finalPosition = reader.Position + expectedLength;

            while (reader.Position < finalPosition)
            {
                // this checks that we have at least the minimum amount of data to retrieve a header
                if (reader.Available(MessageHeaderSize) == false)
                {
                    break;
                }

                var offset      = reader.ReadInt64();
                var messageSize = reader.ReadInt32();

                // if the stream does not have enough left in the payload, we got only a partial message
                if (reader.Available(messageSize) == false)
                {
                    throw new BufferUnderRunException($"Message header size of {MessageHeaderSize} is not fully available.");
                }

                try {
                    messages = messages.AddRange(reader.ReadMessage(messageSize, offset, partitionId));
                } catch (EndOfStreamException ex) {
                    throw new BufferUnderRunException($"Message size of {messageSize} is not available.", ex);
                }
            }
            return(messages);
        }
Exemple #6
0
        /// <summary>
        /// Decode messages from a payload and assign it a given kafka offset.
        /// </summary>
        /// <param name="reader">The reader</param>
        /// <param name="messageSize">The size of the message, for Crc Hash calculation</param>
        /// <param name="offset">The offset represting the log entry from kafka of this message.</param>
        /// <param name="partitionId">The partition being read</param>
        /// <returns>Enumerable representing stream of messages decoded from byte[].</returns>
        /// <remarks>The return type is an Enumerable as the message could be a compressed message set.</remarks>
        public static IImmutableList <Message> ReadMessage(this IKafkaReader reader, int messageSize, long offset, int partitionId = 0)
        {
            var crc     = BitConverter.ToUInt32(reader.RawRead(4), 0);
            var crcHash = BitConverter.ToUInt32(reader.CrcHash(messageSize - 4), 0);

            if (crc != crcHash)
            {
                throw new CrcValidationException("Buffer did not match CRC validation.")
                      {
                          Crc = crc, CalculatedCrc = crcHash
                      }
            }
            ;

            var      messageVersion = reader.ReadByte();
            var      attribute      = reader.ReadByte();
            DateTime?timestamp      = null;

            if (messageVersion >= 1)
            {
                var milliseconds = reader.ReadInt64();
                if (milliseconds >= 0)
                {
                    timestamp = milliseconds.FromUnixEpochMilliseconds();
                }
            }
            var key = reader.ReadBytes();

            var codec = (MessageCodec)(Message.AttributeMask & attribute);

            switch (codec)
            {
            case MessageCodec.CodecNone: {
                var value = reader.ReadBytes();
                return(ImmutableList <Message> .Empty.Add(new Message(value, attribute, offset, partitionId, messageVersion, key, timestamp)));
            }

            case MessageCodec.CodecGzip: {
                var messageLength = reader.ReadInt32();
                var messageStream = new LimitedReadableStream(reader.BaseStream, messageLength);
                using (var gzipReader = new BigEndianBinaryReader(messageStream.Unzip())) {
                    return(gzipReader.ReadMessages(partitionId));
                }
            }

            default:
                throw new NotSupportedException($"Codec type of {codec} is not supported.");
            }
        }
Exemple #7
0
        /// <inheritdoc />
        public IMemberAssignment DecodeAssignment(IKafkaReader reader)
        {
            var expectedLength = reader.ReadInt32();

            if (!reader.HasBytes(expectedLength))
            {
                throw new BufferUnderRunException($"{ProtocolType} Assignment size of {expectedLength} is not fully available.");
            }

            if (expectedLength == 0)
            {
                return(null);
            }
            return(DecodeAssignment(reader, expectedLength));
        }
Exemple #8
0
        /// <inheritdoc />
        public IMemberMetadata DecodeMetadata(string assignmentStrategy, IKafkaReader reader)
        {
            var expectedLength = reader.ReadInt32();

            if (!reader.HasBytes(expectedLength))
            {
                throw new BufferUnderRunException($"{ProtocolType} Metadata size of {expectedLength} is not fully available.");
            }

            if (expectedLength == 0)
            {
                return(null);
            }
            return(DecodeMetadata(assignmentStrategy, reader, expectedLength));
        }