Example #1
0
        public static FetchResponse FromBytes(IRequestContext context, ArraySegment <byte> bytes)
        {
            using (var reader = new KafkaReader(bytes)) {
                TimeSpan?throttleTime = null;

                if (context.ApiVersion >= 1)
                {
                    throttleTime = TimeSpan.FromMilliseconds(reader.ReadInt32());
                }

                var topics     = new List <FetchResponse.Topic>();
                var topicCount = reader.ReadInt32();
                for (var t = 0; t < topicCount; t++)
                {
                    var topicName = reader.ReadString();

                    var partitionCount = reader.ReadInt32();
                    for (var p = 0; p < partitionCount; p++)
                    {
                        var partitionId         = reader.ReadInt32();
                        var errorCode           = (ErrorCode)reader.ReadInt16();
                        var highWaterMarkOffset = reader.ReadInt64();
                        var messages            = reader.ReadMessages();

                        topics.Add(new Topic(topicName, partitionId, highWaterMarkOffset, errorCode, messages));
                    }
                }
                return(new FetchResponse(topics, throttleTime));
            }
        }
Example #2
0
        public static OffsetsResponse FromBytes(IRequestContext context, ArraySegment <byte> bytes)
        {
            using (var reader = new KafkaReader(bytes)) {
                var topics     = new List <Topic>();
                var topicCount = reader.ReadInt32();
                for (var t = 0; t < topicCount; t++)
                {
                    var topicName = reader.ReadString();

                    var partitionCount = reader.ReadInt32();
                    for (var p = 0; p < partitionCount; p++)
                    {
                        var partitionId = reader.ReadInt32();
                        var errorCode   = (ErrorCode)reader.ReadInt16();

                        if (context.ApiVersion == 0)
                        {
                            var offsetsCount = reader.ReadInt32();
                            for (var o = 0; o < offsetsCount; o++)
                            {
                                var offset = reader.ReadInt64();
                                topics.Add(new Topic(topicName, partitionId, errorCode, offset));
                            }
                        }
                        else
                        {
                            var timestamp = reader.ReadInt64();
                            var offset    = reader.ReadInt64();
                            topics.Add(new Topic(topicName, partitionId, errorCode, offset, DateTimeOffset.FromUnixTimeMilliseconds(timestamp)));
                        }
                    }
                }
                return(new OffsetsResponse(topics));
            }
        }
Example #3
0
 public static LeaveGroupResponse FromBytes(IRequestContext context, ArraySegment <byte> bytes)
 {
     using (var reader = new KafkaReader(bytes)) {
         var errorCode = (ErrorCode)reader.ReadInt16();
         return(new LeaveGroupResponse(errorCode));
     }
 }
Example #4
0
        public static DescribeGroupsResponse FromBytes(IRequestContext context, ArraySegment <byte> bytes)
        {
            using (var reader = new KafkaReader(bytes)) {
                var groups = new Group[reader.ReadInt32()];
                for (var g = 0; g < groups.Length; g++)
                {
                    var errorCode    = (ErrorCode)reader.ReadInt16();
                    var groupId      = reader.ReadString();
                    var state        = reader.ReadString();
                    var protocolType = reader.ReadString();
                    var protocol     = reader.ReadString();

                    IMembershipEncoder encoder = null;
                    var members = new Member[reader.ReadInt32()];
                    for (var m = 0; m < members.Length; m++)
                    {
                        encoder = encoder ?? context.GetEncoder(protocolType);
                        var memberId         = reader.ReadString();
                        var clientId         = reader.ReadString();
                        var clientHost       = reader.ReadString();
                        var memberMetadata   = encoder.DecodeMetadata(protocol, reader);
                        var memberAssignment = encoder.DecodeAssignment(reader);
                        members[m] = new Member(memberId, clientId, clientHost, memberMetadata, memberAssignment);
                    }
                    groups[g] = new Group(errorCode, groupId, state, protocolType, protocol, members);
                }

                return(new DescribeGroupsResponse(groups));
            }
        }
        public static SyncGroupResponse FromBytes(IRequestContext context, ArraySegment <byte> bytes)
        {
            using (var reader = new KafkaReader(bytes)) {
                var errorCode = (ErrorCode)reader.ReadInt16();

                var encoder          = context.GetEncoder();
                var memberAssignment = encoder.DecodeAssignment(reader);
                return(new SyncGroupResponse(errorCode, memberAssignment));
            }
        }
        public static GroupCoordinatorResponse FromBytes(IRequestContext context, ArraySegment <byte> bytes)
        {
            using (var reader = new KafkaReader(bytes)) {
                var errorCode       = (ErrorCode)reader.ReadInt16();
                var coordinatorId   = reader.ReadInt32();
                var coordinatorHost = reader.ReadString();
                var coordinatorPort = reader.ReadInt32();

                return(new GroupCoordinatorResponse(errorCode, coordinatorId, coordinatorHost, coordinatorPort));
            }
        }
Example #7
0
 public static DeleteTopicsResponse FromBytes(IRequestContext context, ArraySegment <byte> bytes)
 {
     using (var reader = new KafkaReader(bytes)) {
         var topics = new Topic[reader.ReadInt32()];
         for (var i = 0; i < topics.Length; i++)
         {
             var topicName = reader.ReadString();
             var errorCode = reader.ReadErrorCode();
             topics[i] = new Topic(topicName, errorCode);
         }
         return(new DeleteTopicsResponse(topics));
     }
 }
        public static SaslHandshakeResponse FromBytes(IRequestContext context, ArraySegment <byte> bytes)
        {
            using (var reader = new KafkaReader(bytes)) {
                var errorCode         = (ErrorCode)reader.ReadInt16();
                var enabledMechanisms = new string[reader.ReadInt32()];
                for (var m = 0; m < enabledMechanisms.Length; m++)
                {
                    enabledMechanisms[m] = reader.ReadString();
                }

                return(new SaslHandshakeResponse(errorCode, enabledMechanisms));
            }
        }
        public static ListGroupsResponse FromBytes(IRequestContext context, ArraySegment <byte> bytes)
        {
            using (var reader = new KafkaReader(bytes)) {
                var errorCode = (ErrorCode)reader.ReadInt16();
                var groups    = new Group[reader.ReadInt32()];
                for (var g = 0; g < groups.Length; g++)
                {
                    var groupId      = reader.ReadString();
                    var protocolType = reader.ReadString();
                    groups[g] = new Group(groupId, protocolType);
                }

                return(new ListGroupsResponse(errorCode, groups));
            }
        }
Example #10
0
        public static ApiVersionsResponse FromBytes(IRequestContext context, ArraySegment <byte> bytes)
        {
            using (var reader = new KafkaReader(bytes)) {
                var errorCode = (ErrorCode)reader.ReadInt16();

                var apiKeys = new VersionSupport[reader.ReadInt32()];
                for (var i = 0; i < apiKeys.Length; i++)
                {
                    var apiKey     = (ApiKey)reader.ReadInt16();
                    var minVersion = reader.ReadInt16();
                    var maxVersion = reader.ReadInt16();
                    apiKeys[i] = new VersionSupport(apiKey, minVersion, maxVersion);
                }
                return(new ApiVersionsResponse(errorCode, apiKeys));
            }
        }
        public static JoinGroupResponse FromBytes(IRequestContext context, ArraySegment <byte> bytes)
        {
            using (var reader = new KafkaReader(bytes)) {
                var errorCode     = (ErrorCode)reader.ReadInt16();
                var generationId  = reader.ReadInt32();
                var groupProtocol = reader.ReadString();
                var leaderId      = reader.ReadString();
                var memberId      = reader.ReadString();

                var encoder = context.GetEncoder(context.ProtocolType);
                var members = new Member[reader.ReadInt32()];
                for (var m = 0; m < members.Length; m++)
                {
                    var id       = reader.ReadString();
                    var metadata = encoder.DecodeMetadata(groupProtocol, reader);
                    members[m] = new Member(id, metadata);
                }

                return(new JoinGroupResponse(errorCode, generationId, groupProtocol, leaderId, memberId, members));
            }
        }
Example #12
0
        public static ProduceResponse FromBytes(IRequestContext context, ArraySegment <byte> bytes)
        {
            using (var reader = new KafkaReader(bytes)) {
                TimeSpan?throttleTime = null;

                var topics     = new List <Topic>();
                var topicCount = reader.ReadInt32();
                for (var i = 0; i < topicCount; i++)
                {
                    var topicName = reader.ReadString();

                    var partitionCount = reader.ReadInt32();
                    for (var j = 0; j < partitionCount; j++)
                    {
                        var            partitionId = reader.ReadInt32();
                        var            errorCode   = (ErrorCode)reader.ReadInt16();
                        var            offset      = reader.ReadInt64();
                        DateTimeOffset?timestamp   = null;

                        if (context.ApiVersion >= 2)
                        {
                            var milliseconds = reader.ReadInt64();
                            if (milliseconds >= 0)
                            {
                                timestamp = DateTimeOffset.FromUnixTimeMilliseconds(milliseconds);
                            }
                        }

                        topics.Add(new Topic(topicName, partitionId, errorCode, offset, timestamp));
                    }
                }

                if (context.ApiVersion >= 1)
                {
                    throttleTime = TimeSpan.FromMilliseconds(reader.ReadInt32());
                }
                return(new ProduceResponse(topics, throttleTime));
            }
        }
Example #13
0
        public static OffsetCommitResponse FromBytes(IRequestContext context, ArraySegment <byte> bytes)
        {
            using (var reader = new KafkaReader(bytes)) {
                var topics     = new List <TopicResponse>();
                var topicCount = reader.ReadInt32();
                for (var t = 0; t < topicCount; t++)
                {
                    var topicName = reader.ReadString();

                    var partitionCount = reader.ReadInt32();
                    for (var p = 0; p < partitionCount; p++)
                    {
                        var partitionId = reader.ReadInt32();
                        var errorCode   = (ErrorCode)reader.ReadInt16();

                        topics.Add(new TopicResponse(topicName, partitionId, errorCode));
                    }
                }

                return(new OffsetCommitResponse(topics));
            }
        }
Example #14
0
        /// <summary>
        /// Decode messages from a payload and assign it a given kafka offset.
        /// </summary>
        /// <param name="reader">The reader</param>
        /// <param name="messageSize">The size of the message, for Crc Hash calculation</param>
        /// <param name="offset">The offset represting the log entry from kafka of this message.</param>
        /// <returns>Enumerable representing stream of messages decoded from byte[].</returns>
        /// <remarks>The return type is an Enumerable as the message could be a compressed message set.</remarks>
        public static IImmutableList <Message> ReadMessage(this IKafkaReader reader, int messageSize, long offset)
        {
            var crc     = reader.ReadUInt32();
            var crcHash = reader.ReadCrc(messageSize - 4);

            if (crc != crcHash)
            {
                throw new CrcValidationException(crc, crcHash);
            }

            var            messageVersion = reader.ReadByte();
            var            attribute      = reader.ReadByte();
            DateTimeOffset?timestamp      = null;

            if (messageVersion >= 1)
            {
                var milliseconds = reader.ReadInt64();
                if (milliseconds >= 0)
                {
                    timestamp = DateTimeOffset.FromUnixTimeMilliseconds(milliseconds);
                }
            }
            var key   = reader.ReadBytes();
            var value = reader.ReadBytes();

            var codec = (MessageCodec)(Message.CodecMask & attribute);

            if (codec == MessageCodec.None)
            {
                return(ImmutableList <Message> .Empty.Add(new Message(value, key, attribute, offset, messageVersion, timestamp)));
            }
            var uncompressedBytes = value.ToUncompressed(codec);

            using (var messageSetReader = new KafkaReader(uncompressedBytes)) {
                return(messageSetReader.ReadMessages(codec));
            }
        }
        public static MetadataResponse FromBytes(IRequestContext context, ArraySegment <byte> bytes)
        {
            using (var reader = new KafkaReader(bytes)) {
                var brokers = new Server[reader.ReadInt32()];
                for (var b = 0; b < brokers.Length; b++)
                {
                    var    brokerId = reader.ReadInt32();
                    var    host     = reader.ReadString();
                    var    port     = reader.ReadInt32();
                    string rack     = null;
                    if (context.ApiVersion >= 1)
                    {
                        rack = reader.ReadString();
                    }

                    brokers[b] = new Server(brokerId, host, port, rack);
                }

                string clusterId = null;
                if (context.ApiVersion >= 2)
                {
                    clusterId = reader.ReadString();
                }

                int?controllerId = null;
                if (context.ApiVersion >= 1)
                {
                    controllerId = reader.ReadInt32();
                }

                var topics = new Topic[reader.ReadInt32()];
                for (var t = 0; t < topics.Length; t++)
                {
                    var  topicError = (ErrorCode)reader.ReadInt16();
                    var  topicName  = reader.ReadString();
                    bool?isInternal = null;
                    if (context.ApiVersion >= 1)
                    {
                        isInternal = reader.ReadBoolean();
                    }

                    var partitions = new Partition[reader.ReadInt32()];
                    for (var p = 0; p < partitions.Length; p++)
                    {
                        var partitionError = (ErrorCode)reader.ReadInt16();
                        var partitionId    = reader.ReadInt32();
                        var leaderId       = reader.ReadInt32();

                        var replicaCount = reader.ReadInt32();
                        var replicas     = replicaCount.Repeat(reader.ReadInt32).ToArray();

                        var isrCount = reader.ReadInt32();
                        var isrs     = isrCount.Repeat(reader.ReadInt32).ToArray();

                        partitions[p] = new Partition(partitionId, leaderId, partitionError, replicas, isrs);
                    }
                    topics[t] = new Topic(topicName, topicError, partitions, isInternal);
                }

                return(new MetadataResponse(brokers, topics, controllerId, clusterId));
            }
        }