/// <summary> /// Consumes messages from Kafka. /// </summary> /// <param name="request">The request to send to Kafka.</param> /// <returns>A list of messages from Kafka.</returns> public List <Message> Consume(ConsumerRequest request) { List <Message> messages = new List <Message>(); using (KafkaConnection connection = new KafkaConnection(Server, Port)) { connection.Write(request.GetBytes()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength > 0) { byte[] data = connection.Read(dataLength); // TODO: need to check in on kafka error codes...assume all's good for now byte[] unbufferedData = data.Skip(2).ToArray(); int processed = 0; int length = unbufferedData.Length - 4; int messageSize = 0; while (processed <= length) { messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Skip(processed).Take(4).ToArray <byte>()), 0); messages.Add(Message.ParseFrom(unbufferedData.Skip(processed).Take(messageSize + 4).ToArray <byte>())); processed += 4 + messageSize; } } } return(messages); }
/// <summary> /// Consumes messages from Kafka. /// </summary> /// <param name="request">The request to send to Kafka.</param> /// <returns>A list of messages from Kafka.</returns> public List <Message> Consume(FetchRequest request) { List <Message> messages = new List <Message>(); using (KafkaConnection connection = new KafkaConnection(Server, Port)) { connection.Write(request.GetBytes()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength > 0) { byte[] data = connection.Read(dataLength); int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray <byte>()), 0); if (errorCode != KafkaException.NoError) { throw new KafkaException(errorCode); } // skip the error code and process the rest byte[] unbufferedData = data.Skip(2).ToArray(); int processed = 0; int length = unbufferedData.Length - 4; int messageSize = 0; while (processed <= length) { messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Skip(processed).Take(4).ToArray <byte>()), 0); messages.Add(Message.ParseFrom(unbufferedData.Skip(processed).Take(messageSize + 4).ToArray <byte>())); processed += 4 + messageSize; } } } return(messages); }
/// <summary> /// Executes a multi-fetch operation. /// </summary> /// <param name="request">The request to push to Kafka.</param> /// <returns> /// A list containing sets of messages. The message sets should match the request order. /// </returns> public List <List <Message> > Consume(MultiFetchRequest request) { int fetchRequests = request.ConsumerRequests.Count; List <List <Message> > messages = new List <List <Message> >(); using (KafkaConnection connection = new KafkaConnection(Server, Port)) { connection.Write(request.GetBytes()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength > 0) { byte[] data = connection.Read(dataLength); int position = 0; int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray <byte>()), 0); if (errorCode != KafkaException.NoError) { throw new KafkaException(errorCode); } // skip the error code and process the rest position = position + 2; for (int ix = 0; ix < fetchRequests; ix++) { messages.Add(new List <Message>()); int messageSetSize = BitConverter.ToInt32(BitWorks.ReverseBytes(data.Skip(position).Take(4).ToArray <byte>()), 0); position = position + 4; errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Skip(position).Take(2).ToArray <byte>()), 0); if (errorCode != KafkaException.NoError) { throw new KafkaException(errorCode); } // skip the error code and process the rest position = position + 2; byte[] messageSetBytes = data.Skip(position).ToArray <byte>().Take(messageSetSize).ToArray <byte>(); int processed = 0; int messageSize = 0; // dropped 2 bytes at the end...padding??? while (processed < messageSetBytes.Length - 2) { messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(messageSetBytes.Skip(processed).Take(4).ToArray <byte>()), 0); messages[ix].Add(Message.ParseFrom(messageSetBytes.Skip(processed).Take(messageSize + 4).ToArray <byte>())); processed += 4 + messageSize; } position = position + processed; } } } return(messages); }