/// <summary>
/// Consumes messages from Kafka.
/// </summary>
/// <param name="request">The request to send to Kafka.</param>
/// <returns>A list of messages from Kafka.</returns>
public List<Message> Consume(FetchRequest request)
{
List<Message> messages = new List<Message>();
using (KafkaConnection connection = new KafkaConnection(Server, Port))
{
connection.Write(request.GetBytes());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = connection.Read(dataLength);
int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray<byte>()), 0);
if (errorCode != KafkaException.NoError)
{
throw new KafkaException(errorCode);
}
// skip the error code and process the rest
byte[] unbufferedData = data.Skip(2).ToArray();
int processed = 0;
int length = unbufferedData.Length - 4;
int messageSize = 0;
while (processed <= length)
{
messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Skip(processed).Take(4).ToArray<byte>()), 0);
messages.Add(Message.ParseFrom(unbufferedData.Skip(processed).Take(messageSize + 4).ToArray<byte>()));
processed += 4 + messageSize;
}
}
}
return messages;
}
public void GetBytesValidStructure()
{
string topicName = "topic";
FetchRequest request = new FetchRequest(topicName, 1, 10L, 100);
// REQUEST TYPE ID + TOPIC LENGTH + TOPIC + PARTITION + OFFSET + MAX SIZE
int requestSize = 2 + 2 + topicName.Length + 4 + 8 + 4;
byte[] bytes = request.GetBytes();
Assert.IsNotNull(bytes);
// add 4 bytes for the length of the message at the beginning
Assert.AreEqual(requestSize + 4, bytes.Length);
// first 4 bytes = the message length
Assert.AreEqual(25, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray<byte>()), 0));
// next 2 bytes = the request type
Assert.AreEqual((short)RequestType.Fetch, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray<byte>()), 0));
// next 2 bytes = the topic length
Assert.AreEqual((short)topicName.Length, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray<byte>()), 0));
// next few bytes = the topic
Assert.AreEqual(topicName, Encoding.ASCII.GetString(bytes.Skip(8).Take(topicName.Length).ToArray<byte>()));
// next 4 bytes = the partition
Assert.AreEqual(1, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(8 + topicName.Length).Take(4).ToArray<byte>()), 0));
// next 8 bytes = the offset
Assert.AreEqual(10, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(12 + topicName.Length).Take(8).ToArray<byte>()), 0));
// last 4 bytes = the max size
Assert.AreEqual(100, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(20 + +topicName.Length).Take(4).ToArray<byte>()), 0));
}
public FetchResponse Fetch(string topic, int partition, int correlationId, string clientId, long offset, int maxBytes, bool fromStart)
{
if (fromStart)
offset = OffsetRequest.LatestTime;
FetchRequest fetchRequest = new FetchRequest(correlationId, clientId);
fetchRequest.AddTopic(topic, partition, offset, maxBytes);
return Fetch(fetchRequest);
}
public FetchResponse Fetch(FetchRequest request)
{
using (KafkaConnection connection = new KafkaConnection(server, port))
{
connection.Write(request.GetRequestBytes().ToArray());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = connection.Read(dataLength);
var fetchResponse = new FetchResponse(data);
return fetchResponse;
}
return null;
}
}
public void IsValidNulltopic()
{
FetchRequest request = new FetchRequest(null, 1, 10L, 100);
Assert.IsFalse(request.IsValid());
}
public void IsValidNoTopic()
{
FetchRequest request = new FetchRequest(" ", 1, 10L, 100);
Assert.IsFalse(request.IsValid());
}
public void IsValidTrue()
{
FetchRequest request = new FetchRequest("topic", 1, 10L, 100);
Assert.IsTrue(request.IsValid());
}
public FetchResponse Fetch(string topic, int partition, int correlationId, string clientId, long offset, int maxBytes)
{
FetchRequest fetchRequest = new FetchRequest(correlationId, clientId);
fetchRequest.AddTopic(topic, partition, offset, maxBytes);
return Fetch(fetchRequest);
}
