public void GetBytesValidFormat()
{
List<FetchRequest> requests = new List<FetchRequest>
{
new FetchRequest("topic a", 0, 0),
new FetchRequest("topic a", 0, 0),
new FetchRequest("topic b", 0, 0),
new FetchRequest("topic c", 0, 0)
};
MultiFetchRequest request = new MultiFetchRequest(requests);
// format = len(request) + requesttype + requestcount + requestpackage
// total byte count = 4 + (2 + 2 + 100)
byte[] bytes = request.GetBytes();
Assert.IsNotNull(bytes);
Assert.AreEqual(108, bytes.Length);
// first 4 bytes = the length of the request
Assert.AreEqual(104, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray<byte>()), 0));
// next 2 bytes = the RequestType which in this case should be Produce
Assert.AreEqual((short)RequestType.MultiFetch, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray<byte>()), 0));
// next 2 bytes = the number of messages
Assert.AreEqual((short)4, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray<byte>()), 0));
}
public void IsValidTrue()
{
List<FetchRequest> requests = new List<FetchRequest>
{
new FetchRequest("topic a", 0, 0),
new FetchRequest("topic a", 0, 0),
new FetchRequest("topic b", 0, 0),
new FetchRequest("topic c", 0, 0)
};
MultiFetchRequest multiRequest = new MultiFetchRequest(requests);
Assert.IsTrue(multiRequest.IsValid());
}
/// <summary>
/// Executes a multi-fetch operation.
/// </summary>
/// <param name="request">The request to push to Kafka.</param>
/// <returns>
/// A list containing sets of messages. The message sets should match the request order.
/// </returns>
public List<List<Message>> Consume(MultiFetchRequest request)
{
int fetchRequests = request.ConsumerRequests.Count;
List<List<Message>> messages = new List<List<Message>>();
using (KafkaConnection connection = new KafkaConnection(Server, Port))
{
connection.Write(request.GetBytes());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = connection.Read(dataLength);
int position = 0;
int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray<byte>()), 0);
if (errorCode != KafkaException.NoError)
{
throw new KafkaException(errorCode);
}
// skip the error code and process the rest
position = position + 2;
for (int ix = 0; ix < fetchRequests; ix++)
{
messages.Add(new List<Message>());
int messageSetSize = BitConverter.ToInt32(BitWorks.ReverseBytes(data.Skip(position).Take(4).ToArray<byte>()), 0);
position = position + 4;
errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Skip(position).Take(2).ToArray<byte>()), 0);
if (errorCode != KafkaException.NoError)
{
throw new KafkaException(errorCode);
}
// skip the error code and process the rest
position = position + 2;
byte[] messageSetBytes = data.Skip(position).ToArray<byte>().Take(messageSetSize).ToArray<byte>();
int processed = 0;
int messageSize = 0;
// dropped 2 bytes at the end...padding???
while (processed < messageSetBytes.Length - 2)
{
messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(messageSetBytes.Skip(processed).Take(4).ToArray<byte>()), 0);
messages[ix].Add(Message.ParseFrom(messageSetBytes.Skip(processed).Take(messageSize + 4).ToArray<byte>()));
processed += 4 + messageSize;
}
position = position + processed;
}
}
}
return messages;
}
public void IsValidNullRequests()
{
MultiFetchRequest multiRequest = new MultiFetchRequest(null);
Assert.IsFalse(multiRequest.IsValid());
}
public void IsValidNoRequests()
{
MultiFetchRequest multiRequest = new MultiFetchRequest(new List<FetchRequest>());
Assert.IsFalse(multiRequest.IsValid());
}
public void ConsumerMultiFetchGetsMessage()
{
ProducerSendMultiRequest();
Consumer consumer = new Consumer(KafkaServer, KafkaPort);
MultiFetchRequest request = new MultiFetchRequest(new List<FetchRequest>
{
new FetchRequest("test", 0, 0),
new FetchRequest("test", 0, 0),
new FetchRequest("testa", 0, 0)
});
List<List<Message>> messages = consumer.Consume(request);
for (int ix = 0; ix < messages.Count; ix++)
{
List<Message> messageSet = messages[ix];
Console.WriteLine(string.Format("Request #{0}-->", ix));
foreach (Message msg in messageSet)
{
Console.WriteLine(msg);
}
}
}