public async Task FetchMessagesCacheContainsAllRequestTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_options);
var protocolGateway = new ProtocolGateway(_kafkaUri);
Producer producer = new Producer(brokerRouter);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = await consumer.FetchLastOffset();
// Creating 5 messages
List<Message> messages = CreateTestMessages(10, 1);
await producer.SendMessageAsync(_topic, messages, partition: _partitionId, timeout: TimeSpan.FromSeconds(3));
// Now let's consume
var result = (await consumer.FetchMessages(5, offset)).ToList();
CheckMessages(messages.Take(5).ToList(), result);
// Now let's consume again
result = (await consumer.FetchMessages(5, offset + 5)).ToList();
CheckMessages(messages.Skip(5).ToList(), result);
}
public async Task ManualConsumerFailure()
{
string topic = "TestTopicIssue13-3R-1P";
var manualConsumer = new ManualConsumer(0, topic, new ProtocolGateway(_options), "test client", 10000);
long offset = await manualConsumer.FetchLastOffset();
var router = new BrokerRouter(_options);
var producer = new Producer(router);
SandMessageForever(producer, topic);
await ReadMessageForever(manualConsumer, offset);
}
public async Task FetchMessagesSimpleTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_options);
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 1;
var topic = "ManualConsumerTestTopic";
Producer producer = new Producer(brokerRouter);
ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = await consumer.FetchLastOffset();
// Creating 5 messages
List<Message> messages = CreateTestMessages(5, 1);
await producer.SendMessageAsync(topic, messages, partition: partitionId, timeout: TimeSpan.FromSeconds(3));
// Now let's consume
var result = (await consumer.FetchMessages(5, offset)).ToList();
CheckMessages(messages, result);
}
public async Task FetchLastOffsetPartitionDoesntExistTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 100;
var topic = _topic;
ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);
await consumer.FetchLastOffset();
}
public async Task FetchLastOffsetSimpleTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var topic = _topic;
ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);
var offset = await consumer.FetchLastOffset();
Assert.AreNotEqual(-1, offset);
}
public async Task UpdateOrCreateOffsetNegativeOffsetTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 0;
var topic = _topic + Guid.NewGuid();
var consumerGroup = _defaultConsumerGroup;
ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offest = -5;
await consumer.UpdateOrCreateOffset(consumerGroup, offest);
}
public async Task UpdateOrCreateOffsetPartitionDoesntExistTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 100;
var consumerGroup = Guid.NewGuid().ToString();
ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offest = 5;
await consumer.UpdateOrCreateOffset(consumerGroup, offest);
}
public async Task UpdateOrCreateOffsetConsumerGroupExistsTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 0;
var consumerGroup = _defaultConsumerGroup + Guid.NewGuid();
ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offest = 5;
var newOffset = 10;
await consumer.UpdateOrCreateOffset(consumerGroup, offest);
var res = await consumer.FetchOffset(consumerGroup);
Assert.AreEqual(offest, res);
await consumer.UpdateOrCreateOffset(consumerGroup, newOffset);
res = await consumer.FetchOffset(consumerGroup);
Assert.AreEqual(newOffset, res);
}
public async Task FetchOffsetConsumerGroupIsEmptyTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 0;
var consumerGroup = _defaultConsumerGroup;
ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offest = 5;
await consumer.UpdateOrCreateOffset(consumerGroup, offest);
await consumer.FetchOffset(string.Empty);
}
public async Task FetchMessagesTopicDoesntExist()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 1;
var topic = _topic + Guid.NewGuid();
ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);
var offset = await consumer.FetchLastOffset();
// Now let's consume
var result = (await consumer.FetchMessages(5, offset)).ToList();
Assert.AreEqual(0, result.Count);
}
public async Task FetchMessagesBufferUnderRunTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_options);
var protocolGateway = new ProtocolGateway(_kafkaUri);
var smallMessageSet = 4096 / 2;
Producer producer = new Producer(brokerRouter);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", smallMessageSet);
var offset = await consumer.FetchLastOffset();
// Creating 5 messages
List<Message> messages = CreateTestMessages(10, 4096);
await producer.SendMessageAsync(_topic, messages, partition: _partitionId, timeout: TimeSpan.FromSeconds(3));
// Now let's consume
var result = (await consumer.FetchMessages(5, offset)).ToList();
CheckMessages(messages.Take(5).ToList(), result);
}
public async Task FetchMessagesInvalidOffsetTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = -1;
// Now let's consume
await consumer.FetchMessages(5, offset);
}
public async Task FetchMessagesOffsetBiggerThanLastOffsetInQueueTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = await consumer.FetchLastOffset();
// Now let's consume
var result = (await consumer.FetchMessages(5, offset + 1)).ToList();
Assert.AreEqual(0, result.Count, "Should not get any messages");
}
private async Task ReadMessageForever(ManualConsumer manualConsumer, long offset)
{
while (true)
{
try
{
var messages = await manualConsumer.FetchMessages(1000, offset);
if (messages.Any())
{
foreach (var message in messages)
{
_log.InfoFormat("Offset{0} ", message.Meta.Offset);
}
offset = messages.Max(x => x.Meta.Offset) + 1;
}
else
{
await Task.Delay(100);
}
}
catch (Exception ex)
{
_log.InfoFormat("can't read:\n" + ex);
}
}
}
public async Task FetchLastOffsetTopicDoesntExistTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var topic = _topic + Guid.NewGuid();
ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);
var res = await consumer.FetchLastOffset();
Assert.AreEqual(0, res);
}
public async Task FetchOffsetTopicDoesntExistTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var topic = _topic + Guid.NewGuid();
var consumerGroup = _defaultConsumerGroup;
ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
await consumer.FetchOffset(consumerGroup);
}
public async Task FetchMessagesOffsetBiggerThanLastOffsetInQueueTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = await consumer.FetchLastOffset();
// Now let's consume throw KafkaApplicationException
await consumer.FetchMessages(5, offset + 1);
}