public async Task FetchMessagesCacheContainsNoneOfRequestTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_options);
var protocolGateway = new ProtocolGateway(_kafkaUri);
Producer producer = new Producer(brokerRouter);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = await consumer.FetchLastOffset();
// Creating 5 messages
List <Message> messages = CreateTestMessages(10, 4096);
await producer.SendMessageAsync(_topic, messages, partition : _partitionId, timeout : TimeSpan.FromSeconds(3));
// Now let's consume
var result = (await consumer.FetchMessages(7, offset)).ToList();
CheckMessages(messages.Take(7).ToList(), result);
// Now let's consume again
result = (await consumer.FetchMessages(2, offset + 8)).ToList();
CheckMessages(messages.Skip(8).ToList(), result);
}
private async Task ReadMessageForever(ManualConsumer manualConsumer, long offset)
{
while (true)
{
try
{
var messages = await manualConsumer.FetchMessages(1000, offset);
if (messages.Any())
{
foreach (var message in messages)
{
_log.InfoFormat("Offset{0} ", message.Meta.Offset);
}
offset = messages.Max(x => x.Meta.Offset) + 1;
}
else
{
await Task.Delay(100);
}
}
catch (Exception ex)
{
_log.InfoFormat("can't read:\n" + ex);
}
}
}
public async Task FetchLastOffsetPartitionDoesntExistTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 100;
var topic = _topic;
ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);
await consumer.FetchLastOffset();
}
public async Task FetchOffsetConsumerGroupDoesntExistTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 0;
var consumerGroup = Guid.NewGuid().ToString();
ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
await consumer.FetchOffset(consumerGroup);
}
public IQueueAdapterReceiver CreateReceiver(QueueId queueId)
{
var partitionId = (int)queueId.GetNumericId();
var clientName = Environment.MachineName + AppDomain.CurrentDomain.FriendlyName;
// Creating a receiver
var manualConsumer = new ManualConsumer(partitionId, _options.TopicName, _gateway, clientName, _options.MaxBytesInMessageSet);
var receiverToReturn = new KafkaQueueAdapterReceiver(queueId, manualConsumer, _options, _batchFactory, _logger);
return(receiverToReturn);
}
public async Task FetchMessagesInvalidOffsetTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = -1;
// Now let's consume
await consumer.FetchMessages(5, offset);
}
public async Task FetchMessagesOffsetBiggerThanLastOffsetInQueueTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = await consumer.FetchLastOffset();
// Now let's consume throw KafkaApplicationException
await consumer.FetchMessages(5, offset + 1);
}
public async Task ManualConsumerFailure()
{
string topic = "TestTopicIssue13-3R-1P";
var manualConsumer = new ManualConsumer(0, topic, new ProtocolGateway(_options), "test client", 10000);
long offset = await manualConsumer.FetchLastOffset();
var router = new BrokerRouter(_options);
var producer = new Producer(router);
SandMessageForever(producer, topic);
await ReadMessageForever(manualConsumer, offset);
}
public async Task FetchLastOffsetSimpleTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var topic = _topic;
ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);
var offset = await consumer.FetchLastOffset();
Assert.AreNotEqual(-1, offset);
}
public async Task FetchLastOffsetTopicDoesntExistTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var topic = _topic + Guid.NewGuid();
ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);
var res = await consumer.FetchLastOffset();
Assert.AreEqual(0, res);
}
public async Task UpdateOrCreateOffsetConsumerGroupEmptyTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 0;
var topic = _topic + Guid.NewGuid();
ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offest = 5;
await consumer.UpdateOrCreateOffset(string.Empty, offest);
}
public async Task FetchMessagesNoNewMessagesInQueueTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = await consumer.FetchLastOffset();
// Now let's consume
var result = (await consumer.FetchMessages(5, offset)).ToList();
Assert.AreEqual(0, result.Count, "Should not get any messages");
}
public async Task FetchOffsetConsumerGroupIsNullTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 0;
var consumerGroup = _defaultConsumerGroup;
ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offest = 5;
await consumer.UpdateOrCreateOffset(consumerGroup, offest);
await consumer.FetchOffset(null);
}
public async Task FetchMessagesSimpleTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_options);
var protocolGateway = new ProtocolGateway(_kafkaUri);
var topic = "ManualConsumerTestTopic";
Producer producer = new Producer(brokerRouter);
ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = await consumer.FetchLastOffset();
// Creating 5 messages
List <Message> messages = CreateTestMessages(5, 1);
await producer.SendMessageAsync(topic, messages, partition : _partitionId, timeout : TimeSpan.FromSeconds(3));
// Now let's consume
var result = (await consumer.FetchMessages(5, offset)).ToList();
CheckMessages(messages, result);
}
public async Task FetchMessagesBufferUnderRunTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_options);
var protocolGateway = new ProtocolGateway(_kafkaUri);
var smallMessageSet = 4096 / 2;
Producer producer = new Producer(brokerRouter);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", smallMessageSet);
var offset = await consumer.FetchLastOffset();
// Creating 5 messages
List <Message> messages = CreateTestMessages(10, 4096);
await producer.SendMessageAsync(_topic, messages, partition : _partitionId, timeout : TimeSpan.FromSeconds(3));
// Now let's consume
var result = (await consumer.FetchMessages(5, offset)).ToList();
CheckMessages(messages.Take(5).ToList(), result);
}
public async Task UpdateOrCreateOffsetConsumerGroupExistsTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 0;
var consumerGroup = _defaultConsumerGroup + Guid.NewGuid();
ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offest = 5;
var newOffset = 10;
await consumer.UpdateOrCreateOffset(consumerGroup, offest);
var res = await consumer.FetchOffset(consumerGroup);
Assert.AreEqual(offest, res);
await consumer.UpdateOrCreateOffset(consumerGroup, newOffset);
res = await consumer.FetchOffset(consumerGroup);
Assert.AreEqual(newOffset, res);
}
private static void Consumer(string brokerList, List <string> topics, string mode)
{
Console.WriteLine($"Started consumer, Ctrl-C to stop consuming");
ConsumerApp.Interfaces.Consumer consumer;
//Modes of the producer
//single - each line is write in the console
//multi - produces a lot of messages
if (String.IsNullOrEmpty(mode))
{
mode = ConsumerTypes.Database.ToString();
}
else
{
ConsumerTypes producerType;
if (Enum.TryParse(mode, out producerType))
{
if (Enum.IsDefined(typeof(ConsumerTypes), mode))
{
mode = producerType.ToString();
}
else
{
Console.WriteLine("The producer type passed as argument doesn't exits. mode defaults to multi");
mode = ConsumerTypes.Database.ToString();
}
}
else
{
mode = ConsumerTypes.Database.ToString();
}
}
if (mode == ConsumerTypes.single.ToString())
{
consumer = new SimpleConsumer();
}
else if (mode == ConsumerTypes.manual.ToString())
{
consumer = new ManualConsumer();
}
else if (mode == ConsumerTypes.EntityFramework.ToString())
{
consumer = new EntityFrameworkConsumer();
}
else if (mode == ConsumerTypes.Database.ToString())
{
consumer = new DatabaseConsumer();
}
else
{
consumer = new DatabaseConsumer();
}
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true; // prevent the process from terminating.
cts.Cancel();
};
consumer.Run(brokerList, topics, cts.Token);
// switch (mode)
// {
// case "subscribe":
// Run_Consume(brokerList, topics, cts.Token);
// break;
// case "manual":
// Run_ManualAssign(brokerList, topics, cts.Token);
// break;
// case "multi":
// Run_MultiAssign_Multi(brokerList, topics, cts.Token);
// break;
// case "multijson":
// Run_MultiAssign_Json(brokerList, topics, cts.Token);
// break;
// default:
// PrintUsage();
// break;
// }
}
