public async Task FetchMessagesCacheContainsNoneOfRequestTest() { // Creating a broker router and a protocol gateway for the producer and consumer var brokerRouter = new BrokerRouter(_options); var protocolGateway = new ProtocolGateway(_kafkaUri); Producer producer = new Producer(brokerRouter); ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize); var offset = await consumer.FetchLastOffset(); // Creating 5 messages List <Message> messages = CreateTestMessages(10, 4096); await producer.SendMessageAsync(_topic, messages, partition : _partitionId, timeout : TimeSpan.FromSeconds(3)); // Now let's consume var result = (await consumer.FetchMessages(7, offset)).ToList(); CheckMessages(messages.Take(7).ToList(), result); // Now let's consume again result = (await consumer.FetchMessages(2, offset + 8)).ToList(); CheckMessages(messages.Skip(8).ToList(), result); }
private async Task ReadMessageForever(ManualConsumer manualConsumer, long offset) { while (true) { try { var messages = await manualConsumer.FetchMessages(1000, offset); if (messages.Any()) { foreach (var message in messages) { _log.InfoFormat("Offset{0} ", message.Meta.Offset); } offset = messages.Max(x => x.Meta.Offset) + 1; } else { await Task.Delay(100); } } catch (Exception ex) { _log.InfoFormat("can't read:\n" + ex); } } }
public async Task FetchLastOffsetPartitionDoesntExistTest() { // Creating a broker router and a protocol gateway for the producer and consumer var protocolGateway = new ProtocolGateway(_kafkaUri); var partitionId = 100; var topic = _topic; ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2); await consumer.FetchLastOffset(); }
public async Task FetchOffsetConsumerGroupDoesntExistTest() { // Creating a broker router and a protocol gateway for the producer and consumer var protocolGateway = new ProtocolGateway(_kafkaUri); var partitionId = 0; var consumerGroup = Guid.NewGuid().ToString(); ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize); await consumer.FetchOffset(consumerGroup); }
public IQueueAdapterReceiver CreateReceiver(QueueId queueId) { var partitionId = (int)queueId.GetNumericId(); var clientName = Environment.MachineName + AppDomain.CurrentDomain.FriendlyName; // Creating a receiver var manualConsumer = new ManualConsumer(partitionId, _options.TopicName, _gateway, clientName, _options.MaxBytesInMessageSet); var receiverToReturn = new KafkaQueueAdapterReceiver(queueId, manualConsumer, _options, _batchFactory, _logger); return(receiverToReturn); }
public async Task FetchMessagesInvalidOffsetTest() { // Creating a broker router and a protocol gateway for the producer and consumer var protocolGateway = new ProtocolGateway(_kafkaUri); ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize); var offset = -1; // Now let's consume await consumer.FetchMessages(5, offset); }
public async Task FetchMessagesOffsetBiggerThanLastOffsetInQueueTest() { // Creating a broker router and a protocol gateway for the producer and consumer var protocolGateway = new ProtocolGateway(_kafkaUri); ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize); var offset = await consumer.FetchLastOffset(); // Now let's consume throw KafkaApplicationException await consumer.FetchMessages(5, offset + 1); }
public async Task ManualConsumerFailure() { string topic = "TestTopicIssue13-3R-1P"; var manualConsumer = new ManualConsumer(0, topic, new ProtocolGateway(_options), "test client", 10000); long offset = await manualConsumer.FetchLastOffset(); var router = new BrokerRouter(_options); var producer = new Producer(router); SandMessageForever(producer, topic); await ReadMessageForever(manualConsumer, offset); }
public async Task FetchLastOffsetSimpleTest() { // Creating a broker router and a protocol gateway for the producer and consumer var protocolGateway = new ProtocolGateway(_kafkaUri); var topic = _topic; ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2); var offset = await consumer.FetchLastOffset(); Assert.AreNotEqual(-1, offset); }
public async Task FetchLastOffsetTopicDoesntExistTest() { // Creating a broker router and a protocol gateway for the producer and consumer var protocolGateway = new ProtocolGateway(_kafkaUri); var topic = _topic + Guid.NewGuid(); ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2); var res = await consumer.FetchLastOffset(); Assert.AreEqual(0, res); }
public async Task UpdateOrCreateOffsetConsumerGroupEmptyTest() { // Creating a broker router and a protocol gateway for the producer and consumer var protocolGateway = new ProtocolGateway(_kafkaUri); var partitionId = 0; var topic = _topic + Guid.NewGuid(); ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize); var offest = 5; await consumer.UpdateOrCreateOffset(string.Empty, offest); }
public async Task FetchMessagesNoNewMessagesInQueueTest() { // Creating a broker router and a protocol gateway for the producer and consumer var protocolGateway = new ProtocolGateway(_kafkaUri); ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize); var offset = await consumer.FetchLastOffset(); // Now let's consume var result = (await consumer.FetchMessages(5, offset)).ToList(); Assert.AreEqual(0, result.Count, "Should not get any messages"); }
public async Task FetchOffsetConsumerGroupIsNullTest() { // Creating a broker router and a protocol gateway for the producer and consumer var protocolGateway = new ProtocolGateway(_kafkaUri); var partitionId = 0; var consumerGroup = _defaultConsumerGroup; ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize); var offest = 5; await consumer.UpdateOrCreateOffset(consumerGroup, offest); await consumer.FetchOffset(null); }
public async Task FetchMessagesSimpleTest() { // Creating a broker router and a protocol gateway for the producer and consumer var brokerRouter = new BrokerRouter(_options); var protocolGateway = new ProtocolGateway(_kafkaUri); var topic = "ManualConsumerTestTopic"; Producer producer = new Producer(brokerRouter); ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize); var offset = await consumer.FetchLastOffset(); // Creating 5 messages List <Message> messages = CreateTestMessages(5, 1); await producer.SendMessageAsync(topic, messages, partition : _partitionId, timeout : TimeSpan.FromSeconds(3)); // Now let's consume var result = (await consumer.FetchMessages(5, offset)).ToList(); CheckMessages(messages, result); }
public async Task FetchMessagesBufferUnderRunTest() { // Creating a broker router and a protocol gateway for the producer and consumer var brokerRouter = new BrokerRouter(_options); var protocolGateway = new ProtocolGateway(_kafkaUri); var smallMessageSet = 4096 / 2; Producer producer = new Producer(brokerRouter); ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", smallMessageSet); var offset = await consumer.FetchLastOffset(); // Creating 5 messages List <Message> messages = CreateTestMessages(10, 4096); await producer.SendMessageAsync(_topic, messages, partition : _partitionId, timeout : TimeSpan.FromSeconds(3)); // Now let's consume var result = (await consumer.FetchMessages(5, offset)).ToList(); CheckMessages(messages.Take(5).ToList(), result); }
public async Task UpdateOrCreateOffsetConsumerGroupExistsTest() { // Creating a broker router and a protocol gateway for the producer and consumer var protocolGateway = new ProtocolGateway(_kafkaUri); var partitionId = 0; var consumerGroup = _defaultConsumerGroup + Guid.NewGuid(); ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize); var offest = 5; var newOffset = 10; await consumer.UpdateOrCreateOffset(consumerGroup, offest); var res = await consumer.FetchOffset(consumerGroup); Assert.AreEqual(offest, res); await consumer.UpdateOrCreateOffset(consumerGroup, newOffset); res = await consumer.FetchOffset(consumerGroup); Assert.AreEqual(newOffset, res); }
private static void Consumer(string brokerList, List <string> topics, string mode) { Console.WriteLine($"Started consumer, Ctrl-C to stop consuming"); ConsumerApp.Interfaces.Consumer consumer; //Modes of the producer //single - each line is write in the console //multi - produces a lot of messages if (String.IsNullOrEmpty(mode)) { mode = ConsumerTypes.Database.ToString(); } else { ConsumerTypes producerType; if (Enum.TryParse(mode, out producerType)) { if (Enum.IsDefined(typeof(ConsumerTypes), mode)) { mode = producerType.ToString(); } else { Console.WriteLine("The producer type passed as argument doesn't exits. mode defaults to multi"); mode = ConsumerTypes.Database.ToString(); } } else { mode = ConsumerTypes.Database.ToString(); } } if (mode == ConsumerTypes.single.ToString()) { consumer = new SimpleConsumer(); } else if (mode == ConsumerTypes.manual.ToString()) { consumer = new ManualConsumer(); } else if (mode == ConsumerTypes.EntityFramework.ToString()) { consumer = new EntityFrameworkConsumer(); } else if (mode == ConsumerTypes.Database.ToString()) { consumer = new DatabaseConsumer(); } else { consumer = new DatabaseConsumer(); } CancellationTokenSource cts = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; // prevent the process from terminating. cts.Cancel(); }; consumer.Run(brokerList, topics, cts.Token); // switch (mode) // { // case "subscribe": // Run_Consume(brokerList, topics, cts.Token); // break; // case "manual": // Run_ManualAssign(brokerList, topics, cts.Token); // break; // case "multi": // Run_MultiAssign_Multi(brokerList, topics, cts.Token); // break; // case "multijson": // Run_MultiAssign_Json(brokerList, topics, cts.Token); // break; // default: // PrintUsage(); // break; // } }