public void SetupBus() { _pingConsumer = new PingConsumer(); //var testTopic = $"test-ping-{DateTime.Now.Ticks}"; var topic = $"test-ping"; // some unique string across all application instances var instanceId = "1"; // address to your Kafka broker var kafkaBrokers = "127.0.0.1:9092"; var messageBusBuilder = new MessageBusBuilder() .Publish <PingMessage>(x => { x.DefaultTopic(topic); }) .SubscribeTo <PingMessage>(x => { x.Topic(topic) .Group("subscriber2") .WithSubscriber <PingConsumer>() .Instances(2); }) .ExpectRequestResponses(x => { x.ReplyToTopic($"worker-{instanceId}-response"); x.Group($"worker-{instanceId}"); x.DefaultTimeout(TimeSpan.FromSeconds(10)); }) .WithSerializer(new JsonMessageSerializer()) .WithDependencyResolver(new FakeDependencyResolver(_pingConsumer)) .WithProviderKafka(new KafkaMessageBusSettings(kafkaBrokers) { ProducerConfigFactory = () => new Dictionary <string, object> { { "socket.blocking.max.ms", 1 }, { "queue.buffering.max.ms", 1 }, { "socket.nagle.disable", true } }, ConsumerConfigFactory = (group) => new Dictionary <string, object> { { "socket.blocking.max.ms", 1 }, { "fetch.error.backoff.ms", 1 }, { "statistics.interval.ms", 500000 }, { "socket.nagle.disable", true } } }); _bus = (KafkaMessageBus)messageBusBuilder.Build(); }
private IList <Tuple <PingMessage, int> > ConsumeFromTopic(PingConsumer pingConsumer) { var lastMessageCount = 0; var lastMessageStopwatch = Stopwatch.StartNew(); const int newMessagesAwatingTimeout = 10; while (lastMessageStopwatch.Elapsed.TotalSeconds < newMessagesAwatingTimeout) { Thread.Sleep(200); if (pingConsumer.Messages.Count != lastMessageCount) { lastMessageCount = pingConsumer.Messages.Count; lastMessageStopwatch.Restart(); } } lastMessageStopwatch.Stop(); return(pingConsumer.Messages); }
public FakeDependencyResolver(PingConsumer pingConsumer) { _pingConsumer = pingConsumer; }
public async Task BasicPubSub() { // arrange // ensure the topic has 2 partitions var topic = $"{TopicPrefix}test-ping"; var pingConsumer = new PingConsumer(_loggerFactory.CreateLogger <PingConsumer>()); MessageBusBuilder .Produce <PingMessage>(x => { x.DefaultTopic(topic); // Partition #0 for even counters // Partition #1 for odd counters x.PartitionProvider((m, t) => m.Counter % 2); }) .Consume <PingMessage>(x => { x.Topic(topic) .WithConsumer <PingConsumer>() .Group("subscriber") .Instances(2) .CheckpointEvery(1000) .CheckpointAfter(TimeSpan.FromSeconds(600)); }) .WithDependencyResolver(new LookupDependencyResolver(f => { if (f == typeof(PingConsumer)) { return(pingConsumer); } throw new InvalidOperationException(); })); var messageBus = MessageBus.Value; // act // publish var stopwatch = Stopwatch.StartNew(); var messages = Enumerable .Range(0, NumberOfMessages) .Select(i => new PingMessage { Counter = i, Timestamp = DateTime.UtcNow }) .ToList(); await Task.WhenAll(messages.Select(m => messageBus.Publish(m))); stopwatch.Stop(); _logger.LogInformation("Published {0} messages in {1}", messages.Count, stopwatch.Elapsed); // consume stopwatch.Restart(); await WaitWhileMessagesAreFlowing(() => pingConsumer.Messages.Count); var messagesReceived = pingConsumer.Messages; stopwatch.Stop(); _logger.LogInformation("Consumed {0} messages in {1}", messagesReceived.Count, stopwatch.Elapsed); // assert // all messages got back messagesReceived.Count.Should().Be(messages.Count); // Partition #0 => Messages with even counter messagesReceived .Where(x => x.Item2 == 0) .All(x => x.Item1.Counter % 2 == 0) .Should().BeTrue(); // Partition #1 => Messages with odd counter messagesReceived .Where(x => x.Item2 == 1) .All(x => x.Item1.Counter % 2 == 1) .Should().BeTrue(); }
public void BasicPubSub() { // arrange // ensure the topic has 2 partitions var topic = $"test-ping"; var pingConsumer = new PingConsumer(); MessageBusBuilder .Publish <PingMessage>(x => { x.DefaultTopic(topic); // Partition #0 for even counters // Partition #1 for odd counters x.PartitionProvider((m, t) => m.Counter % 2); }) .SubscribeTo <PingMessage>(x => x.Topic(topic) .Group("subscriber") .WithSubscriber <PingConsumer>() .Instances(2)) .WithDependencyResolver(new LookupDependencyResolver(f => { if (f == typeof(PingConsumer)) { return(pingConsumer); } throw new InvalidOperationException(); })); var kafkaMessageBus = MessageBus.Value; // act // publish var stopwatch = Stopwatch.StartNew(); var messages = Enumerable .Range(0, NumberOfMessages) .Select(i => new PingMessage { Counter = i, Timestamp = DateTime.UtcNow }) .ToList(); messages .AsParallel() .ForAll(m => kafkaMessageBus.Publish(m).Wait()); stopwatch.Stop(); Log.InfoFormat("Published {0} messages in {1}", messages.Count, stopwatch.Elapsed); // consume stopwatch.Restart(); var messagesReceived = ConsumeFromTopic(pingConsumer); stopwatch.Stop(); Log.InfoFormat("Consumed {0} messages in {1}", messagesReceived.Count, stopwatch.Elapsed); // assert // all messages got back messagesReceived.Count.Should().Be(messages.Count); // Partition #0 => Messages with even counter messagesReceived .Where(x => x.Item2 == 0) .All(x => x.Item1.Counter % 2 == 0) .Should().BeTrue(); // Partition #1 => Messages with odd counter messagesReceived .Where(x => x.Item2 == 1) .All(x => x.Item1.Counter % 2 == 1) .Should().BeTrue(); }