public async Task Rebalance_DefaultSettings_ProducedAndConsumedAfterRebalance()
{
var serviceProvider = Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(options => options.AddMockedKafka())
.AddEndpoints(
endpoints => endpoints
.AddOutbound <IIntegrationEvent>(new KafkaProducerEndpoint(DefaultTopicName))
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration = new KafkaConsumerConfig
{
GroupId = "consumer1",
AutoCommitIntervalMs = 100
}
}))
.AddSingletonSubscriber <OutboundInboundSubscriber>())
.Run();
var publisher = serviceProvider.GetRequiredService <IEventPublisher>();
for (int i = 1; i <= 5; i++)
{
await publisher.PublishAsync(
new TestEventOne
{
Content = $"{i}"
});
}
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
Subscriber.OutboundEnvelopes.Should().HaveCount(5);
Subscriber.InboundEnvelopes.Should().HaveCount(5);
DefaultTopic.Rebalance();
for (int i = 1; i <= 5; i++)
{
await publisher.PublishAsync(
new TestEventOne
{
Content = $"{i}"
});
}
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
Subscriber.OutboundEnvelopes.Should().HaveCount(10);
Subscriber.InboundEnvelopes.Should().HaveCount(10);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(10);
}
public async Task Rebalance_WithoutAutoCommit_PendingOffsetsCommitted()
{
int receivedMessages = 0;
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(options => options.AddMockedKafka())
.AddKafkaEndpoints(
endpoints => endpoints
.Configure(config => { config.BootstrapServers = "PLAINTEXT://e2e"; })
.AddOutbound <IIntegrationEvent>(
endpoint => endpoint.ProduceTo(DefaultTopicName))
.AddInbound(
endpoint => endpoint
.ConsumeFrom(DefaultTopicName)
.Configure(
config =>
{
config.GroupId = "consumer1";
config.EnableAutoCommit = false;
config.CommitOffsetEach = 10;
})))
.AddDelegateSubscriber(
(TestEventOne _) => Interlocked.Increment(ref receivedMessages)))
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(
new TestEventOne
{
Content = "one"
});
await publisher.PublishAsync(
new TestEventOne
{
Content = "two"
});
await publisher.PublishAsync(
new TestEventOne
{
Content = "three"
});
await AsyncTestingUtil.WaitAsync(() => receivedMessages == 3);
DefaultTopic.Rebalance();
await AsyncTestingUtil.WaitAsync(() => DefaultTopic.GetCommittedOffsetsCount("consumer1") == 3);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(3);
}
public async Task Rebalance_DefaultSettings_ProducedAndConsumedAfterRebalance()
{
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(options => options.AddMockedKafka())
.AddKafkaEndpoints(
endpoints => endpoints
.Configure(config => { config.BootstrapServers = "PLAINTEXT://e2e"; })
.AddOutbound <IIntegrationEvent>(
endpoint => endpoint.ProduceTo(DefaultTopicName))
.AddInbound(
endpoint => endpoint
.ConsumeFrom(DefaultTopicName)
.Configure(config => { config.GroupId = "consumer1"; })))
.AddIntegrationSpyAndSubscriber())
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
for (int i = 1; i <= 5; i++)
{
await publisher.PublishAsync(
new TestEventOne
{
Content = $"{i}"
});
}
await Helper.WaitUntilAllMessagesAreConsumedAsync();
Helper.Spy.OutboundEnvelopes.Should().HaveCount(5);
Helper.Spy.InboundEnvelopes.Should().HaveCount(5);
DefaultTopic.Rebalance();
for (int i = 1; i <= 5; i++)
{
await publisher.PublishAsync(
new TestEventOne
{
Content = $"{i}"
});
}
await Helper.WaitUntilAllMessagesAreConsumedAsync();
Helper.Spy.OutboundEnvelopes.Should().HaveCount(10);
Helper.Spy.InboundEnvelopes.Should().HaveCount(10);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(10);
}
public async Task Rebalance_WithoutAutoCommit_PendingOffsetsCommitted()
{
int receivedMessages = 0;
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(options => options.AddMockedKafka())
.AddEndpoints(
endpoints => endpoints
.AddOutbound <IIntegrationEvent>(new KafkaProducerEndpoint(DefaultTopicName))
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration =
{
GroupId = "consumer1",
EnableAutoCommit = false,
AutoCommitIntervalMs = 50,
CommitOffsetEach = 10
}
}))
.AddDelegateSubscriber((TestEventOne _) => receivedMessages++))
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(
new TestEventOne
{
Content = "one"
});
await publisher.PublishAsync(
new TestEventOne
{
Content = "two"
});
await publisher.PublishAsync(
new TestEventOne
{
Content = "three"
});
await AsyncTestingUtil.WaitAsync(() => receivedMessages == 3);
DefaultTopic.Rebalance();
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(3);
}
public async Task Rebalance_WithPendingBatch_AbortedAndConsumedAfterRebalance()
{
var receivedBatches = new List <List <TestEventOne> >();
var completedBatches = 0;
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddKafkaEndpoints(
endpoints => endpoints
.Configure(config => { config.BootstrapServers = "PLAINTEXT://e2e"; })
.AddOutbound <IIntegrationEvent>(
endpoint => endpoint.ProduceTo(DefaultTopicName))
.AddInbound(
endpoint => endpoint
.ConsumeFrom(DefaultTopicName)
.Configure(
config =>
{
config.GroupId = "consumer1";
config.EnableAutoCommit = false;
config.CommitOffsetEach = 1;
})
.EnableBatchProcessing(10)))
.AddDelegateSubscriber(
async(IAsyncEnumerable <TestEventOne> eventsStream) =>
{
var list = new List <TestEventOne>();
receivedBatches.ThreadSafeAdd(list);
await foreach (var message in eventsStream)
{
list.Add(message);
}
Interlocked.Increment(ref completedBatches);
}))
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
for (int i = 1; i <= 15; i++)
{
await publisher.PublishAsync(new TestEventOne { Content = $"{i}" });
}
await AsyncTestingUtil.WaitAsync(() => receivedBatches.Sum(batch => batch.Count) == 15);
receivedBatches.Should().HaveCount(2);
receivedBatches[0].Should().HaveCount(10);
receivedBatches[1].Should().HaveCount(5);
completedBatches.Should().Be(1);
receivedBatches.Sum(batch => batch.Count).Should().Be(15);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(10);
DefaultTopic.Rebalance();
await AsyncTestingUtil.WaitAsync(() => receivedBatches.Sum(batch => batch.Count) == 20);
receivedBatches.Should().HaveCount(3);
receivedBatches[0].Should().HaveCount(10);
receivedBatches[1].Should().HaveCount(5);
receivedBatches[2].Should().HaveCount(5);
completedBatches.Should().Be(1);
receivedBatches.Sum(batch => batch.Count).Should().Be(20);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(10);
for (int i = 16; i <= 20; i++)
{
await publisher.PublishAsync(new TestEventOne { Content = $"{i}" });
}
await Helper.WaitUntilAllMessagesAreConsumedAsync();
receivedBatches.Should().HaveCount(3);
receivedBatches[0].Should().HaveCount(10);
receivedBatches[1].Should().HaveCount(5);
receivedBatches[2].Should().HaveCount(10);
completedBatches.Should().Be(2);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(20);
}
public async Task StatusInfo_ConsumingAndRebalanceAndDisconnecting_StatusIsCorrectlySet()
{
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions =>
mockedKafkaOptions.DelayPartitionsAssignment(
TimeSpan.FromMilliseconds(100))))
.AddKafkaEndpoints(
endpoints => endpoints
.Configure(
config =>
{
config.BootstrapServers = "PLAINTEXT://e2e";
})
.AddOutbound <IIntegrationEvent>(
endpoint => endpoint.ProduceTo(DefaultTopicName))
.AddInbound(
endpoint => endpoint
.ConsumeFrom(DefaultTopicName)
.Configure(
config =>
{
config.GroupId = "consumer1";
config.EnableAutoCommit = false;
config.CommitOffsetEach = 1;
})))
.AddIntegrationSpyAndSubscriber())
.Run(waitUntilBrokerConnected: false);
var consumer = Helper.Broker.Consumers[0];
consumer.IsConnected.Should().BeTrue();
consumer.StatusInfo.Status.Should().Be(ConsumerStatus.Connected);
await AsyncTestingUtil.WaitAsync(() => consumer.StatusInfo.Status == ConsumerStatus.Ready);
consumer.StatusInfo.Status.Should().Be(ConsumerStatus.Ready);
DefaultTopic.Rebalance();
consumer.StatusInfo.Status.Should().Be(ConsumerStatus.Connected);
await AsyncTestingUtil.WaitAsync(() => consumer.StatusInfo.Status == ConsumerStatus.Ready);
consumer.StatusInfo.Status.Should().Be(ConsumerStatus.Ready);
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(new TestEventOne());
await Helper.WaitUntilAllMessagesAreConsumedAsync();
consumer.IsConnected.Should().BeTrue();
consumer.StatusInfo.Status.Should().Be(ConsumerStatus.Consuming);
await Helper.Broker.DisconnectAsync();
consumer.IsConnected.Should().BeFalse();
consumer.StatusInfo.Status.Should().Be(ConsumerStatus.Disconnected);
}