public async Task RetryAndSkipPolicies_JsonChunkSequenceStillFailingAfterRetries_OffsetCommitted()
{
var tryCount = 0;
var serviceProvider = Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddEndpoints(
endpoints => endpoints
.AddOutbound <IIntegrationEvent>(
new KafkaProducerEndpoint(DefaultTopicName)
{
Chunk = new ChunkSettings
{
Size = 10
}
})
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration = new KafkaConsumerConfig
{
GroupId = "consumer1",
EnableAutoCommit = false,
CommitOffsetEach = 1
},
ErrorPolicy = ErrorPolicy.Chain(
ErrorPolicy.Retry().MaxFailedAttempts(10),
ErrorPolicy.Skip())
}))
.AddSingletonBrokerBehavior <SpyBrokerBehavior>()
.AddDelegateSubscriber(
(IIntegrationEvent _, IServiceProvider sp) =>
{
var logger = sp.GetRequiredService <ISilverbackLogger <ErrorHandlingTests> >();
tryCount++;
logger.LogInformation($"Handling message ({tryCount})...");
throw new InvalidOperationException("Retry!");
}))
.Run();
var publisher = serviceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(
new TestEventOne
{
Content = "Hello E2E!"
});
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
tryCount.Should().Be(11);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(3);
}
public async Task HandleErrorAsync_Whatever_TrueReturned()
{
var policy = ErrorPolicy.Skip().Build(_serviceProvider);
var envelope = new InboundEnvelope(
"hey oh!",
new MemoryStream(),
null,
new TestOffset(),
TestConsumerEndpoint.GetDefault(),
TestConsumerEndpoint.GetDefault().Name);
var result = await policy.HandleErrorAsync(
ConsumerPipelineContextHelper.CreateSubstitute(envelope, _serviceProvider),
new InvalidOperationException("test"));
result.Should().BeTrue();
}
public async Task HandleError_Whatever_ConsumerCommittedButTransactionAborted()
{
var policy = ErrorPolicy.Skip().Build(_serviceProvider);
var envelope = new InboundEnvelope(
"hey oh!",
new MemoryStream(Encoding.UTF8.GetBytes("hey oh!")),
null,
new TestOffset(),
new TestConsumerEndpoint("source-endpoint"),
"source-endpoint");
var transactionManager = Substitute.For <IConsumerTransactionManager>();
await policy.HandleErrorAsync(
ConsumerPipelineContextHelper.CreateSubstitute(envelope, _serviceProvider, transactionManager),
new InvalidOperationException("test"));
await transactionManager.Received(1).RollbackAsync(Arg.Any <InvalidOperationException>(), true);
}
public async Task SkipPolicy_ChunkedJsonDeserializationError_SequenceSkipped()
{
var message = new TestEventOne {
Content = "Hello E2E!"
};
byte[] rawMessage = (await Endpoint.DefaultSerializer.SerializeAsync(
message,
new MessageHeaderCollection(),
MessageSerializationContext.Empty)).ReadAll() ??
throw new InvalidOperationException("Serializer returned null");
byte[] invalidRawMessage = Encoding.UTF8.GetBytes("<what?!><what?!><what?!>");
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddEndpoints(
endpoints => endpoints
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration =
{
GroupId = "consumer1",
EnableAutoCommit = false,
CommitOffsetEach = 1
},
ErrorPolicy = ErrorPolicy.Skip()
}))
.AddSingletonSubscriber <OutboundInboundSubscriber>())
.Run();
var producer = Broker.GetProducer(new KafkaProducerEndpoint(DefaultTopicName));
await producer.RawProduceAsync(
invalidRawMessage.Take(10).ToArray(),
HeadersHelper.GetChunkHeaders("1", 0, typeof(TestEventOne)));
await producer.RawProduceAsync(
invalidRawMessage.Skip(10).Take(10).ToArray(),
HeadersHelper.GetChunkHeaders("1", 1, typeof(TestEventOne)));
await producer.RawProduceAsync(
invalidRawMessage.Skip(20).ToArray(),
HeadersHelper.GetChunkHeaders("1", 2, true, typeof(TestEventOne)));
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
Subscriber.InboundEnvelopes.Should().BeEmpty();
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(3);
await producer.RawProduceAsync(
rawMessage.Take(10).ToArray(),
HeadersHelper.GetChunkHeaders("2", 0, typeof(TestEventOne)));
await producer.RawProduceAsync(
rawMessage.Skip(10).Take(10).ToArray(),
HeadersHelper.GetChunkHeaders("2", 1, typeof(TestEventOne)));
await producer.RawProduceAsync(
rawMessage.Skip(20).ToArray(),
HeadersHelper.GetChunkHeaders("2", 2, true, typeof(TestEventOne)));
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
Subscriber.InboundEnvelopes.Should().HaveCount(1);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(6);
}