public async Task Chunking_JsonConsumedInBatch_ProducedAndConsumed()
{
var batches = new List <List <TestEventOne> >();
string?failedCommit = null;
string?enumerationAborted = null;
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddEndpoints(
endpoints => endpoints
.AddOutbound <IIntegrationEvent>(
new KafkaProducerEndpoint(DefaultTopicName)
{
Chunk = new ChunkSettings
{
Size = 10
}
})
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration =
{
GroupId = "consumer1",
EnableAutoCommit = false,
CommitOffsetEach = 1
},
Batch = new BatchSettings
{
Size = 3
}
}))
.AddDelegateSubscriber(
async(IMessageStreamEnumerable <TestEventOne> streamEnumerable) =>
{
var list = new List <TestEventOne>();
batches.Add(list);
await foreach (var message in streamEnumerable)
{
list.Add(message);
var actualCommittedOffsets = DefaultTopic.GetCommittedOffsetsCount("consumer1");
var expectedCommittedOffsets = 9 * (batches.Count - 1);
if (actualCommittedOffsets != expectedCommittedOffsets)
{
failedCommit ??= $"{actualCommittedOffsets} != {expectedCommittedOffsets} " +
$"({batches.Count}.{list.Count})";
}
}
if (list.Count != 3)
{
enumerationAborted ??= $"Enumeration completed after {list.Count} messages " +
$"({batches.Count}.{list.Count})";
}
}))
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
for (int i = 1; i <= 9; i++)
{
await publisher.PublishAsync(new TestEventOne { Content = $"Long message {i}" });
}
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
failedCommit.Should().BeNull();
enumerationAborted.Should().BeNull();
batches.Should().HaveCount(3);
batches[0].Should().HaveCount(3);
batches[1].Should().HaveCount(3);
batches[2].Should().HaveCount(3);
batches[0][0].Content.Should().Be("Long message 1");
batches[0][1].Content.Should().Be("Long message 2");
batches[0][2].Content.Should().Be("Long message 3");
batches[1][0].Content.Should().Be("Long message 4");
batches[1][1].Content.Should().Be("Long message 5");
batches[1][2].Content.Should().Be("Long message 6");
batches[2][0].Content.Should().Be("Long message 7");
batches[2][1].Content.Should().Be("Long message 8");
batches[2][2].Content.Should().Be("Long message 9");
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(27);
}
public async Task RetryPolicy_JsonChunkSequenceProcessedAfterSomeTries_RetriedMultipleTimesAndCommitted()
{
var tryCount = 0;
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddEndpoints(
endpoints => endpoints
.AddOutbound <IIntegrationEvent>(
new KafkaProducerEndpoint(DefaultTopicName)
{
Chunk = new ChunkSettings
{
Size = 10
}
})
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration =
{
GroupId = "consumer1",
EnableAutoCommit = false,
CommitOffsetEach = 1
},
ErrorPolicy = ErrorPolicy.Retry().MaxFailedAttempts(10)
}))
.AddSingletonBrokerBehavior <SpyBrokerBehavior>()
.AddDelegateSubscriber(
(IIntegrationEvent _) =>
{
tryCount++;
if (tryCount % 2 != 0)
{
throw new InvalidOperationException("Retry!");
}
}))
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(
new TestEventOne
{
Content = "Long message one"
});
await publisher.PublishAsync(
new TestEventOne
{
Content = "Long message two"
});
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
SpyBehavior.OutboundEnvelopes.Should().HaveCount(6);
SpyBehavior.OutboundEnvelopes.ForEach(
envelope =>
{
envelope.RawMessage.Should().NotBeNull();
envelope.RawMessage !.Length.Should().BeLessOrEqualTo(10);
});
tryCount.Should().Be(4);
SpyBehavior.InboundEnvelopes.Should().HaveCount(4);
SpyBehavior.InboundEnvelopes[0].Message.As <TestEventOne>().Content.Should().Be("Long message one");
SpyBehavior.InboundEnvelopes[1].Message.As <TestEventOne>().Content.Should().Be("Long message one");
SpyBehavior.InboundEnvelopes[2].Message.As <TestEventOne>().Content.Should().Be("Long message two");
SpyBehavior.InboundEnvelopes[3].Message.As <TestEventOne>().Content.Should().Be("Long message two");
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(6);
}
public async Task Encryption_SimpleMessages_EncryptedAndDecrypted()
{
var message1 = new TestEventOne {
Content = "Message 1"
};
var message2 = new TestEventOne {
Content = "Message 2"
};
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddKafkaEndpoints(
endpoints => endpoints
.Configure(
config =>
{
config.BootstrapServers = "PLAINTEXT://e2e";
})
.AddOutbound <IIntegrationEvent>(
endpoint => endpoint
.ProduceTo(DefaultTopicName)
.EncryptUsingAes(AesEncryptionKey))
.AddInbound(
endpoint => endpoint
.ConsumeFrom(DefaultTopicName)
.DecryptUsingAes(AesEncryptionKey)
.Configure(
config =>
{
config.GroupId = "consumer1";
})))
.AddIntegrationSpyAndSubscriber())
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(message1);
await publisher.PublishAsync(message2);
await Helper.WaitUntilAllMessagesAreConsumedAsync();
Helper.Spy.OutboundEnvelopes.Should().HaveCount(2);
Helper.Spy.OutboundEnvelopes[0].RawMessage.Should().BeOfType <SymmetricEncryptStream>();
Helper.Spy.OutboundEnvelopes[1].RawMessage.Should().BeOfType <SymmetricEncryptStream>();
Helper.Spy.OutboundEnvelopes.ForEach(
envelope =>
envelope.Headers.GetValue(DefaultMessageHeaders.EncryptionKeyId).Should().BeNull());
Helper.Spy.InboundEnvelopes.Should().HaveCount(2);
Helper.Spy.InboundEnvelopes[0].Message.Should().BeEquivalentTo(message1);
Helper.Spy.InboundEnvelopes[1].Message.Should().BeEquivalentTo(message2);
Helper.Spy.InboundEnvelopes.ForEach(
envelope =>
envelope.Headers.GetValue(DefaultMessageHeaders.EncryptionKeyId).Should().BeNull());
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(2);
}
public async Task Streaming_UnboundedObservableProcessingFailed_ConsumerStopped()
{
var receivedMessages = new List <TestEventOne>();
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.AsObservable()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddKafkaEndpoints(
endpoints => endpoints
.Configure(
config =>
{
config.BootstrapServers = "PLAINTEXT://e2e";
})
.AddOutbound <IIntegrationEvent>(
endpoint => endpoint.ProduceTo(DefaultTopicName))
.AddInbound(
endpoint => endpoint
.ConsumeFrom(DefaultTopicName)
.Configure(
config =>
{
config.GroupId = "consumer1";
config.EnableAutoCommit = false;
config.CommitOffsetEach = 1;
})))
.AddDelegateSubscriber(
(IMessageStreamObservable <TestEventOne> observable) =>
observable.Subscribe(
message =>
{
receivedMessages.Add(message);
if (receivedMessages.Count == 2)
{
throw new InvalidOperationException("Test");
}
})))
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(
new TestEventOne
{
Content = "Message 1"
});
await publisher.PublishAsync(
new TestEventOne
{
Content = "Message 2"
});
await publisher.PublishAsync(
new TestEventOne
{
Content = "Message 3"
});
await Helper.WaitUntilAllMessagesAreConsumedAsync();
await AsyncTestingUtil.WaitAsync(() => receivedMessages.Count >= 2);
receivedMessages.Should().HaveCount(2);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(1);
Helper.Broker.Consumers[0].IsConnected.Should().BeFalse();
}
public async Task OutboundAndInbound_MqttToKafka_ProducedAndConsumed()
{
int eventOneCount = 0;
int eventTwoCount = 0;
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options
.AddMockedMqtt()
.AddMockedKafka())
.AddMqttEndpoints(
endpoints => endpoints
.Configure(
config => config
.WithClientId("e2e-test")
.ConnectViaTcp("e2e-mqtt-broker"))
.AddOutbound <TestEventOne>(endpoint => endpoint.ProduceTo(DefaultTopicName))
.AddInbound(endpoint => endpoint.ConsumeFrom(DefaultTopicName)))
.AddKafkaEndpoints(
endpoints => endpoints
.Configure(config => { config.BootstrapServers = "PLAINTEXT://tests"; })
.AddOutbound <TestEventTwo>(endpoint => endpoint.ProduceTo(DefaultTopicName))
.AddInbound(
endpoint => endpoint
.ConsumeFrom(DefaultTopicName)
.Configure(
config =>
{
config.GroupId = "consumer1";
})))
.AddDelegateSubscriber(
(TestEventOne eventOne) =>
{
Interlocked.Increment(ref eventOneCount);
return(new TestEventTwo {
Content = eventOne.Content
});
})
.AddDelegateSubscriber((TestEventTwo _) => Interlocked.Increment(ref eventTwoCount)))
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
var mqttTestingHelper = Host.ServiceProvider.GetRequiredService <IMqttTestingHelper>();
await mqttTestingHelper.WaitUntilConnectedAsync();
for (int i = 1; i <= 15; i++)
{
await publisher.PublishAsync(
new TestEventOne
{
Content = $"{i}"
});
}
await AsyncTestingUtil.WaitAsync(() => eventOneCount >= 15);
await AsyncTestingUtil.WaitAsync(() => eventTwoCount >= 15);
await Helper.WaitUntilAllMessagesAreConsumedAsync();
eventOneCount.Should().Be(15);
eventTwoCount.Should().Be(15);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(15);
}
public async Task StatusInfo_ConsumingAndRebalanceAndDisconnecting_StatusIsCorrectlySet()
{
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions =>
mockedKafkaOptions.DelayPartitionsAssignment(
TimeSpan.FromMilliseconds(100))))
.AddKafkaEndpoints(
endpoints => endpoints
.Configure(
config =>
{
config.BootstrapServers = "PLAINTEXT://e2e";
})
.AddOutbound <IIntegrationEvent>(
endpoint => endpoint.ProduceTo(DefaultTopicName))
.AddInbound(
endpoint => endpoint
.ConsumeFrom(DefaultTopicName)
.Configure(
config =>
{
config.GroupId = "consumer1";
config.EnableAutoCommit = false;
config.CommitOffsetEach = 1;
})))
.AddIntegrationSpyAndSubscriber())
.Run(waitUntilBrokerConnected: false);
var consumer = Helper.Broker.Consumers[0];
consumer.IsConnected.Should().BeTrue();
consumer.StatusInfo.Status.Should().Be(ConsumerStatus.Connected);
await AsyncTestingUtil.WaitAsync(() => consumer.StatusInfo.Status == ConsumerStatus.Ready);
consumer.StatusInfo.Status.Should().Be(ConsumerStatus.Ready);
DefaultTopic.Rebalance();
consumer.StatusInfo.Status.Should().Be(ConsumerStatus.Connected);
await AsyncTestingUtil.WaitAsync(() => consumer.StatusInfo.Status == ConsumerStatus.Ready);
consumer.StatusInfo.Status.Should().Be(ConsumerStatus.Ready);
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(new TestEventOne());
await Helper.WaitUntilAllMessagesAreConsumedAsync();
consumer.IsConnected.Should().BeTrue();
consumer.StatusInfo.Status.Should().Be(ConsumerStatus.Consuming);
await Helper.Broker.DisconnectAsync();
consumer.IsConnected.Should().BeFalse();
consumer.StatusInfo.Status.Should().Be(ConsumerStatus.Disconnected);
}
public async Task ExactlyOnce_InMemoryInboundLog_DuplicatedMessagesIgnored()
{
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options
.AddMockedKafka(mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1))
.AddInMemoryInboundLog())
.AddEndpoints(
endpoints => endpoints
.AddOutbound <IIntegrationEvent>(new KafkaProducerEndpoint(DefaultTopicName))
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration =
{
GroupId = "consumer1",
AutoCommitIntervalMs = 100
},
ExactlyOnceStrategy = ExactlyOnceStrategy.Log()
}))
.AddSingletonSubscriber <OutboundInboundSubscriber>())
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(new TestEventOne());
await publisher.PublishAsync(
new TestEventWithUniqueKey
{
UniqueKey = "1"
});
await publisher.PublishAsync(
new TestEventWithUniqueKey
{
UniqueKey = "2"
});
await publisher.PublishAsync(
new TestEventWithUniqueKey
{
UniqueKey = "1"
});
await publisher.PublishAsync(
new TestEventWithUniqueKey
{
UniqueKey = "2"
});
await publisher.PublishAsync(new TestEventOne());
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
Subscriber.InboundEnvelopes.Should().HaveCount(4);
Subscriber.InboundEnvelopes[0].Message.Should().BeOfType <TestEventOne>();
Subscriber.InboundEnvelopes[1].Message.Should().BeOfType <TestEventWithUniqueKey>();
Subscriber.InboundEnvelopes[1].Message.As <TestEventWithUniqueKey>().UniqueKey.Should().Be("1");
Subscriber.InboundEnvelopes[2].Message.Should().BeOfType <TestEventWithUniqueKey>();
Subscriber.InboundEnvelopes[2].Message.As <TestEventWithUniqueKey>().UniqueKey.Should().Be("2");
Subscriber.InboundEnvelopes[3].Message.Should().BeOfType <TestEventOne>();
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(6);
}
public async Task Chunking_BinaryFileConsumedInBatch_ProducedAndConsumed()
{
var batches = new List <List <string?> >();
string?failedCommit = null;
string?enumerationAborted = null;
var serviceProvider = Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddEndpoints(
endpoints => endpoints
.AddOutbound <IBinaryFileMessage>(
new KafkaProducerEndpoint(DefaultTopicName)
{
Chunk = new ChunkSettings
{
Size = 10
}
})
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration = new KafkaConsumerConfig
{
GroupId = "consumer1",
EnableAutoCommit = false,
CommitOffsetEach = 1
},
Batch = new BatchSettings
{
Size = 5
}
}))
.AddDelegateSubscriber(
async(IMessageStreamEnumerable <BinaryFileMessage> streamEnumerable) =>
{
var list = new List <string?>();
batches.Add(list);
await foreach (var message in streamEnumerable)
{
var actualCommittedOffsets = DefaultTopic.GetCommittedOffsetsCount("consumer1");
var expectedCommittedOffsets = 10 * (batches.Count - 1);
if (actualCommittedOffsets != expectedCommittedOffsets)
{
failedCommit ??= $"{actualCommittedOffsets} != {expectedCommittedOffsets} " +
$"({batches.Count}.{list.Count})";
}
var readAll = await message.Content.ReadAllAsync();
list.Add(readAll != null ? Encoding.UTF8.GetString(readAll) : null);
}
if (list.Count != 5)
{
enumerationAborted ??= $"Enumeration completed after {list.Count} messages " +
$"({batches.Count}.{list.Count})";
}
}))
.Run();
var publisher = serviceProvider.GetRequiredService <IPublisher>();
for (int i = 1; i <= 15; i++)
{
await publisher.PublishAsync(new BinaryFileMessage(Encoding.UTF8.GetBytes($"Long message {i}")));
}
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
failedCommit.Should().BeNull();
enumerationAborted.Should().BeNull();
batches.Should().HaveCount(3);
batches[0].Should().HaveCount(5);
batches[1].Should().HaveCount(5);
batches[2].Should().HaveCount(5);
batches[0][0].Should().Be("Long message 1");
batches[0][1].Should().Be("Long message 2");
batches[0][2].Should().Be("Long message 3");
batches[0][3].Should().Be("Long message 4");
batches[0][4].Should().Be("Long message 5");
batches[1][0].Should().Be("Long message 6");
batches[1][1].Should().Be("Long message 7");
batches[1][2].Should().Be("Long message 8");
batches[1][3].Should().Be("Long message 9");
batches[1][4].Should().Be("Long message 10");
batches[2][0].Should().Be("Long message 11");
batches[2][1].Should().Be("Long message 12");
batches[2][2].Should().Be("Long message 13");
batches[2][3].Should().Be("Long message 14");
batches[2][4].Should().Be("Long message 15");
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(30);
}
public async Task ThrowIfUnhandled_Batch_ConsumerStoppedWhenUnhandled()
{
var receivedMessages = new List <object>();
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddEndpoints(
endpoints => endpoints
.AddOutbound <IIntegrationEvent>(new KafkaProducerEndpoint(DefaultTopicName))
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration =
{
GroupId = "consumer1",
EnableAutoCommit = false,
CommitOffsetEach = 1
},
Batch = new BatchSettings
{
Size = 3
}
}))
.AddDelegateSubscriber(
(IEnumerable <TestEventOne> messages) =>
{
foreach (var message in messages)
{
receivedMessages.Add(message);
}
})
.AddDelegateSubscriber(
async(IMessageStreamEnumerable <TestEventTwo> stream) =>
{
await foreach (var message in stream)
{
receivedMessages.Add(message);
}
}))
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(new TestEventOne());
await publisher.PublishAsync(new TestEventTwo());
await publisher.PublishAsync(new TestEventOne());
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
var consumer = Host.ScopedServiceProvider.GetRequiredService <IBroker>().Consumers[0];
receivedMessages.Should().HaveCount(3);
consumer.IsConnected.Should().BeTrue();
await publisher.PublishAsync(new TestEventTwo());
await publisher.PublishAsync(new TestEventThree());
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
await AsyncTestingUtil.WaitAsync(() => !consumer.IsConnected);
receivedMessages.Should().HaveCount(4);
consumer.IsConnected.Should().BeFalse();
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(3);
}
public async Task Batch_DisconnectWhileEnumerating_EnumerationAborted()
{
bool aborted = false;
var receivedMessages = new List <TestEventOne>();
var serviceProvider = Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(3)))
.AddEndpoints(
endpoints => endpoints
.AddOutbound <IIntegrationEvent>(new KafkaProducerEndpoint(DefaultTopicName))
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration = new KafkaConsumerConfig
{
GroupId = "consumer1",
AutoCommitIntervalMs = 100
},
Batch = new BatchSettings
{
Size = 10
}
}))
.AddDelegateSubscriber(
async(IMessageStreamEnumerable <TestEventOne> eventsStream) =>
{
try
{
await foreach (var message in eventsStream)
{
receivedMessages.Add(message);
}
}
catch (OperationCanceledException)
{
aborted = true;
}
}))
.Run();
var publisher = serviceProvider.GetRequiredService <IEventPublisher>();
for (int i = 1; i <= 10; i++)
{
await publisher.PublishAsync(new TestEventOne { Content = $"{i}" });
}
await AsyncTestingUtil.WaitAsync(() => receivedMessages.Count > 3);
receivedMessages.Should().HaveCountGreaterThan(3);
await Broker.DisconnectAsync();
await AsyncTestingUtil.WaitAsync(() => aborted);
aborted.Should().BeTrue();
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(0);
}
public async Task ExactlyOnce_InMemoryInboundLog_DuplicatedMessagesIgnored()
{
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options
.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1))
.AddInMemoryInboundLog())
.AddKafkaEndpoints(
endpoints => endpoints
.Configure(config => { config.BootstrapServers = "PLAINTEXT://e2e"; })
.AddOutbound <IIntegrationEvent>(
endpoint => endpoint.ProduceTo(DefaultTopicName))
.AddInbound(
endpoint => endpoint
.ConsumeFrom(DefaultTopicName)
.EnsureExactlyOnce(strategy => strategy.LogMessages())
.Configure(
config =>
{
config.GroupId = "consumer1";
config.EnableAutoCommit = false;
config.CommitOffsetEach = 1;
})))
.AddIntegrationSpyAndSubscriber())
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(new TestEventOne());
await publisher.PublishAsync(
new TestEventWithUniqueKey
{
UniqueKey = "1"
});
await publisher.PublishAsync(
new TestEventWithUniqueKey
{
UniqueKey = "2"
});
await publisher.PublishAsync(
new TestEventWithUniqueKey
{
UniqueKey = "1"
});
await publisher.PublishAsync(
new TestEventWithUniqueKey
{
UniqueKey = "2"
});
await publisher.PublishAsync(new TestEventOne());
await Helper.WaitUntilAllMessagesAreConsumedAsync();
Helper.Spy.InboundEnvelopes.Should().HaveCount(4);
Helper.Spy.InboundEnvelopes[0].Message.Should().BeOfType <TestEventOne>();
Helper.Spy.InboundEnvelopes[1].Message.Should().BeOfType <TestEventWithUniqueKey>();
Helper.Spy.InboundEnvelopes[1].Message.As <TestEventWithUniqueKey>().UniqueKey.Should().Be("1");
Helper.Spy.InboundEnvelopes[2].Message.Should().BeOfType <TestEventWithUniqueKey>();
Helper.Spy.InboundEnvelopes[2].Message.As <TestEventWithUniqueKey>().UniqueKey.Should().Be("2");
Helper.Spy.InboundEnvelopes[3].Message.Should().BeOfType <TestEventOne>();
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(6);
}
public async Task Batch_WithTimeout_IncompleteBatchCompletedAfterTimeout()
{
var receivedBatches = new List <List <TestEventOne> >();
var completedBatches = 0;
var serviceProvider = Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddEndpoints(
endpoints => endpoints
.AddOutbound <IIntegrationEvent>(new KafkaProducerEndpoint(DefaultTopicName))
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration = new KafkaConsumerConfig
{
GroupId = "consumer1",
EnableAutoCommit = false,
CommitOffsetEach = 1
},
Batch = new BatchSettings
{
Size = 10,
MaxWaitTime = TimeSpan.FromMilliseconds(500)
}
}))
.AddDelegateSubscriber(
async(IMessageStreamEnumerable <TestEventOne> eventsStream) =>
{
var list = new List <TestEventOne>();
receivedBatches.Add(list);
await foreach (var message in eventsStream)
{
list.Add(message);
}
completedBatches++;
}))
.Run();
var publisher = serviceProvider.GetRequiredService <IEventPublisher>();
for (int i = 1; i <= 15; i++)
{
await publisher.PublishAsync(new TestEventOne { Content = $"{i}" });
}
await AsyncTestingUtil.WaitAsync(() => receivedBatches.Sum(batch => batch.Count) == 15);
receivedBatches.Should().HaveCount(2);
receivedBatches[0].Should().HaveCount(10);
receivedBatches[1].Should().HaveCount(5);
completedBatches.Should().Be(1);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(10);
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
receivedBatches.Should().HaveCount(2);
receivedBatches[0].Should().HaveCount(10);
receivedBatches[1].Should().HaveCount(5);
completedBatches.Should().Be(2);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(15);
}
public async Task SkipPolicy_ChunkedJsonDeserializationError_SequenceSkipped()
{
var message = new TestEventOne {
Content = "Hello E2E!"
};
byte[] rawMessage = (await Endpoint.DefaultSerializer.SerializeAsync(
message,
new MessageHeaderCollection(),
MessageSerializationContext.Empty)).ReadAll() ??
throw new InvalidOperationException("Serializer returned null");
byte[] invalidRawMessage = Encoding.UTF8.GetBytes("<what?!><what?!><what?!>");
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddEndpoints(
endpoints => endpoints
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration =
{
GroupId = "consumer1",
EnableAutoCommit = false,
CommitOffsetEach = 1
},
ErrorPolicy = ErrorPolicy.Skip()
}))
.AddSingletonSubscriber <OutboundInboundSubscriber>())
.Run();
var producer = Broker.GetProducer(new KafkaProducerEndpoint(DefaultTopicName));
await producer.RawProduceAsync(
invalidRawMessage.Take(10).ToArray(),
HeadersHelper.GetChunkHeaders("1", 0, typeof(TestEventOne)));
await producer.RawProduceAsync(
invalidRawMessage.Skip(10).Take(10).ToArray(),
HeadersHelper.GetChunkHeaders("1", 1, typeof(TestEventOne)));
await producer.RawProduceAsync(
invalidRawMessage.Skip(20).ToArray(),
HeadersHelper.GetChunkHeaders("1", 2, true, typeof(TestEventOne)));
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
Subscriber.InboundEnvelopes.Should().BeEmpty();
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(3);
await producer.RawProduceAsync(
rawMessage.Take(10).ToArray(),
HeadersHelper.GetChunkHeaders("2", 0, typeof(TestEventOne)));
await producer.RawProduceAsync(
rawMessage.Skip(10).Take(10).ToArray(),
HeadersHelper.GetChunkHeaders("2", 1, typeof(TestEventOne)));
await producer.RawProduceAsync(
rawMessage.Skip(20).ToArray(),
HeadersHelper.GetChunkHeaders("2", 2, true, typeof(TestEventOne)));
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
Subscriber.InboundEnvelopes.Should().HaveCount(1);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(6);
}
public async Task RetryPolicy_BinaryFileChunkSequenceProcessedAfterSomeTries_RetriedMultipleTimesAndCommitted()
{
var message1 = new BinaryFileMessage
{
Content = new MemoryStream(
new byte[]
{
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10,
0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x20,
0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x30
}),
ContentType = "application/pdf"
};
var message2 = new BinaryFileMessage
{
Content = new MemoryStream(
new byte[]
{
0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x30,
0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40,
0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x50
}),
ContentType = "text/plain"
};
var tryCount = 0;
var receivedFiles = new List <byte[]?>();
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddEndpoints(
endpoints => endpoints
.AddOutbound <IBinaryFileMessage>(
new KafkaProducerEndpoint(DefaultTopicName)
{
Chunk = new ChunkSettings
{
Size = 10
}
})
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration =
{
GroupId = "consumer1",
EnableAutoCommit = false,
CommitOffsetEach = 1
},
ErrorPolicy = ErrorPolicy.Retry().MaxFailedAttempts(10)
}))
.AddDelegateSubscriber(
(BinaryFileMessage binaryFile) =>
{
if (binaryFile.ContentType != "text/plain")
{
tryCount++;
if (tryCount != 2)
{
// Read only first chunk
var buffer = new byte[10];
binaryFile.Content !.Read(buffer, 0, 10);
throw new InvalidOperationException("Retry!");
}
}
lock (receivedFiles)
{
receivedFiles.Add(binaryFile.Content.ReadAll());
}
})
.AddSingletonBrokerBehavior <SpyBrokerBehavior>())
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IPublisher>();
await publisher.PublishAsync(message1);
await publisher.PublishAsync(message2);
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
tryCount.Should().Be(2);
SpyBehavior.OutboundEnvelopes.Should().HaveCount(6);
SpyBehavior.OutboundEnvelopes.ForEach(envelope => envelope.RawMessage.ReReadAll() !.Length.Should().Be(10));
SpyBehavior.InboundEnvelopes.Should().HaveCount(3);
SpyBehavior.InboundEnvelopes[0].Message.As <BinaryFileMessage>().ContentType.Should().Be("application/pdf");
SpyBehavior.InboundEnvelopes[1].Message.As <BinaryFileMessage>().ContentType.Should().Be("application/pdf");
SpyBehavior.InboundEnvelopes[2].Message.As <BinaryFileMessage>().ContentType.Should().Be("text/plain");
receivedFiles.Should().HaveCount(2);
receivedFiles[0].Should().BeEquivalentTo(message1.Content.ReReadAll());
receivedFiles[1].Should().BeEquivalentTo(message2.Content.ReReadAll());
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(6);
}
public async Task Encryption_ChunkedMessages_EncryptedAndDecrypted()
{
var message1 = new TestEventOne {
Content = "Message 1"
};
var rawMessageStream1 = await Endpoint.DefaultSerializer.SerializeAsync(
message1,
new MessageHeaderCollection(),
MessageSerializationContext.Empty);
var message2 = new TestEventOne {
Content = "Message 2"
};
var rawMessageStream2 = await Endpoint.DefaultSerializer.SerializeAsync(
message2,
new MessageHeaderCollection(),
MessageSerializationContext.Empty);
var serviceProvider = Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddEndpoints(
endpoints => endpoints
.AddOutbound <IIntegrationEvent>(
new KafkaProducerEndpoint(DefaultTopicName)
{
Chunk = new ChunkSettings
{
Size = 10
},
Encryption = new SymmetricEncryptionSettings
{
Key = AesEncryptionKey
}
})
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration = new KafkaConsumerConfig
{
GroupId = "consumer1",
EnableAutoCommit = false,
CommitOffsetEach = 1
},
Encryption = new SymmetricEncryptionSettings
{
Key = AesEncryptionKey
}
}))
.AddSingletonBrokerBehavior <SpyBrokerBehavior>()
.AddSingletonSubscriber <OutboundInboundSubscriber>())
.Run();
var publisher = serviceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(message1);
await publisher.PublishAsync(message2);
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
SpyBehavior.OutboundEnvelopes.Should().HaveCount(12);
for (int i = 0; i < 6; i++)
{
SpyBehavior.OutboundEnvelopes[i].RawMessage.Should().NotBeNull();
SpyBehavior.OutboundEnvelopes[i].RawMessage !.Length.Should().BeLessOrEqualTo(10);
SpyBehavior.OutboundEnvelopes[i].RawMessage.ReReadAll().Should()
.NotBeEquivalentTo(rawMessageStream1.ReReadAll() !.Skip(i * 10).Take(10));
}
for (int i = 0; i < 6; i++)
{
SpyBehavior.OutboundEnvelopes[i + 6].RawMessage.Should().NotBeNull();
SpyBehavior.OutboundEnvelopes[i + 6].RawMessage !.Length.Should().BeLessOrEqualTo(10);
SpyBehavior.OutboundEnvelopes[i + 6].RawMessage.ReReadAll().Should()
.NotBeEquivalentTo(rawMessageStream2.ReReadAll() !.Skip(i * 10).Take(10));
}
SpyBehavior.InboundEnvelopes.Should().HaveCount(2);
SpyBehavior.InboundEnvelopes[0].Message.Should().BeEquivalentTo(message1);
SpyBehavior.InboundEnvelopes[1].Message.Should().BeEquivalentTo(message2);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(12);
}
public async Task ThrowIfUnhandled_ExceptionDisabled_MessageIgnoredWhenUnhandled()
{
var receivedMessages = new List <object>();
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(options => options.AddMockedKafka())
.AddEndpoints(
endpoints => endpoints
.AddOutbound <IIntegrationEvent>(new KafkaProducerEndpoint(DefaultTopicName))
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration =
{
GroupId = "consumer1",
EnableAutoCommit = false,
CommitOffsetEach = 1
},
ThrowIfUnhandled = false
}))
.AddSingletonBrokerBehavior <SpyBrokerBehavior>()
.AddDelegateSubscriber((TestEventOne message) => { receivedMessages.Add(message); })
.AddDelegateSubscriber(
(IEnumerable <TestEventTwo> messages) =>
{
foreach (var message in messages)
{
receivedMessages.Add(message);
}
})
.AddDelegateSubscriber(
async(IMessageStreamEnumerable <TestEventThree> stream) =>
{
await foreach (var message in stream)
{
receivedMessages.Add(message);
}
}))
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(new TestEventOne());
await publisher.PublishAsync(new TestEventTwo());
await publisher.PublishAsync(new TestEventThree());
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
var consumer = Host.ScopedServiceProvider.GetRequiredService <IBroker>().Consumers[0];
SpyBehavior.InboundEnvelopes.Should().HaveCount(3);
receivedMessages.Should().HaveCount(3);
consumer.IsConnected.Should().BeTrue();
await publisher.PublishAsync(new TestEventFour());
await publisher.PublishAsync(new TestEventThree());
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
SpyBehavior.InboundEnvelopes.Should().HaveCount(5);
receivedMessages.Should().HaveCount(4);
consumer.IsConnected.Should().BeTrue();
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(5);
}
public async Task Encryption_SimpleMessages_EncryptedAndDecrypted()
{
var message1 = new TestEventOne {
Content = "Message 1"
};
var message2 = new TestEventOne {
Content = "Message 2"
};
var serviceProvider = Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddEndpoints(
endpoints => endpoints
.AddOutbound <IIntegrationEvent>(
new KafkaProducerEndpoint(DefaultTopicName)
{
Encryption = new SymmetricEncryptionSettings
{
Key = AesEncryptionKey
}
})
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration = new KafkaConsumerConfig
{
GroupId = "consumer1",
AutoCommitIntervalMs = 100
},
Encryption = new SymmetricEncryptionSettings
{
Key = AesEncryptionKey
}
}))
.AddSingletonBrokerBehavior <SpyBrokerBehavior>()
.AddSingletonSubscriber <OutboundInboundSubscriber>())
.Run();
var publisher = serviceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(message1);
await publisher.PublishAsync(message2);
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
SpyBehavior.OutboundEnvelopes.Should().HaveCount(2);
SpyBehavior.OutboundEnvelopes[0].RawMessage.Should().BeOfType <SymmetricEncryptStream>();
SpyBehavior.OutboundEnvelopes[1].RawMessage.Should().BeOfType <SymmetricEncryptStream>();
SpyBehavior.InboundEnvelopes.Should().HaveCount(2);
SpyBehavior.InboundEnvelopes[0].Message.Should().BeEquivalentTo(message1);
SpyBehavior.InboundEnvelopes[1].Message.Should().BeEquivalentTo(message2);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(2);
}
public async Task Streaming_DisconnectWhileEnumerating_EnumerationAborted()
{
bool aborted = false;
var receivedMessages = new List <TestEventOne>();
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddEndpoints(
endpoints => endpoints
.AddOutbound <IIntegrationEvent>(new KafkaProducerEndpoint(DefaultTopicName))
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration =
{
GroupId = "consumer1",
AutoCommitIntervalMs = 100
}
}))
.AddDelegateSubscriber(
(IMessageStreamEnumerable <TestEventOne> eventsStream) =>
{
try
{
foreach (var message in eventsStream)
{
receivedMessages.Add(message);
}
}
catch (OperationCanceledException)
{
aborted = true;
}
}))
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(
new TestEventOne
{
Content = "Message 1"
});
await publisher.PublishAsync(
new TestEventOne
{
Content = "Message 2"
});
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
receivedMessages.Should().HaveCount(2);
await Broker.DisconnectAsync();
await AsyncTestingUtil.WaitAsync(() => aborted); // TODO: Necessary?
aborted.Should().BeTrue();
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(2);
}
public async Task StopAndStart_DefaultSettings_MessagesConsumedAfterRestart()
{
var serviceProvider = Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(options => options.AddMockedKafka())
.AddEndpoints(
endpoints => endpoints
.AddOutbound <IIntegrationEvent>(new KafkaProducerEndpoint(DefaultTopicName))
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration = new KafkaConsumerConfig
{
GroupId = "consumer1",
AutoCommitIntervalMs = 100
}
}))
.AddSingletonSubscriber <OutboundInboundSubscriber>())
.Run();
var publisher = serviceProvider.GetRequiredService <IEventPublisher>();
for (int i = 1; i <= 5; i++)
{
await publisher.PublishAsync(
new TestEventOne
{
Content = $"{i}"
});
}
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
Subscriber.OutboundEnvelopes.Should().HaveCount(5);
Subscriber.InboundEnvelopes.Should().HaveCount(5);
Broker.Consumers[0].Stop();
for (int i = 1; i <= 5; i++)
{
await publisher.PublishAsync(
new TestEventOne
{
Content = $"{i}"
});
}
await Task.Delay(200);
Subscriber.OutboundEnvelopes.Should().HaveCount(10);
Subscriber.InboundEnvelopes.Should().HaveCount(5);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(5);
Broker.Consumers[0].Start();
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
Subscriber.InboundEnvelopes.Should().HaveCount(10);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(10);
}
public async Task Streaming_UnboundedEnumerableProcessingFailed_ConsumerStopped()
{
var receivedMessages = new List <TestEventOne>();
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.AsObservable()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddEndpoints(
endpoints => endpoints
.AddOutbound <IIntegrationEvent>(new KafkaProducerEndpoint(DefaultTopicName))
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration =
{
GroupId = "consumer1",
EnableAutoCommit = false,
CommitOffsetEach = 1
}
}))
.AddDelegateSubscriber(
async(IMessageStreamEnumerable <TestEventOne> enumerable) =>
{
await foreach (var message in enumerable)
{
receivedMessages.Add(message);
if (receivedMessages.Count == 2)
{
throw new InvalidOperationException("Test");
}
}
}))
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(
new TestEventOne
{
Content = "Message 1"
});
await publisher.PublishAsync(
new TestEventOne
{
Content = "Message 2"
});
await publisher.PublishAsync(
new TestEventOne
{
Content = "Message 3"
});
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
await AsyncTestingUtil.WaitAsync(() => receivedMessages.Count >= 2);
receivedMessages.Should().HaveCount(2);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(1);
Broker.Consumers[0].IsConnected.Should().BeFalse();
}
public async Task Streaming_DisconnectWhileEnumerating_EnumerationAborted()
{
bool aborted = false;
var receivedMessages = new List <TestEventOne>();
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddKafkaEndpoints(
endpoints => endpoints
.Configure(
config =>
{
config.BootstrapServers = "PLAINTEXT://e2e";
})
.AddOutbound <IIntegrationEvent>(
endpoint => endpoint.ProduceTo(DefaultTopicName))
.AddInbound(
endpoint => endpoint
.ConsumeFrom(DefaultTopicName)
.Configure(
config =>
{
config.GroupId = "consumer1";
})))
.AddDelegateSubscriber(
(IEnumerable <TestEventOne> eventsStream) =>
{
try
{
foreach (var message in eventsStream)
{
receivedMessages.Add(message);
}
}
catch (OperationCanceledException)
{
Task.Delay(300).Wait();
aborted = true;
}
}))
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(
new TestEventOne
{
Content = "Message 1"
});
await publisher.PublishAsync(
new TestEventOne
{
Content = "Message 2"
});
await Helper.WaitUntilAllMessagesAreConsumedAsync();
receivedMessages.Should().HaveCount(2);
await Helper.Broker.DisconnectAsync();
aborted.Should().BeTrue();
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(2);
}
public async Task Encryption_ChunkedMessages_EncryptedAndDecrypted()
{
var message1 = new TestEventOne {
Content = "Message 1"
};
var rawMessageStream1 = await Endpoint.DefaultSerializer.SerializeAsync(
message1,
new MessageHeaderCollection(),
MessageSerializationContext.Empty);
var message2 = new TestEventOne {
Content = "Message 2"
};
var rawMessageStream2 = await Endpoint.DefaultSerializer.SerializeAsync(
message2,
new MessageHeaderCollection(),
MessageSerializationContext.Empty);
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddKafkaEndpoints(
endpoints => endpoints
.Configure(config => { config.BootstrapServers = "PLAINTEXT://e2e"; })
.AddOutbound <IIntegrationEvent>(
endpoint => endpoint
.ProduceTo(DefaultTopicName)
.EnableChunking(10)
.EncryptUsingAes(AesEncryptionKey))
.AddInbound(
endpoint => endpoint
.ConsumeFrom(DefaultTopicName)
.DecryptUsingAes(AesEncryptionKey)
.Configure(
config =>
{
config.GroupId = "consumer1";
config.EnableAutoCommit = false;
config.CommitOffsetEach = 1;
})))
.AddIntegrationSpyAndSubscriber())
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(message1);
await publisher.PublishAsync(message2);
await Helper.WaitUntilAllMessagesAreConsumedAsync();
Helper.Spy.RawOutboundEnvelopes.Should().HaveCount(12);
for (int i = 0; i < 6; i++)
{
Helper.Spy.RawOutboundEnvelopes[i].RawMessage.Should().NotBeNull();
Helper.Spy.RawOutboundEnvelopes[i].RawMessage !.Length.Should().BeLessOrEqualTo(10);
Helper.Spy.RawOutboundEnvelopes[i].RawMessage.ReReadAll().Should()
.NotBeEquivalentTo(rawMessageStream1.ReReadAll() !.Skip(i * 10).Take(10));
}
for (int i = 0; i < 6; i++)
{
Helper.Spy.RawOutboundEnvelopes[i + 6].RawMessage.Should().NotBeNull();
Helper.Spy.RawOutboundEnvelopes[i + 6].RawMessage !.Length.Should().BeLessOrEqualTo(10);
Helper.Spy.RawOutboundEnvelopes[i + 6].RawMessage.ReReadAll().Should()
.NotBeEquivalentTo(rawMessageStream2.ReReadAll() !.Skip(i * 10).Take(10));
}
Helper.Spy.InboundEnvelopes.Should().HaveCount(2);
Helper.Spy.InboundEnvelopes[0].Message.Should().BeEquivalentTo(message1);
Helper.Spy.InboundEnvelopes[1].Message.Should().BeEquivalentTo(message2);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(12);
}
public async Task Streaming_NotProcessingPartitionsIndependently_PublishedSingleStream()
{
var receivedMessages = new ConcurrentBag <TestEventOne>();
var receivedStreams = new ConcurrentBag <IMessageStreamEnumerable <TestEventOne> >();
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(3)))
.AddKafkaEndpoints(
endpoints => endpoints
.Configure(
config =>
{
config.BootstrapServers = "PLAINTEXT://e2e";
})
.AddOutbound <IIntegrationEvent>(
endpoint => endpoint
.ProduceTo(DefaultTopicName)
.WithKafkaKey <TestEventOne>(envelope => envelope.Message?.Content))
.AddInbound(
endpoint => endpoint
.ConsumeFrom(DefaultTopicName)
.ProcessAllPartitionsTogether()
.Configure(
config =>
{
config.GroupId = "consumer1";
config.EnableAutoCommit = false;
config.CommitOffsetEach = 1;
})))
.AddDelegateSubscriber(
async(IMessageStreamEnumerable <TestEventOne> eventsStream) =>
{
receivedStreams.Add(eventsStream);
await foreach (var message in eventsStream)
{
receivedMessages.Add(message);
}
}))
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
for (int i = 1; i <= 15; i++)
{
await publisher.PublishAsync(new TestEventOne { Content = $"{i}" });
}
await Helper.WaitUntilAllMessagesAreConsumedAsync();
receivedStreams.Should().HaveCount(1);
receivedMessages.Should().HaveCount(15);
receivedMessages.Select(message => message.Content)
.Should().BeEquivalentTo(Enumerable.Range(1, 15).Select(i => $"{i}"));
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(15);
}
public async Task StopAsyncAndStartAsync_DefaultSettings_MessagesConsumedAfterRestart()
{
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(options => options.AddMockedKafka())
.AddKafkaEndpoints(
endpoints => endpoints
.Configure(
config =>
{
config.BootstrapServers = "PLAINTEXT://e2e";
})
.AddOutbound <IIntegrationEvent>(
endpoint => endpoint.ProduceTo(DefaultTopicName))
.AddInbound(
endpoint => endpoint
.ConsumeFrom(DefaultTopicName)
.Configure(
config =>
{
config.GroupId = "consumer1";
})))
.AddIntegrationSpyAndSubscriber())
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
for (int i = 1; i <= 5; i++)
{
await publisher.PublishAsync(
new TestEventOne
{
Content = $"{i}"
});
}
await Helper.WaitUntilAllMessagesAreConsumedAsync();
Helper.Spy.OutboundEnvelopes.Should().HaveCount(5);
Helper.Spy.InboundEnvelopes.Should().HaveCount(5);
await Helper.Broker.Consumers[0].StopAsync();
for (int i = 1; i <= 5; i++)
{
await publisher.PublishAsync(
new TestEventOne
{
Content = $"{i}"
});
}
await Task.Delay(200);
Helper.Spy.OutboundEnvelopes.Should().HaveCount(10);
Helper.Spy.InboundEnvelopes.Should().HaveCount(5);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(5);
await Helper.Broker.Consumers[0].StartAsync();
await Helper.WaitUntilAllMessagesAreConsumedAsync();
Helper.Spy.InboundEnvelopes.Should().HaveCount(10);
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(10);
}
public async Task Chunking_SingleChunkBinaryFileConsumedInBatch_ProducedAndConsumed()
{
var batches = new List <List <string?> >();
string?failedCommit = null;
string?enumerationAborted = null;
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(
options => options.AddMockedKafka(
mockedKafkaOptions => mockedKafkaOptions.WithDefaultPartitionsCount(1)))
.AddKafkaEndpoints(
endpoints => endpoints
.Configure(config => { config.BootstrapServers = "PLAINTEXT://e2e"; })
.AddOutbound <IBinaryFileMessage>(
endpoint => endpoint
.ProduceTo(DefaultTopicName)
.EnableChunking(50))
.AddInbound(
endpoint => endpoint
.ConsumeFrom(DefaultTopicName)
.EnableBatchProcessing(5)
.Configure(
config =>
{
config.GroupId = "consumer1";
config.EnableAutoCommit = false;
config.CommitOffsetEach = 1;
})))
.AddDelegateSubscriber(
async(IAsyncEnumerable <BinaryFileMessage> streamEnumerable) =>
{
var list = new List <string?>();
batches.ThreadSafeAdd(list);
await foreach (var message in streamEnumerable)
{
var actualCommittedOffsets =
DefaultTopic.GetCommittedOffsetsCount("consumer1");
var expectedCommittedOffsets = 5 * (batches.Count - 1);
if (actualCommittedOffsets != expectedCommittedOffsets)
{
failedCommit ??=
$"{actualCommittedOffsets} != {expectedCommittedOffsets} " +
$"({batches.Count}.{list.Count})";
}
var readAll = await message.Content.ReadAllAsync();
list.Add(readAll != null ? Encoding.UTF8.GetString(readAll) : null);
}
if (list.Count != 5)
{
enumerationAborted ??=
$"Enumeration completed after {list.Count} messages " +
$"({batches.Count}.{list.Count})";
}
}))
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IPublisher>();
for (int i = 1; i <= 15; i++)
{
await publisher.PublishAsync(
new BinaryFileMessage(Encoding.UTF8.GetBytes($"Long message {i}")));
}
await Helper.WaitUntilAllMessagesAreConsumedAsync();
failedCommit.Should().BeNull();
enumerationAborted.Should().BeNull();
batches.Should().HaveCount(3);
batches[0].Should().HaveCount(5);
batches[1].Should().HaveCount(5);
batches[2].Should().HaveCount(5);
batches[0][0].Should().Be("Long message 1");
batches[0][1].Should().Be("Long message 2");
batches[0][2].Should().Be("Long message 3");
batches[0][3].Should().Be("Long message 4");
batches[0][4].Should().Be("Long message 5");
batches[1][0].Should().Be("Long message 6");
batches[1][1].Should().Be("Long message 7");
batches[1][2].Should().Be("Long message 8");
batches[1][3].Should().Be("Long message 9");
batches[1][4].Should().Be("Long message 10");
batches[2][0].Should().Be("Long message 11");
batches[2][1].Should().Be("Long message 12");
batches[2][2].Should().Be("Long message 13");
batches[2][3].Should().Be("Long message 14");
batches[2][4].Should().Be("Long message 15");
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(15);
}
public async Task ThrowIfUnhandled_ExceptionDisabled_MessageIgnoredWhenUnhandled()
{
var receivedMessages = new List <object>();
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(options => options.AddMockedKafka())
.AddKafkaEndpoints(
endpoints => endpoints
.Configure(config => { config.BootstrapServers = "PLAINTEXT://e2e"; })
.AddOutbound <IIntegrationEvent>(endpoint => endpoint.ProduceTo(DefaultTopicName))
.AddInbound(
endpoint => endpoint
.ConsumeFrom(DefaultTopicName)
.IgnoreUnhandledMessages()
.Configure(
config =>
{
config.GroupId = "consumer1";
config.EnableAutoCommit = false;
config.CommitOffsetEach = 1;
})))
.AddIntegrationSpy()
.AddDelegateSubscriber((TestEventOne message) => { receivedMessages.Add(message); })
.AddDelegateSubscriber(
(IEnumerable <TestEventTwo> messages) =>
{
foreach (var message in messages)
{
receivedMessages.Add(message);
}
})
.AddDelegateSubscriber(
async(IAsyncEnumerable <TestEventThree> stream) =>
{
await foreach (var message in stream)
{
receivedMessages.Add(message);
}
}))
.Run();
var publisher = Host.ScopedServiceProvider.GetRequiredService <IEventPublisher>();
await publisher.PublishAsync(new TestEventOne());
await publisher.PublishAsync(new TestEventTwo());
await publisher.PublishAsync(new TestEventThree());
await Helper.WaitUntilAllMessagesAreConsumedAsync();
var consumer = Helper.Broker.Consumers[0];
Helper.Spy.InboundEnvelopes.Should().HaveCount(3);
receivedMessages.Should().HaveCount(3);
consumer.IsConnected.Should().BeTrue();
await publisher.PublishAsync(new TestEventFour());
await publisher.PublishAsync(new TestEventThree());
await Helper.WaitUntilAllMessagesAreConsumedAsync();
Helper.Spy.InboundEnvelopes.Should().HaveCount(5);
receivedMessages.Should().HaveCount(4);
consumer.IsConnected.Should().BeTrue();
DefaultTopic.GetCommittedOffsetsCount("consumer1").Should().Be(5);
}
