public async Task InheritedBinaryFileMessage_ProducedAndConsumed() { var message = new CustomBinaryFileMessage { Content = new MemoryStream(new byte[] { 0x01, 0x02, 0x03, 0x04, 0x05 }), ContentType = "application/pdf", CustomHeader = "hello!" }; Host.ConfigureServices( services => services .AddLogging() .AddSilverback() .WithConnectionToMessageBroker(options => options.AddMockedKafka()) .AddEndpoints( endpoints => endpoints .AddOutbound <IBinaryFileMessage>(new KafkaProducerEndpoint("test-e2e")) .AddInbound(new KafkaConsumerEndpoint("test-e2e"))) .AddSingletonBrokerBehavior <SpyBrokerBehavior>()) .Run(); var publisher = Host.ScopedServiceProvider.GetRequiredService <IPublisher>(); await publisher.PublishAsync(message); SpyBehavior.OutboundEnvelopes.Should().HaveCount(1); SpyBehavior.OutboundEnvelopes[0].RawMessage.Should().BeEquivalentTo(message.Content); SpyBehavior.InboundEnvelopes.Should().HaveCount(1); SpyBehavior.InboundEnvelopes[0].Message.Should().BeEquivalentTo(message); SpyBehavior.InboundEnvelopes[0].Headers.Should().ContainEquivalentOf( new MessageHeader("content-type", "application/pdf")); SpyBehavior.InboundEnvelopes[0].Headers.Should().ContainEquivalentOf( new MessageHeader("x-custom-header", "hello!")); }
public async Task OnBinaryFileMessageReceived(CustomBinaryFileMessage binaryFileMessage) { EnsureTargetFolderExists(); var filename = Guid.NewGuid().ToString("N") + binaryFileMessage.Filename; _logger.LogInformation($"Saving binary file as {filename}..."); // Create a FileStream to save the file using var fileStream = File.OpenWrite(Path.Combine(OutputPath, filename)); if (binaryFileMessage.Content != null) { // Asynchronously copy the message content to the FileStream. The message chunks are streamed directly // and the entire file is never loaded into memory. await binaryFileMessage.Content.CopyToAsync(fileStream); } // Randomly simulate a processing exception if (_random.Next(2) == 1) { throw new InvalidOperationException("You must fail!"); } _logger.LogInformation($"Written {fileStream.Length} bytes into {filename}."); }
public async Task <IActionResult> ProduceBinaryFileWithCustomHeadersAsync( string filePath, string?contentType) { // Open specified file stream using var fileStream = System.IO.File.OpenRead(filePath); // Create a CustomBinaryFileMessage that wraps the file stream. The // CustomBinaryFileMessage extends the BinaryFileMessage adding an extra // Filename property that is also exported as header. var binaryFileMessage = new CustomBinaryFileMessage { Content = fileStream, Filename = Path.GetFileName(filePath) }; if (!string.IsNullOrEmpty(contentType)) { binaryFileMessage.ContentType = contentType; } // Publish the BinaryFileMessage that will be routed to the outbound // endpoint. The FileStream will be read and produced chunk by chunk, // without the entire file being loaded into memory. await _publisher.PublishAsync(binaryFileMessage); return(NoContent()); }
public async Task OnBinaryFileMessageReceivedAsync( CustomBinaryFileMessage binaryFileMessage) { EnsureTargetFolderExists(); var filename = Guid.NewGuid().ToString("N") + binaryFileMessage.Filename; _logger.LogInformation($"Saving binary file as {filename}..."); // Create a FileStream to save the file using var fileStream = File.OpenWrite(Path.Combine(OutputPath, filename)); if (binaryFileMessage.Content != null) { // Asynchronously copy the message content to the FileStream. // The message chunks are streamed directly and the entire file is // never loaded into memory. await binaryFileMessage.Content.CopyToAsync(fileStream); } _logger.LogInformation( $"Written {fileStream.Length} bytes into {filename}."); }
public async Task EncryptionAndChunkingOfInheritedBinary_EncryptedAndChunkedThenAggregatedAndDecrypted() { var message = new CustomBinaryFileMessage { Content = new MemoryStream( new byte[] { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x30 }), ContentType = "application/pdf", CustomHeader = "hello!" }; Host.ConfigureServices( services => services .AddLogging() .AddSilverback() .WithConnectionToMessageBroker(options => options.AddMockedKafka()) .AddEndpoints( endpoints => endpoints .AddOutbound <IBinaryFileMessage>( new KafkaProducerEndpoint("test-e2e") { Chunk = new ChunkSettings { Size = 10 }, Encryption = new SymmetricEncryptionSettings { Key = AesEncryptionKey } }) .AddInbound( new KafkaConsumerEndpoint("test-e2e") { Encryption = new SymmetricEncryptionSettings { Key = AesEncryptionKey } })) .AddSingletonBrokerBehavior <SpyBrokerBehavior>()) .Run(); var publisher = Host.ScopedServiceProvider.GetRequiredService <IPublisher>(); await publisher.PublishAsync(message); SpyBehavior.OutboundEnvelopes.Should().HaveCount(5); SpyBehavior.OutboundEnvelopes.SelectMany(envelope => envelope.RawMessage.ReadAll()).Should() .NotBeEquivalentTo(message.Content.ReadAll()); SpyBehavior.OutboundEnvelopes.ForEach( envelope => { envelope.RawMessage.Should().NotBeNull(); envelope.RawMessage !.Length.Should().BeLessOrEqualTo(30); }); SpyBehavior.InboundEnvelopes.Should().HaveCount(1); SpyBehavior.InboundEnvelopes[0].Message.Should().BeOfType <CustomBinaryFileMessage>(); SpyBehavior.InboundEnvelopes[0].Message.As <CustomBinaryFileMessage>().Content.Should() .BeEquivalentTo(message.Content); SpyBehavior.InboundEnvelopes[0].Message.As <CustomBinaryFileMessage>().CustomHeader.Should() .Be("hello!"); }
public async Task BinaryFile_WithoutTypeHeaderAndWithCustomHeaders_Consumed() { var message1 = new CustomBinaryFileMessage { Content = new MemoryStream( new byte[] { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x30 }), ContentType = "application/pdf", CustomHeader = "one" }; var message2 = new CustomBinaryFileMessage { Content = new MemoryStream( new byte[] { 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x30, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x50 }), ContentType = "text/plain", CustomHeader = "two" }; var receivedFiles = new List <byte[]?>(); var serviceProvider = Host.ConfigureServices( services => services .AddLogging() .AddSilverback() .UseModel() .WithConnectionToMessageBroker(options => options.AddMockedKafka()) .AddEndpoints( endpoints => endpoints .AddOutbound <IBinaryFileMessage>(new KafkaProducerEndpoint(DefaultTopicName)) .AddInbound( new KafkaConsumerEndpoint(DefaultTopicName) { Configuration = new KafkaConsumerConfig { GroupId = "consumer1", AutoCommitIntervalMs = 100 }, Serializer = new BinaryFileMessageSerializer <CustomBinaryFileMessage>() })) .AddDelegateSubscriber( (BinaryFileMessage binaryFile) => { lock (receivedFiles) { receivedFiles.Add(binaryFile.Content.ReadAll()); } }) .AddSingletonBrokerBehavior <RemoveMessageTypeHeaderProducerBehavior>() .AddSingletonBrokerBehavior <SpyBrokerBehavior>()) .Run(); var publisher = serviceProvider.GetRequiredService <IPublisher>(); await publisher.PublishAsync(message1); await publisher.PublishAsync(message2); await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync(); SpyBehavior.OutboundEnvelopes.Should().HaveCount(2); SpyBehavior.InboundEnvelopes.Should().HaveCount(2); SpyBehavior.OutboundEnvelopes.Should().HaveCount(2); SpyBehavior.InboundEnvelopes.Should().HaveCount(2); SpyBehavior.InboundEnvelopes.ForEach( envelope => { envelope.Message.Should().BeOfType <CustomBinaryFileMessage>(); envelope.Headers.GetValue(DefaultMessageHeaders.MessageType).Should().BeNull(); }); SpyBehavior.InboundEnvelopes .Select(envelope => envelope.Headers.GetValue("x-custom-header")) .Should().BeEquivalentTo("one", "two"); SpyBehavior.InboundEnvelopes .Select(envelope => envelope.Message.As <CustomBinaryFileMessage>().CustomHeader) .Should().BeEquivalentTo("one", "two"); SpyBehavior.InboundEnvelopes .Select(envelope => envelope.Message.As <BinaryFileMessage>().ContentType) .Should().BeEquivalentTo("application/pdf", "text/plain"); receivedFiles.Should().HaveCount(2); receivedFiles.Should().BeEquivalentTo(message1.Content.ReReadAll(), message2.Content.ReReadAll()); }
public async Task BinaryFile_WithCustomHeaders_ProducedAndConsumed() { var message1 = new CustomBinaryFileMessage { Content = new MemoryStream( new byte[] { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x30 }), ContentType = "application/pdf", CustomHeader = "one" }; var message2 = new CustomBinaryFileMessage { Content = new MemoryStream( new byte[] { 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x30, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x50 }), ContentType = "text/plain", CustomHeader = "two" }; var receivedFiles = new List <byte[]?>(); Host.ConfigureServices( services => services .AddLogging() .AddSilverback() .UseModel() .WithConnectionToMessageBroker(options => options.AddMockedKafka()) .AddKafkaEndpoints( endpoints => endpoints .Configure( config => { config.BootstrapServers = "PLAINTEXT://e2e"; }) .AddOutbound <IBinaryFileMessage>( endpoint => endpoint.ProduceTo(DefaultTopicName)) .AddInbound( endpoint => endpoint .ConsumeFrom(DefaultTopicName) .Configure( config => { config.GroupId = "consumer1"; }) .ConsumeBinaryFiles( serializer => serializer .UseModel <CustomBinaryFileMessage>()))) .AddDelegateSubscriber( (BinaryFileMessage binaryFile) => { lock (receivedFiles) { receivedFiles.Add(binaryFile.Content.ReadAll()); } }) .AddIntegrationSpy()) .Run(); var publisher = Host.ScopedServiceProvider.GetRequiredService <IPublisher>(); await publisher.PublishAsync(message1); await publisher.PublishAsync(message2); await Helper.WaitUntilAllMessagesAreConsumedAsync(); Helper.Spy.OutboundEnvelopes.Should().HaveCount(2); Helper.Spy.InboundEnvelopes.Should().HaveCount(2); Helper.Spy.InboundEnvelopes.ForEach( envelope => envelope.Message.Should().BeOfType <CustomBinaryFileMessage>()); Helper.Spy.InboundEnvelopes .Select(envelope => envelope.Headers.GetValue("x-custom-header")) .Should().BeEquivalentTo("one", "two"); Helper.Spy.InboundEnvelopes .Select(envelope => envelope.Message.As <CustomBinaryFileMessage>().CustomHeader) .Should().BeEquivalentTo("one", "two"); Helper.Spy.InboundEnvelopes .Select(envelope => envelope.Message.As <BinaryFileMessage>().ContentType) .Should().BeEquivalentTo("application/pdf", "text/plain"); receivedFiles.Should().HaveCount(2); receivedFiles.Should().BeEquivalentTo( new[] { message1.Content.ReReadAll(), message2.Content.ReReadAll() }); }