/// <summary> /// Initializes a new instance of the <see cref="CommandProcessor"/> class. /// Use this constructor when both rpc support is required /// </summary> /// <param name="subscriberRegistry">The subscriber registry.</param> /// <param name="handlerFactory">The handler factory.</param> /// <param name="requestContextFactory">The request context factory.</param> /// <param name="policyRegistry">The policy registry.</param> /// <param name="mapperRegistry">The mapper registry.</param> /// <param name="outBox">The outbox</param> /// <param name="producerRegistry">The register of producers via whom we send messages over the external bus</param> /// <param name="replySubscriptions">The Subscriptions for creating the reply queues</param> /// <param name="responseChannelFactory">If we are expecting a response, then we need a channel to listen on</param> /// <param name="outboxTimeout">How long should we wait to write to the outbox</param> /// <param name="featureSwitchRegistry">The feature switch config provider.</param> /// <param name="inboxConfiguration">Do we want to insert an inbox handler into pipelines without the attribute. Null (default = no), yes = how to configure</param> /// <param name="boxTransactionConnectionProvider">The Box Connection Provider to use when Depositing into the outbox.</param> public CommandProcessor( IAmASubscriberRegistry subscriberRegistry, IAmAHandlerFactory handlerFactory, IAmARequestContextFactory requestContextFactory, IPolicyRegistry <string> policyRegistry, IAmAMessageMapperRegistry mapperRegistry, IAmAnOutbox <Message> outBox, IAmAProducerRegistry producerRegistry, IEnumerable <Subscription> replySubscriptions, int outboxTimeout = 300, IAmAFeatureSwitchRegistry featureSwitchRegistry = null, IAmAChannelFactory responseChannelFactory = null, InboxConfiguration inboxConfiguration = null, IAmABoxTransactionConnectionProvider boxTransactionConnectionProvider = null) : this(subscriberRegistry, handlerFactory, requestContextFactory, policyRegistry) { _mapperRegistry = mapperRegistry; _featureSwitchRegistry = featureSwitchRegistry; _responseChannelFactory = responseChannelFactory; _inboxConfiguration = inboxConfiguration; _boxTransactionConnectionProvider = boxTransactionConnectionProvider; _replySubscriptions = replySubscriptions; InitExtServiceBus(policyRegistry, outBox, outboxTimeout, producerRegistry); ConfigureCallbacks(producerRegistry); }
public KafkaMessageConsumerConfluentPreservesOrder(ITestOutputHelper output) { // -- Confluent supply these values, see their .NET examples for your account // You need to set those values as environment variables, which we then read, in order // to run these tests _bootStrapServer = Environment.GetEnvironmentVariable("CONFLUENT_BOOSTRAP_SERVER"); _userName = Environment.GetEnvironmentVariable("CONFLUENT_SASL_USERNAME"); _password = Environment.GetEnvironmentVariable("CONFLUENT_SASL_PASSWORD"); _output = output; _producerRegistry = new KafkaProducerRegistryFactory( new KafkaMessagingGatewayConfiguration { Name = "Kafka Producer Send Test", BootStrapServers = new[] { _bootStrapServer }, SecurityProtocol = Paramore.Brighter.MessagingGateway.Kafka.SecurityProtocol.SaslSsl, SaslMechanisms = Paramore.Brighter.MessagingGateway.Kafka.SaslMechanism.Plain, SaslUsername = _userName, SaslPassword = _password, SslCaLocation = SupplyCertificateLocation() }, new KafkaPublication[] { new KafkaPublication() { Topic = new RoutingKey(_topic), NumPartitions = 1, ReplicationFactor = 3, //These timeouts support running on a container using the same host as the tests, //your production values ought to be lower MessageTimeoutMs = 10000, RequestTimeoutMs = 10000, MakeChannels = OnMissingChannel.Create //This will not make the topic } }).Create(); }
//Create an instance of the ExternalBusServices if one not already set for this app. Note that we do not support reinitialization here, so once you have //set a command processor for the app, you can't call init again to set them - although the properties are not read-only so overwriting is possible //if needed as a "get out of gaol" card. private void InitExtServiceBus(IPolicyRegistry <string> policyRegistry, IAmAnOutbox <Message> outbox, int outboxTimeout, IAmAProducerRegistry producerRegistry) { if (_bus == null) { lock (padlock) { if (_bus == null) { if (producerRegistry == null) { throw new ConfigurationException("A producer registry is required to create an external bus"); } _bus = new ExternalBusServices(); if (outbox is IAmAnOutboxSync <Message> syncOutbox) { _bus.OutBox = syncOutbox; } if (outbox is IAmAnOutboxAsync <Message> asyncOutbox) { _bus.AsyncOutbox = asyncOutbox; } _bus.OutboxTimeout = outboxTimeout; _bus.PolicyRegistry = policyRegistry; _bus.ProducerRegistry = producerRegistry; } } } }
public MsSqlMessageConsumerRequeueTests() { var myCommand = new MyCommand { Value = "Test" }; Guid correlationId = Guid.NewGuid(); string replyTo = "http:\\queueUrl"; string contentType = "text\\plain"; var channelName = $"Consumer-Requeue-Tests-{Guid.NewGuid()}"; _topic = $"Consumer-Requeue-Tests-{Guid.NewGuid()}"; _message = new Message( new MessageHeader(myCommand.Id, _topic, MessageType.MT_COMMAND, correlationId, replyTo, contentType), new MessageBody(JsonSerializer.Serialize(myCommand, JsonSerialisationOptions.Options)) ); var testHelper = new MsSqlTestHelper(); testHelper.SetupQueueDb(); _subscription = new MsSqlSubscription <MyCommand>(new SubscriptionName(channelName), new ChannelName(_topic), new RoutingKey(_topic)); _producerRegistry = new MsSqlProducerRegistryFactory( testHelper.QueueConfiguration, new Publication[] { new Publication() { Topic = new RoutingKey(_topic) } } ).Create(); _channelFactory = new ChannelFactory(new MsSqlMessageConsumerFactory(testHelper.QueueConfiguration)); }
public ASBConsumerTests() { var command = new ASBTestCommand() { CommandValue = "Do the things.", CommandNumber = 26 }; _correlationId = Guid.NewGuid(); _channelName = $"Consumer-Tests-{Guid.NewGuid()}".Truncate(50); _topicName = $"Consumer-Tests-{Guid.NewGuid()}"; var routingKey = new RoutingKey(_topicName); AzureServiceBusSubscription <ASBTestCommand> subscription = new( name : new SubscriptionName(_channelName), channelName : new ChannelName(_channelName), routingKey : routingKey ); _contentType = "application/json"; _message = new Message( new MessageHeader(command.Id, _topicName, MessageType.MT_COMMAND, _correlationId, contentType: _contentType), new MessageBody(JsonSerializer.Serialize(command, JsonSerialisationOptions.Options)) ); _subscriptionConfiguration = new AzureServiceBusSubscriptionConfiguration() { DeadLetteringOnMessageExpiration = true, DefaultMessageTimeToLive = TimeSpan.FromDays(4), LockDuration = TimeSpan.FromMinutes(3), MaxDeliveryCount = 7, SqlFilter = "1=1" }; var clientProvider = ASBCreds.ASBClientProvider; _administrationClient = new AdministrationClientWrapper(clientProvider); _administrationClient.CreateSubscription(_topicName, _channelName, _subscriptionConfiguration); _serviceBusClient = clientProvider.GetServiceBusClient(); var channelFactory = new AzureServiceBusChannelFactory(new AzureServiceBusConsumerFactory(clientProvider, false)); _channel = channelFactory.CreateChannel(subscription); _producerRegistry = new AzureServiceBusProducerRegistryFactory( clientProvider, new AzureServiceBusPublication[] { new AzureServiceBusPublication { Topic = new RoutingKey(_topicName) } } ) .Create(); }
/// <summary> /// The <see cref="CommandProcessor"/> wants to support <see cref="CommandProcessor.Post{T}(T)"/> or <see cref="CommandProcessor.Repost"/> using an external bus. /// You need to provide a policy to specify how QoS issues, specifically <see cref="CommandProcessor.RETRYPOLICY "/> or <see cref="CommandProcessor.CIRCUITBREAKER "/> /// are handled by adding appropriate <see cref="Policies"/> when choosing this option. /// /// </summary> /// <param name="configuration">The Task Queues configuration.</param> /// <param name="outbox">The Outbox.</param> /// <param name="boxTransactionConnectionProvider"></param> /// <returns>INeedARequestContext.</returns> public INeedARequestContext ExternalBus(ExternalBusConfiguration configuration, IAmAnOutbox <Message> outbox, IAmABoxTransactionConnectionProvider boxTransactionConnectionProvider = null) { _useExternalBus = true; _producers = configuration.ProducerRegistry; _outbox = outbox; _overridingBoxTransactionConnectionProvider = boxTransactionConnectionProvider; _messageMapperRegistry = configuration.MessageMapperRegistry; _outboxWriteTimeout = configuration.OutboxWriteTimeout; return(this); }
private static ExternalBusType GetExternalBusType(IAmAProducerRegistry producerRegistry, IUseRpc useRequestResponse) { var externalBusType = producerRegistry == null ? ExternalBusType.None : ExternalBusType.FireAndForget; if (externalBusType == ExternalBusType.FireAndForget && useRequestResponse.RPC) { externalBusType = ExternalBusType.RPC; } return(externalBusType); }
private void ConfigureCallbacks(IAmAProducerRegistry producerRegistry) { //Only register one, to avoid two callbacks where we support both interfaces on a producer foreach (var producer in producerRegistry.Producers) { if (!_bus.ConfigurePublisherCallbackMaybe(producer)) { _bus.ConfigureAsyncPublisherCallbackMaybe(producer); } } }
/// <summary> /// The <see cref="CommandProcessor"/> wants to support <see cref="CommandProcessor.Call{T}(T)"/> using RPC between client and server /// </summary> /// <param name="configuration"></param> /// <param name="outbox">The outbox</param> /// <param name="subscriptions">Subscriptions for creating reply queues</param> /// <returns></returns> public INeedARequestContext ExternalRPC(ExternalBusConfiguration configuration, IAmAnOutbox <Message> outbox, IEnumerable <Subscription> subscriptions) { _useRequestReplyQueues = true; _replySubscriptions = subscriptions; _producers = configuration.ProducerRegistry; _messageMapperRegistry = configuration.MessageMapperRegistry; _outboxWriteTimeout = configuration.OutboxWriteTimeout; _responseChannelFactory = configuration.ResponseChannelFactory; _outbox = outbox; return(this); }
/// <summary> /// Creates the specified configuration. /// </summary> /// <param name="logger">The logger to use</param> /// <param name="outbox">The outbox for outgoing messages to the control bus</param> /// <returns>IAmAControlBusSender.</returns> public IAmAControlBusSender Create(IAmAnOutbox <Message> outbox, IAmAProducerRegistry producerRegistry) { var mapper = new MessageMapperRegistry(new SimpleMessageMapperFactory((_) => new MonitorEventMessageMapper())); mapper.Register <MonitorEvent, MonitorEventMessageMapper>(); return(new ControlBusSender(CommandProcessorBuilder.With() .Handlers(new HandlerConfiguration()) .DefaultPolicy() .ExternalBus(new ExternalBusConfiguration(producerRegistry, mapper), outbox) .RequestContextFactory(new InMemoryRequestContextFactory()) .Build() )); }
/// <summary> /// Initializes a new instance of the <see cref="ExternalBusConfiguration"/> class. /// </summary> /// <param name="producerRegistry">Clients for the external bus by topic they send to. The client details are specialised by transport</param> /// <param name="messageMapperRegistry">The message mapper registry.</param> /// <param name="outboxWriteTimeout">How long to wait when writing to the outbox</param> /// <param name="responseChannelFactory">in a request-response scenario how do we build response pipelie</param> /// <param name="useInbox">Do we want to create an inbox globally i.e. on every handler (as opposed to by hand). Defaults to null, ,by hand</param> public ExternalBusConfiguration( IAmAProducerRegistry producerRegistry, IAmAMessageMapperRegistry messageMapperRegistry, int outboxWriteTimeout = 300, IAmAChannelFactory responseChannelFactory = null, InboxConfiguration useInbox = null ) { ProducerRegistry = producerRegistry; MessageMapperRegistry = messageMapperRegistry; OutboxWriteTimeout = outboxWriteTimeout; ResponseChannelFactory = responseChannelFactory; UseInbox = useInbox; }
public ASBProducerTests() { _command = new ASBTestCommand() { CommandValue = "Do the things.", CommandNumber = 26 }; _correlationId = Guid.NewGuid(); var channelName = $"Producer-Send-Tests-{Guid.NewGuid()}".Truncate(50); _topicName = $"Producer-Send-Tests-{Guid.NewGuid()}"; var routingKey = new RoutingKey(_topicName); AzureServiceBusSubscription <ASBTestCommand> subscription = new( name : new SubscriptionName(channelName), channelName : new ChannelName(channelName), routingKey : routingKey ); _contentType = "application/json"; _message = new Message( new MessageHeader(_command.Id, _topicName, MessageType.MT_COMMAND, _correlationId, contentType: _contentType), new MessageBody(JsonSerializer.Serialize(_command, JsonSerialisationOptions.Options)) ); var clientProvider = ASBCreds.ASBClientProvider; _administrationClient = new AdministrationClientWrapper(clientProvider); _administrationClient.CreateSubscription(_topicName, channelName, new AzureServiceBusSubscriptionConfiguration()); var channelFactory = new AzureServiceBusChannelFactory(new AzureServiceBusConsumerFactory(clientProvider, false)); _channel = channelFactory.CreateChannel(subscription); _producerRegistry = new AzureServiceBusProducerRegistryFactory( clientProvider, new AzureServiceBusPublication[] { new AzureServiceBusPublication { Topic = new RoutingKey(_topicName) } } ) .Create(); }
public OrderTest() { var testHelper = new MsSqlTestHelper(); testHelper.SetupQueueDb(); var sub = new Subscription <MyCommand>(new SubscriptionName(_queueName), new ChannelName(_topic), new RoutingKey(_topic)); _producerRegistry = new MsSqlProducerRegistryFactory( testHelper.QueueConfiguration, new Publication[] { new Publication() { Topic = new RoutingKey(_topic) } } ).Create(); _consumer = new MsSqlMessageConsumerFactory(testHelper.QueueConfiguration).Create(sub); }
public KafkaMessageConsumerSweepOffsets(ITestOutputHelper output) { const string groupId = "Kafka Message Producer Sweep Test"; _output = output; _producerRegistry = new KafkaProducerRegistryFactory( new KafkaMessagingGatewayConfiguration { Name = "Kafka Producer Send Test", BootStrapServers = new[] { "localhost:9092" } }, new KafkaPublication[] { new KafkaPublication() { Topic = new RoutingKey(_topic), NumPartitions = 1, ReplicationFactor = 1, //These timeouts support running on a container using the same host as the tests, //your production values ought to be lower MessageTimeoutMs = 2000, RequestTimeoutMs = 2000, MakeChannels = OnMissingChannel.Create } }).Create(); _consumer = (KafkaMessageConsumer) new KafkaMessageConsumerFactory( new KafkaMessagingGatewayConfiguration { Name = "Kafka Consumer Test", BootStrapServers = new[] { "localhost:9092" } }) .Create(new KafkaSubscription <MyCommand>( channelName: new ChannelName(_queueName), routingKey: new RoutingKey(_topic), groupId: groupId, commitBatchSize: 20, //This large commit batch size may never be sent sweepUncommittedOffsetsIntervalMs: 10000, numOfPartitions: 1, replicationFactor: 1, makeChannels: OnMissingChannel.Create ) ); }
public KafkaConfluentProducerAssumeTests() { string SupplyCertificateLocation() { //For different platforms, we have to figure out how to get the connection right //see: https://docs.confluent.io/platform/current/tutorials/examples/clients/docs/csharp.html return(RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? "/usr/local/etc/[email protected]/cert.pem" : null); } // -- Confluent supply these values, see their .NET examples for your account // You need to set those values as environment variables, which we then read, in order // to run these tests string bootStrapServer = Environment.GetEnvironmentVariable("CONFLUENT_BOOSTRAP_SERVER"); string userName = Environment.GetEnvironmentVariable("CONFLUENT_SASL_USERNAME"); string password = Environment.GetEnvironmentVariable("CONFLUENT_SASL_PASSWORD"); _producerRegistry = new KafkaProducerRegistryFactory( new KafkaMessagingGatewayConfiguration { Name = "Kafka Producer Send Test", BootStrapServers = new[] { bootStrapServer }, SecurityProtocol = Paramore.Brighter.MessagingGateway.Kafka.SecurityProtocol.SaslSsl, SaslMechanisms = Paramore.Brighter.MessagingGateway.Kafka.SaslMechanism.Plain, SaslUsername = userName, SaslPassword = password, SslCaLocation = SupplyCertificateLocation() }, new KafkaPublication[] { new KafkaPublication() { Topic = new RoutingKey(_topic), NumPartitions = 1, ReplicationFactor = 3, //These timeouts support running on a container using the same host as the tests, //your production values ought to be lower MessageTimeoutMs = 10000, RequestTimeoutMs = 10000, MakeChannels = OnMissingChannel.Create //This will not make the topic } }).Create(); }
public KafkaMessageConsumerUpdateOffset(ITestOutputHelper output) { _output = output; _producerRegistry = new KafkaProducerRegistryFactory( new KafkaMessagingGatewayConfiguration { Name = "Kafka Producer Send Test", BootStrapServers = new[] { "localhost:9092" } }, new KafkaPublication[] { new KafkaPublication() { Topic = new RoutingKey(_topic), NumPartitions = 1, ReplicationFactor = 1, //These timeouts support running on a container using the same host as the tests, //your production values ought to be lower MessageTimeoutMs = 2000, RequestTimeoutMs = 2000, MakeChannels = OnMissingChannel.Create } }).Create(); }
/// <summary> /// Initializes a new instance of the <see cref="CommandProcessor"/> class. /// Use this constructor when only posting messages to an external bus is required /// </summary> /// <param name="requestContextFactory">The request context factory.</param> /// <param name="policyRegistry">The policy registry.</param> /// <param name="mapperRegistry">The mapper registry.</param> /// <param name="outBox">The outbox</param> /// <param name="producerRegistry">The register of producers via whom we send messages over the external bus</param> /// <param name="outboxTimeout">How long should we wait to write to the outbox</param> /// <param name="featureSwitchRegistry">The feature switch config provider.</param> /// <param name="inboxConfiguration">Do we want to insert an inbox handler into pipelines without the attribute. Null (default = no), yes = how to configure</param> /// <param name="boxTransactionConnectionProvider">The Box Connection Provider to use when Depositing into the outbox.</param> public CommandProcessor( IAmARequestContextFactory requestContextFactory, IPolicyRegistry <string> policyRegistry, IAmAMessageMapperRegistry mapperRegistry, IAmAnOutbox <Message> outBox, IAmAProducerRegistry producerRegistry, int outboxTimeout = 300, IAmAFeatureSwitchRegistry featureSwitchRegistry = null, InboxConfiguration inboxConfiguration = null, IAmABoxTransactionConnectionProvider boxTransactionConnectionProvider = null) { _requestContextFactory = requestContextFactory; _policyRegistry = policyRegistry; _mapperRegistry = mapperRegistry; _featureSwitchRegistry = featureSwitchRegistry; _inboxConfiguration = inboxConfiguration; _boxTransactionConnectionProvider = boxTransactionConnectionProvider; InitExtServiceBus(policyRegistry, outBox, outboxTimeout, producerRegistry); ConfigureCallbacks(producerRegistry); }
private static INeedARequestContext AddExternalBusMaybe( IBrighterOptions options, IAmAProducerRegistry producerRegistry, INeedMessaging messagingBuilder, MessageMapperRegistry messageMapperRegistry, InboxConfiguration inboxConfiguration, IAmAnOutboxSync <Message> outbox, IAmABoxTransactionConnectionProvider overridingConnectionProvider, IUseRpc useRequestResponse) { ExternalBusType externalBusType = GetExternalBusType(producerRegistry, useRequestResponse); if (externalBusType == ExternalBusType.None) { return(messagingBuilder.NoExternalBus()); } else if (externalBusType == ExternalBusType.FireAndForget) { return(messagingBuilder.ExternalBus( new ExternalBusConfiguration(producerRegistry, messageMapperRegistry, useInbox: inboxConfiguration), outbox, overridingConnectionProvider)); } else if (externalBusType == ExternalBusType.RPC) { return(messagingBuilder.ExternalRPC( new ExternalBusConfiguration( producerRegistry, messageMapperRegistry, responseChannelFactory: options.ChannelFactory, useInbox: inboxConfiguration), outbox, useRequestResponse.ReplyQueueSubscriptions)); } throw new ArgumentOutOfRangeException("The external bus type requested was not understood"); }
/// <summary> /// An external bus is the use of Message Oriented Middleware (MoM) to dispatch a message between a producer and a consumer. The assumption is that this /// is being used for inter-process communication, for example the work queue pattern for distributing work, or between microservicves /// Registers singletons with the service collection :- /// - Producer - the Gateway wrapping access to Middleware /// - UseRpc - do we want to use Rpc i.e. a command blocks waiting for a response, over middleware /// </summary> /// <param name="brighterBuilder">The Brighter builder to add this option to</param> /// <param name="producerRegistry">The collection of producers - clients that connect to a specific transport</param> /// <param name="useRequestResponseQueues">Add support for RPC over MoM by using a reply queue</param> /// <param name="replyQueueSubscriptions">Reply queue subscription</param> /// <returns>The Brighter builder to allow chaining of requests</returns> public static IBrighterBuilder UseExternalBus(this IBrighterBuilder brighterBuilder, IAmAProducerRegistry producerRegistry, bool useRequestResponseQueues = false, IEnumerable <Subscription> replyQueueSubscriptions = null) { brighterBuilder.Services.AddSingleton <IAmAProducerRegistry>(producerRegistry); brighterBuilder.Services.AddSingleton <IUseRpc>(new UseRpc(useRequestResponseQueues, replyQueueSubscriptions)); return(brighterBuilder); }