/// <summary>
/// Initializes a new instance of the <see cref="CommandProcessor"/> class.
/// Use this constructor when both rpc support is required
/// </summary>
/// <param name="subscriberRegistry">The subscriber registry.</param>
/// <param name="handlerFactory">The handler factory.</param>
/// <param name="requestContextFactory">The request context factory.</param>
/// <param name="policyRegistry">The policy registry.</param>
/// <param name="mapperRegistry">The mapper registry.</param>
/// <param name="outBox">The outbox</param>
/// <param name="producerRegistry">The register of producers via whom we send messages over the external bus</param>
/// <param name="replySubscriptions">The Subscriptions for creating the reply queues</param>
/// <param name="responseChannelFactory">If we are expecting a response, then we need a channel to listen on</param>
/// <param name="outboxTimeout">How long should we wait to write to the outbox</param>
/// <param name="featureSwitchRegistry">The feature switch config provider.</param>
/// <param name="inboxConfiguration">Do we want to insert an inbox handler into pipelines without the attribute. Null (default = no), yes = how to configure</param>
/// <param name="boxTransactionConnectionProvider">The Box Connection Provider to use when Depositing into the outbox.</param>
public CommandProcessor(
IAmASubscriberRegistry subscriberRegistry,
IAmAHandlerFactory handlerFactory,
IAmARequestContextFactory requestContextFactory,
IPolicyRegistry <string> policyRegistry,
IAmAMessageMapperRegistry mapperRegistry,
IAmAnOutbox <Message> outBox,
IAmAProducerRegistry producerRegistry,
IEnumerable <Subscription> replySubscriptions,
int outboxTimeout = 300,
IAmAFeatureSwitchRegistry featureSwitchRegistry = null,
IAmAChannelFactory responseChannelFactory = null,
InboxConfiguration inboxConfiguration = null,
IAmABoxTransactionConnectionProvider boxTransactionConnectionProvider = null)
: this(subscriberRegistry, handlerFactory, requestContextFactory, policyRegistry)
{
_mapperRegistry = mapperRegistry;
_featureSwitchRegistry = featureSwitchRegistry;
_responseChannelFactory = responseChannelFactory;
_inboxConfiguration = inboxConfiguration;
_boxTransactionConnectionProvider = boxTransactionConnectionProvider;
_replySubscriptions = replySubscriptions;
InitExtServiceBus(policyRegistry, outBox, outboxTimeout, producerRegistry);
ConfigureCallbacks(producerRegistry);
}
public KafkaMessageConsumerConfluentPreservesOrder(ITestOutputHelper output)
{
// -- Confluent supply these values, see their .NET examples for your account
// You need to set those values as environment variables, which we then read, in order
// to run these tests
_bootStrapServer = Environment.GetEnvironmentVariable("CONFLUENT_BOOSTRAP_SERVER");
_userName = Environment.GetEnvironmentVariable("CONFLUENT_SASL_USERNAME");
_password = Environment.GetEnvironmentVariable("CONFLUENT_SASL_PASSWORD");
_output = output;
_producerRegistry = new KafkaProducerRegistryFactory(
new KafkaMessagingGatewayConfiguration
{
Name = "Kafka Producer Send Test",
BootStrapServers = new[] { _bootStrapServer },
SecurityProtocol = Paramore.Brighter.MessagingGateway.Kafka.SecurityProtocol.SaslSsl,
SaslMechanisms = Paramore.Brighter.MessagingGateway.Kafka.SaslMechanism.Plain,
SaslUsername = _userName,
SaslPassword = _password,
SslCaLocation = SupplyCertificateLocation()
},
new KafkaPublication[] { new KafkaPublication()
{
Topic = new RoutingKey(_topic),
NumPartitions = 1,
ReplicationFactor = 3,
//These timeouts support running on a container using the same host as the tests,
//your production values ought to be lower
MessageTimeoutMs = 10000,
RequestTimeoutMs = 10000,
MakeChannels = OnMissingChannel.Create //This will not make the topic
} }).Create();
}
//Create an instance of the ExternalBusServices if one not already set for this app. Note that we do not support reinitialization here, so once you have
//set a command processor for the app, you can't call init again to set them - although the properties are not read-only so overwriting is possible
//if needed as a "get out of gaol" card.
private void InitExtServiceBus(IPolicyRegistry <string> policyRegistry,
IAmAnOutbox <Message> outbox,
int outboxTimeout,
IAmAProducerRegistry producerRegistry)
{
if (_bus == null)
{
lock (padlock)
{
if (_bus == null)
{
if (producerRegistry == null)
{
throw new ConfigurationException("A producer registry is required to create an external bus");
}
_bus = new ExternalBusServices();
if (outbox is IAmAnOutboxSync <Message> syncOutbox)
{
_bus.OutBox = syncOutbox;
}
if (outbox is IAmAnOutboxAsync <Message> asyncOutbox)
{
_bus.AsyncOutbox = asyncOutbox;
}
_bus.OutboxTimeout = outboxTimeout;
_bus.PolicyRegistry = policyRegistry;
_bus.ProducerRegistry = producerRegistry;
}
}
}
}
public MsSqlMessageConsumerRequeueTests()
{
var myCommand = new MyCommand {
Value = "Test"
};
Guid correlationId = Guid.NewGuid();
string replyTo = "http:\\queueUrl";
string contentType = "text\\plain";
var channelName = $"Consumer-Requeue-Tests-{Guid.NewGuid()}";
_topic = $"Consumer-Requeue-Tests-{Guid.NewGuid()}";
_message = new Message(
new MessageHeader(myCommand.Id, _topic, MessageType.MT_COMMAND, correlationId, replyTo, contentType),
new MessageBody(JsonSerializer.Serialize(myCommand, JsonSerialisationOptions.Options))
);
var testHelper = new MsSqlTestHelper();
testHelper.SetupQueueDb();
_subscription = new MsSqlSubscription <MyCommand>(new SubscriptionName(channelName),
new ChannelName(_topic), new RoutingKey(_topic));
_producerRegistry = new MsSqlProducerRegistryFactory(
testHelper.QueueConfiguration,
new Publication[] { new Publication()
{
Topic = new RoutingKey(_topic)
} }
).Create();
_channelFactory = new ChannelFactory(new MsSqlMessageConsumerFactory(testHelper.QueueConfiguration));
}
public ASBConsumerTests()
{
var command = new ASBTestCommand()
{
CommandValue = "Do the things.",
CommandNumber = 26
};
_correlationId = Guid.NewGuid();
_channelName = $"Consumer-Tests-{Guid.NewGuid()}".Truncate(50);
_topicName = $"Consumer-Tests-{Guid.NewGuid()}";
var routingKey = new RoutingKey(_topicName);
AzureServiceBusSubscription <ASBTestCommand> subscription = new(
name : new SubscriptionName(_channelName),
channelName : new ChannelName(_channelName),
routingKey : routingKey
);
_contentType = "application/json";
_message = new Message(
new MessageHeader(command.Id, _topicName, MessageType.MT_COMMAND, _correlationId, contentType: _contentType),
new MessageBody(JsonSerializer.Serialize(command, JsonSerialisationOptions.Options))
);
_subscriptionConfiguration = new AzureServiceBusSubscriptionConfiguration()
{
DeadLetteringOnMessageExpiration = true,
DefaultMessageTimeToLive = TimeSpan.FromDays(4),
LockDuration = TimeSpan.FromMinutes(3),
MaxDeliveryCount = 7,
SqlFilter = "1=1"
};
var clientProvider = ASBCreds.ASBClientProvider;
_administrationClient = new AdministrationClientWrapper(clientProvider);
_administrationClient.CreateSubscription(_topicName, _channelName, _subscriptionConfiguration);
_serviceBusClient = clientProvider.GetServiceBusClient();
var channelFactory =
new AzureServiceBusChannelFactory(new AzureServiceBusConsumerFactory(clientProvider, false));
_channel = channelFactory.CreateChannel(subscription);
_producerRegistry = new AzureServiceBusProducerRegistryFactory(
clientProvider,
new AzureServiceBusPublication[]
{
new AzureServiceBusPublication {
Topic = new RoutingKey(_topicName)
}
}
)
.Create();
}
/// <summary>
/// The <see cref="CommandProcessor"/> wants to support <see cref="CommandProcessor.Post{T}(T)"/> or <see cref="CommandProcessor.Repost"/> using an external bus.
/// You need to provide a policy to specify how QoS issues, specifically <see cref="CommandProcessor.RETRYPOLICY "/> or <see cref="CommandProcessor.CIRCUITBREAKER "/>
/// are handled by adding appropriate <see cref="Policies"/> when choosing this option.
///
/// </summary>
/// <param name="configuration">The Task Queues configuration.</param>
/// <param name="outbox">The Outbox.</param>
/// <param name="boxTransactionConnectionProvider"></param>
/// <returns>INeedARequestContext.</returns>
public INeedARequestContext ExternalBus(ExternalBusConfiguration configuration, IAmAnOutbox <Message> outbox, IAmABoxTransactionConnectionProvider boxTransactionConnectionProvider = null)
{
_useExternalBus = true;
_producers = configuration.ProducerRegistry;
_outbox = outbox;
_overridingBoxTransactionConnectionProvider = boxTransactionConnectionProvider;
_messageMapperRegistry = configuration.MessageMapperRegistry;
_outboxWriteTimeout = configuration.OutboxWriteTimeout;
return(this);
}
private static ExternalBusType GetExternalBusType(IAmAProducerRegistry producerRegistry, IUseRpc useRequestResponse)
{
var externalBusType = producerRegistry == null ? ExternalBusType.None : ExternalBusType.FireAndForget;
if (externalBusType == ExternalBusType.FireAndForget && useRequestResponse.RPC)
{
externalBusType = ExternalBusType.RPC;
}
return(externalBusType);
}
private void ConfigureCallbacks(IAmAProducerRegistry producerRegistry)
{
//Only register one, to avoid two callbacks where we support both interfaces on a producer
foreach (var producer in producerRegistry.Producers)
{
if (!_bus.ConfigurePublisherCallbackMaybe(producer))
{
_bus.ConfigureAsyncPublisherCallbackMaybe(producer);
}
}
}
/// <summary>
/// The <see cref="CommandProcessor"/> wants to support <see cref="CommandProcessor.Call{T}(T)"/> using RPC between client and server
/// </summary>
/// <param name="configuration"></param>
/// <param name="outbox">The outbox</param>
/// <param name="subscriptions">Subscriptions for creating reply queues</param>
/// <returns></returns>
public INeedARequestContext ExternalRPC(ExternalBusConfiguration configuration, IAmAnOutbox <Message> outbox, IEnumerable <Subscription> subscriptions)
{
_useRequestReplyQueues = true;
_replySubscriptions = subscriptions;
_producers = configuration.ProducerRegistry;
_messageMapperRegistry = configuration.MessageMapperRegistry;
_outboxWriteTimeout = configuration.OutboxWriteTimeout;
_responseChannelFactory = configuration.ResponseChannelFactory;
_outbox = outbox;
return(this);
}
/// <summary>
/// Creates the specified configuration.
/// </summary>
/// <param name="logger">The logger to use</param>
/// <param name="outbox">The outbox for outgoing messages to the control bus</param>
/// <returns>IAmAControlBusSender.</returns>
public IAmAControlBusSender Create(IAmAnOutbox <Message> outbox, IAmAProducerRegistry producerRegistry)
{
var mapper = new MessageMapperRegistry(new SimpleMessageMapperFactory((_) => new MonitorEventMessageMapper()));
mapper.Register <MonitorEvent, MonitorEventMessageMapper>();
return(new ControlBusSender(CommandProcessorBuilder.With()
.Handlers(new HandlerConfiguration())
.DefaultPolicy()
.ExternalBus(new ExternalBusConfiguration(producerRegistry, mapper), outbox)
.RequestContextFactory(new InMemoryRequestContextFactory())
.Build()
));
}
/// <summary>
/// Initializes a new instance of the <see cref="ExternalBusConfiguration"/> class.
/// </summary>
/// <param name="producerRegistry">Clients for the external bus by topic they send to. The client details are specialised by transport</param>
/// <param name="messageMapperRegistry">The message mapper registry.</param>
/// <param name="outboxWriteTimeout">How long to wait when writing to the outbox</param>
/// <param name="responseChannelFactory">in a request-response scenario how do we build response pipelie</param>
/// <param name="useInbox">Do we want to create an inbox globally i.e. on every handler (as opposed to by hand). Defaults to null, ,by hand</param>
public ExternalBusConfiguration(
IAmAProducerRegistry producerRegistry,
IAmAMessageMapperRegistry messageMapperRegistry,
int outboxWriteTimeout = 300,
IAmAChannelFactory responseChannelFactory = null,
InboxConfiguration useInbox = null
)
{
ProducerRegistry = producerRegistry;
MessageMapperRegistry = messageMapperRegistry;
OutboxWriteTimeout = outboxWriteTimeout;
ResponseChannelFactory = responseChannelFactory;
UseInbox = useInbox;
}
public ASBProducerTests()
{
_command = new ASBTestCommand()
{
CommandValue = "Do the things.",
CommandNumber = 26
};
_correlationId = Guid.NewGuid();
var channelName = $"Producer-Send-Tests-{Guid.NewGuid()}".Truncate(50);
_topicName = $"Producer-Send-Tests-{Guid.NewGuid()}";
var routingKey = new RoutingKey(_topicName);
AzureServiceBusSubscription <ASBTestCommand> subscription = new(
name : new SubscriptionName(channelName),
channelName : new ChannelName(channelName),
routingKey : routingKey
);
_contentType = "application/json";
_message = new Message(
new MessageHeader(_command.Id, _topicName, MessageType.MT_COMMAND, _correlationId, contentType: _contentType),
new MessageBody(JsonSerializer.Serialize(_command, JsonSerialisationOptions.Options))
);
var clientProvider = ASBCreds.ASBClientProvider;
_administrationClient = new AdministrationClientWrapper(clientProvider);
_administrationClient.CreateSubscription(_topicName, channelName, new AzureServiceBusSubscriptionConfiguration());
var channelFactory =
new AzureServiceBusChannelFactory(new AzureServiceBusConsumerFactory(clientProvider, false));
_channel = channelFactory.CreateChannel(subscription);
_producerRegistry = new AzureServiceBusProducerRegistryFactory(
clientProvider,
new AzureServiceBusPublication[]
{
new AzureServiceBusPublication {
Topic = new RoutingKey(_topicName)
}
}
)
.Create();
}
public OrderTest()
{
var testHelper = new MsSqlTestHelper();
testHelper.SetupQueueDb();
var sub = new Subscription <MyCommand>(new SubscriptionName(_queueName),
new ChannelName(_topic), new RoutingKey(_topic));
_producerRegistry = new MsSqlProducerRegistryFactory(
testHelper.QueueConfiguration,
new Publication[] { new Publication()
{
Topic = new RoutingKey(_topic)
} }
).Create();
_consumer = new MsSqlMessageConsumerFactory(testHelper.QueueConfiguration).Create(sub);
}
public KafkaMessageConsumerSweepOffsets(ITestOutputHelper output)
{
const string groupId = "Kafka Message Producer Sweep Test";
_output = output;
_producerRegistry = new KafkaProducerRegistryFactory(
new KafkaMessagingGatewayConfiguration
{
Name = "Kafka Producer Send Test",
BootStrapServers = new[] { "localhost:9092" }
},
new KafkaPublication[] { new KafkaPublication()
{
Topic = new RoutingKey(_topic),
NumPartitions = 1,
ReplicationFactor = 1,
//These timeouts support running on a container using the same host as the tests,
//your production values ought to be lower
MessageTimeoutMs = 2000,
RequestTimeoutMs = 2000,
MakeChannels = OnMissingChannel.Create
} }).Create();
_consumer = (KafkaMessageConsumer) new KafkaMessageConsumerFactory(
new KafkaMessagingGatewayConfiguration
{
Name = "Kafka Consumer Test",
BootStrapServers = new[] { "localhost:9092" }
})
.Create(new KafkaSubscription <MyCommand>(
channelName: new ChannelName(_queueName),
routingKey: new RoutingKey(_topic),
groupId: groupId,
commitBatchSize: 20, //This large commit batch size may never be sent
sweepUncommittedOffsetsIntervalMs: 10000,
numOfPartitions: 1,
replicationFactor: 1,
makeChannels: OnMissingChannel.Create
)
);
}
public KafkaConfluentProducerAssumeTests()
{
string SupplyCertificateLocation()
{
//For different platforms, we have to figure out how to get the connection right
//see: https://docs.confluent.io/platform/current/tutorials/examples/clients/docs/csharp.html
return(RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? "/usr/local/etc/[email protected]/cert.pem" : null);
}
// -- Confluent supply these values, see their .NET examples for your account
// You need to set those values as environment variables, which we then read, in order
// to run these tests
string bootStrapServer = Environment.GetEnvironmentVariable("CONFLUENT_BOOSTRAP_SERVER");
string userName = Environment.GetEnvironmentVariable("CONFLUENT_SASL_USERNAME");
string password = Environment.GetEnvironmentVariable("CONFLUENT_SASL_PASSWORD");
_producerRegistry = new KafkaProducerRegistryFactory(
new KafkaMessagingGatewayConfiguration
{
Name = "Kafka Producer Send Test",
BootStrapServers = new[] { bootStrapServer },
SecurityProtocol = Paramore.Brighter.MessagingGateway.Kafka.SecurityProtocol.SaslSsl,
SaslMechanisms = Paramore.Brighter.MessagingGateway.Kafka.SaslMechanism.Plain,
SaslUsername = userName,
SaslPassword = password,
SslCaLocation = SupplyCertificateLocation()
},
new KafkaPublication[] { new KafkaPublication()
{
Topic = new RoutingKey(_topic),
NumPartitions = 1,
ReplicationFactor = 3,
//These timeouts support running on a container using the same host as the tests,
//your production values ought to be lower
MessageTimeoutMs = 10000,
RequestTimeoutMs = 10000,
MakeChannels = OnMissingChannel.Create //This will not make the topic
} }).Create();
}
public KafkaMessageConsumerUpdateOffset(ITestOutputHelper output)
{
_output = output;
_producerRegistry = new KafkaProducerRegistryFactory(
new KafkaMessagingGatewayConfiguration
{
Name = "Kafka Producer Send Test",
BootStrapServers = new[] { "localhost:9092" }
},
new KafkaPublication[] { new KafkaPublication()
{
Topic = new RoutingKey(_topic),
NumPartitions = 1,
ReplicationFactor = 1,
//These timeouts support running on a container using the same host as the tests,
//your production values ought to be lower
MessageTimeoutMs = 2000,
RequestTimeoutMs = 2000,
MakeChannels = OnMissingChannel.Create
} }).Create();
}
/// <summary>
/// Initializes a new instance of the <see cref="CommandProcessor"/> class.
/// Use this constructor when only posting messages to an external bus is required
/// </summary>
/// <param name="requestContextFactory">The request context factory.</param>
/// <param name="policyRegistry">The policy registry.</param>
/// <param name="mapperRegistry">The mapper registry.</param>
/// <param name="outBox">The outbox</param>
/// <param name="producerRegistry">The register of producers via whom we send messages over the external bus</param>
/// <param name="outboxTimeout">How long should we wait to write to the outbox</param>
/// <param name="featureSwitchRegistry">The feature switch config provider.</param>
/// <param name="inboxConfiguration">Do we want to insert an inbox handler into pipelines without the attribute. Null (default = no), yes = how to configure</param>
/// <param name="boxTransactionConnectionProvider">The Box Connection Provider to use when Depositing into the outbox.</param>
public CommandProcessor(
IAmARequestContextFactory requestContextFactory,
IPolicyRegistry <string> policyRegistry,
IAmAMessageMapperRegistry mapperRegistry,
IAmAnOutbox <Message> outBox,
IAmAProducerRegistry producerRegistry,
int outboxTimeout = 300,
IAmAFeatureSwitchRegistry featureSwitchRegistry = null,
InboxConfiguration inboxConfiguration = null,
IAmABoxTransactionConnectionProvider boxTransactionConnectionProvider = null)
{
_requestContextFactory = requestContextFactory;
_policyRegistry = policyRegistry;
_mapperRegistry = mapperRegistry;
_featureSwitchRegistry = featureSwitchRegistry;
_inboxConfiguration = inboxConfiguration;
_boxTransactionConnectionProvider = boxTransactionConnectionProvider;
InitExtServiceBus(policyRegistry, outBox, outboxTimeout, producerRegistry);
ConfigureCallbacks(producerRegistry);
}
private static INeedARequestContext AddExternalBusMaybe(
IBrighterOptions options,
IAmAProducerRegistry producerRegistry,
INeedMessaging messagingBuilder,
MessageMapperRegistry messageMapperRegistry,
InboxConfiguration inboxConfiguration,
IAmAnOutboxSync <Message> outbox,
IAmABoxTransactionConnectionProvider overridingConnectionProvider,
IUseRpc useRequestResponse)
{
ExternalBusType externalBusType = GetExternalBusType(producerRegistry, useRequestResponse);
if (externalBusType == ExternalBusType.None)
{
return(messagingBuilder.NoExternalBus());
}
else if (externalBusType == ExternalBusType.FireAndForget)
{
return(messagingBuilder.ExternalBus(
new ExternalBusConfiguration(producerRegistry, messageMapperRegistry, useInbox: inboxConfiguration),
outbox,
overridingConnectionProvider));
}
else if (externalBusType == ExternalBusType.RPC)
{
return(messagingBuilder.ExternalRPC(
new ExternalBusConfiguration(
producerRegistry,
messageMapperRegistry,
responseChannelFactory: options.ChannelFactory,
useInbox: inboxConfiguration),
outbox,
useRequestResponse.ReplyQueueSubscriptions));
}
throw new ArgumentOutOfRangeException("The external bus type requested was not understood");
}
/// <summary>
/// An external bus is the use of Message Oriented Middleware (MoM) to dispatch a message between a producer and a consumer. The assumption is that this
/// is being used for inter-process communication, for example the work queue pattern for distributing work, or between microservicves
/// Registers singletons with the service collection :-
/// - Producer - the Gateway wrapping access to Middleware
/// - UseRpc - do we want to use Rpc i.e. a command blocks waiting for a response, over middleware
/// </summary>
/// <param name="brighterBuilder">The Brighter builder to add this option to</param>
/// <param name="producerRegistry">The collection of producers - clients that connect to a specific transport</param>
/// <param name="useRequestResponseQueues">Add support for RPC over MoM by using a reply queue</param>
/// <param name="replyQueueSubscriptions">Reply queue subscription</param>
/// <returns>The Brighter builder to allow chaining of requests</returns>
public static IBrighterBuilder UseExternalBus(this IBrighterBuilder brighterBuilder, IAmAProducerRegistry producerRegistry, bool useRequestResponseQueues = false, IEnumerable <Subscription> replyQueueSubscriptions = null)
{
brighterBuilder.Services.AddSingleton <IAmAProducerRegistry>(producerRegistry);
brighterBuilder.Services.AddSingleton <IUseRpc>(new UseRpc(useRequestResponseQueues, replyQueueSubscriptions));
return(brighterBuilder);
}
