public EventHubMessageBusIt()
{
_loggerFactory = NullLoggerFactory.Instance;
_logger = _loggerFactory.CreateLogger <EventHubMessageBusIt>();
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
Secrets.Load(@"..\..\..\..\..\secrets.txt");
// connection details to the Azure Event Hub
var connectionString = Secrets.Service.PopulateSecrets(configuration["Azure:EventHub"]);
var storageConnectionString = Secrets.Service.PopulateSecrets(configuration["Azure:Storage"]);
var storageContainerName = configuration["Azure:ContainerName"];
Settings = new EventHubMessageBusSettings(connectionString, storageConnectionString, storageContainerName);
MessageBusBuilder = MessageBusBuilder.Create()
.WithLoggerFacory(_loggerFactory)
.WithSerializer(new JsonMessageSerializer())
.WithProviderEventHub(Settings);
MessageBus = new Lazy <EventHubMessageBus>(() => (EventHubMessageBus)MessageBusBuilder.Build());
}
/// <summary>
/// Registers the specified assemblies.
/// </summary>
/// <param name="assemblies">The assemblies.</param>
public static void Register(IEnumerable <Assembly> assemblies)
{
var registeredTypes = new HashSet <Type>();
var builder = MessageBusBuilder.Create()
.WithDependencyResolver(new MessageBusDependencyResolver())
.WithProviderMemory(new MemoryMessageBusSettings()
{
EnableMessageSerialization = false
})
.Do(builder =>
{
assemblies.ToList().ForEach(assembly =>
{
assembly.GetTypes().Where(t => t.IsClass && !t.IsAbstract)
.SelectMany(t => t.GetInterfaces(), (t, i) => new { Type = t, Interface = i })
.Where(x => x.Interface.IsGenericType && x.Interface.GetGenericTypeDefinition() == typeof(IMessageBusConsumer <>))
.Select(x => new { HandlerType = x.Type, EventType = x.Interface.GetGenericArguments()[0] })
.ToList()
.ForEach(find =>
{
// Yeah, samples are really generic. I could not be more sarcastic about this.
if (!registeredTypes.Contains(find.EventType))
{
registeredTypes.Add(find.EventType);
builder.Produce(find.EventType, x => x.DefaultTopic(x.Settings.MessageType.Name));
}
builder.Consume(find.EventType, x => x.Topic(x.MessageType.Name).WithConsumer(find.HandlerType));
});
});
});
var mbus = new MessageBus(builder.Build(), registeredTypes);
DIContainer.Container.RegisterInstance <IMessageBus>(mbus);
}
private IMessageBus BuildMessageBus(IServiceProvider serviceProvider)
{
var domainAssembly = typeof(OrderSubmittedEvent).Assembly;
var mbb = MessageBusBuilder.Create()
// declare that OrderSubmittedEvent will be produced
.Produce <OrderSubmittedEvent>(x => x.DefaultTopic(x.Settings.MessageType.Name))
// declare that OrderSubmittedEvent will be consumed by OrderSubmittedHandler
//.SubscribeTo<OrderSubmittedEvent>(x => x.Topic(x.MessageType.Name).WithSubscriber<OrderSubmittedHandler>())
// Note: we could discover messages and handlers using reflection and register them automatically
.Do(builder => domainAssembly
.GetTypes()
.Where(t => t.IsClass && !t.IsAbstract)
.SelectMany(t => t.GetInterfaces(), (t, i) => new { Type = t, Interface = i })
.Where(x => x.Interface.IsGenericType && x.Interface.GetGenericTypeDefinition() == typeof(IConsumer <>))
.Select(x => new { HandlerType = x.Type, EventType = x.Interface.GetGenericArguments()[0] })
.ToList()
.ForEach(find =>
{
builder.Consume(find.EventType, x => x.Topic(x.MessageType.Name).WithConsumer(find.HandlerType));
})
)
//.WithSerializer(new JsonMessageSerializer()) // No need to use the serializer because of `MemoryMessageBusSettings.EnableMessageSerialization = false`
.WithDependencyResolver(new AspNetCoreMessageBusDependencyResolver(serviceProvider))
.WithProviderMemory(new MemoryMessageBusSettings
{
// Do not serialize the domain events and rather pass the same instance across handlers
EnableMessageSerialization = false
});
return(mbb.Build());
}
public MessageBusBaseTest()
{
_timeZero = DateTimeOffset.Now;
_timeNow = _timeZero;
BusBuilder = MessageBusBuilder.Create()
.Produce <RequestA>(x =>
{
x.DefaultTopic("a-requests");
x.DefaultTimeout(TimeSpan.FromSeconds(TimeoutForA10));
})
.Produce <RequestB>(x =>
{
x.DefaultTopic("b-requests");
})
.ExpectRequestResponses(x =>
{
x.ReplyToTopic("app01-responses");
x.DefaultTimeout(TimeSpan.FromSeconds(TimeoutDefault20));
})
.WithDependencyResolver(new LookupDependencyResolver(t => null))
.WithSerializer(new JsonMessageSerializer())
.WithProvider(s =>
{
var bus = new MessageBusTested(s)
{
// provide current time
CurrentTimeProvider = () => _timeNow
};
return(bus);
});
_busLazy = new Lazy <MessageBusTested>(() => (MessageBusTested)BusBuilder.Build());
}
private static IMessageBus BuildMessageBus(IServiceProvider serviceProvider)
{
var domainAssembly = typeof(Startup).Assembly;
var mbb = MessageBusBuilder.Create()
.Produce <UserLeft>(x => x.DefaultTopic(x.Settings.MessageType.Name))
.Produce <UserJoined>(x => x.DefaultTopic(x.Settings.MessageType.Name))
.Produce <UserListRequested>(x => x.DefaultTopic(x.Settings.MessageType.Name))
.Produce <SendTextMessage>(x => x.DefaultTopic(x.Settings.MessageType.Name))
.Do(builder =>
{
var consumers = domainAssembly
.GetTypes()
.Where(t => t.IsClass && !t.IsAbstract)
.SelectMany(t => t.GetInterfaces(), (t, i) => new { Type = t, Interface = i })
.Where(x => x.Interface.IsGenericType && x.Interface.GetGenericTypeDefinition() == typeof(IConsumer <>))
.Select(x => new { HandlerType = x.Type, EventType = x.Interface.GetGenericArguments()[0] })
.ToList();
foreach (var consumerType in consumers)
{
builder.Consume(consumerType.EventType,
x => x.Topic(x.MessageType.Name).WithConsumer(consumerType.HandlerType));
}
})
.WithDependencyResolver(new AspNetCoreMessageBusDependencyResolver(serviceProvider))
.WithProviderMemory(new MemoryMessageBusSettings
{
EnableMessageSerialization = false
});
return(mbb.Build());
}
private IMessageBus BuildMessageBus(IServiceProvider serviceProvider)
{
// unique id across instances of this application (e.g. 1, 2, 3)
var instanceId = Configuration["InstanceId"];
var kafkaBrokers = Configuration["Kafka:Brokers"];
var instanceGroup = $"webapi-{instanceId}";
var instanceReplyTo = $"webapi-{instanceId}-response";
var messageBusBuilder = MessageBusBuilder.Create()
.Produce <GenerateThumbnailRequest>(x =>
{
// Default response timeout for this request type
//x.DefaultTimeout(TimeSpan.FromSeconds(10));
x.DefaultTopic("thumbnail-generation");
})
.ExpectRequestResponses(x =>
{
x.ReplyToTopic(instanceReplyTo);
x.Group(instanceGroup);
// Default global response timeout
x.DefaultTimeout(TimeSpan.FromSeconds(30));
})
//.WithDependencyResolverAsServiceLocator()
//.WithDependencyResolverAsAutofac()
.WithDependencyResolver(new AspNetCoreMessageBusDependencyResolver(serviceProvider))
.WithSerializer(new JsonMessageSerializer())
.WithProviderKafka(new KafkaMessageBusSettings(kafkaBrokers));
var messageBus = messageBusBuilder.Build();
return(messageBus);
}
private static IMessageBus BuildMessageBus(IConfigurationRoot configuration, IComponentContext x)
{
// unique id across instances of this application (e.g. 1, 2, 3)
var instanceId = configuration["InstanceId"];
var kafkaBrokers = configuration["Kafka:Brokers"];
var instanceGroup = $"worker-{instanceId}";
var sharedGroup = "workers";
var messageBusBuilder = MessageBusBuilder
.Create()
.Handle <GenerateThumbnailRequest, GenerateThumbnailResponse>(s =>
{
s.Topic("thumbnail-generation", t =>
{
t.WithHandler <GenerateThumbnailRequestHandler>()
.Group(sharedGroup)
.Instances(3);
});
})
.WithDependencyResolver(new AutofacMessageBusDependencyResolver(x.Resolve <ILogger <AutofacMessageBusDependencyResolver> >()))
.WithSerializer(new JsonMessageSerializer())
.WithProviderKafka(new KafkaMessageBusSettings(kafkaBrokers)
{
ConsumerConfig = (config) =>
{
config.StatisticsIntervalMs = 60000;
config.AutoOffsetReset = Confluent.Kafka.AutoOffsetReset.Latest;
}
});
var messageBus = messageBusBuilder.Build();
return(messageBus);
}
protected virtual MessageBusBase BuildBus(Action <MessageBusBuilder> builderFunc)
{
var builder = MessageBusBuilder.Create();
builder.MergeFrom(Settings);
builderFunc(builder);
var bus = builder.Build();
return((MessageBusBase)bus);
}
private static IMessageBus BuildMessageBus(IConfigurationRoot configuration)
{
var instanceId = configuration["InstanceId"];
var kafkaBrokers = configuration["Kafka:Brokers"];
var instanceGroup = $"worker-{instanceId}";
var sharedGroup = "workers";
var messageBusBuilder = MessageBusBuilder
.Create()
.Handle <GetAccountsRequest, GetAccountsResponse>(s =>
{
s.Topic("accounts-service", t =>
{
t.WithHandler <GetAccountsHandler>()
.Group(sharedGroup)
.Instances(10);
});
})
.WithDependencyResolver(new AutofacMessageBusDependencyResolver())
.WithSerializer(new JsonMessageSerializer())
.WithProviderKafka(new KafkaMessageBusSettings(kafkaBrokers)
{
ProducerConfigFactory = () => new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "queue.buffering.max.ms", 1 },
{ "socket.nagle.disable", true },
{ "message.max.bytes", 52428800 }
},
ConsumerConfigFactory = group => new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "fetch.error.backoff.ms", 1 },
{ "socket.nagle.disable", true },
{ "message.max.bytes", 52428800 },
{ "max.partition.fetch.bytes", 10048576 },
{ "fetch.max.bytes", 52428800 },
{ KafkaConfigKeys.ConsumerKeys.AutoCommitEnableMs, 5000 },
{ KafkaConfigKeys.ConsumerKeys.StatisticsIntervalMs, 500000 },
{
"default.topic.config", new Dictionary <string, object>
{
{ KafkaConfigKeys.ConsumerKeys.AutoOffsetReset, KafkaConfigValues.AutoOffsetReset.Latest }
}
}
}
});
var messageBus = messageBusBuilder.Build();
return(messageBus);
}
private IMessageBus BuildMessageBus(IServiceProvider serviceProvider)
{
// unique id across instances of this application (e.g. 1, 2, 3)
var instanceId = Configuration["InstanceId"];
var kafkaBrokers = Configuration["Kafka:Brokers"];
var instanceGroup = $"webapi-{instanceId}";
var instanceReplyTo = $"webapi-{instanceId}-response";
var messageBusBuilder = MessageBusBuilder.Create()
.Produce <GetAccountsRequest>(x =>
{
// Default response timeout for this request type
//x.DefaultTimeout(TimeSpan.FromSeconds(10));
x.DefaultTopic("accounts-service");
})
.ExpectRequestResponses(x =>
{
x.ReplyToTopic(instanceReplyTo);
x.Group(instanceGroup);
// Default global response timeout
x.DefaultTimeout(TimeSpan.FromSeconds(60));
})
.WithDependencyResolver(new AspNetCoreMessageBusDependencyResolver(serviceProvider))
.WithSerializer(new JsonMessageSerializer())
.WithProviderKafka(new KafkaMessageBusSettings(kafkaBrokers)
{
ProducerConfigFactory = () => new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "queue.buffering.max.ms", 1 },
{ "socket.nagle.disable", true },
{ "message.max.bytes", 52428800 }
},
ConsumerConfigFactory = (group) => new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "fetch.error.backoff.ms", 1 },
{ "statistics.interval.ms", 500000 },
{ "socket.nagle.disable", true },
{ "max.partition.fetch.bytes", 10048576 },
{ "fetch.max.bytes", 52428800 }
}
});
var messageBus = messageBusBuilder.Build();
return(messageBus);
}
public KafkaMessageBusIt()
{
_loggerFactory = NullLoggerFactory.Instance;
_logger = _loggerFactory.CreateLogger <KafkaMessageBusIt>();
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
Secrets.Load(@"..\..\..\..\..\secrets.txt");
var kafkaBrokers = configuration["Kafka:Brokers"];
var kafkaUsername = Secrets.Service.PopulateSecrets(configuration["Kafka:Username"]);
var kafkaPassword = Secrets.Service.PopulateSecrets(configuration["Kafka:Password"]);
// Topics on cloudkarafka.com are prefixed with username
TopicPrefix = $"{kafkaUsername}-";
KafkaSettings = new KafkaMessageBusSettings(kafkaBrokers)
{
ProducerConfig = (config) =>
{
AddSsl(kafkaUsername, kafkaPassword, config);
config.LingerMs = 5; // 5ms
config.SocketNagleDisable = true;
},
ConsumerConfig = (config) => {
AddSsl(kafkaUsername, kafkaPassword, config);
config.FetchErrorBackoffMs = 1;
config.SocketNagleDisable = true;
config.StatisticsIntervalMs = 500000;
config.AutoOffsetReset = AutoOffsetReset.Earliest;
}
};
MessageBusBuilder = MessageBusBuilder.Create()
.WithLoggerFacory(_loggerFactory)
.WithSerializer(new JsonMessageSerializer())
.WithProviderKafka(KafkaSettings);
MessageBus = new Lazy <KafkaMessageBus>(() => (KafkaMessageBus)MessageBusBuilder.Build());
}
public RedisMessageBusIt()
{
LogManager.Adapter = new DebugLoggerFactoryAdapter();
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
Secrets.Load(@"..\..\..\..\..\secrets.txt");
var connectionString = Secrets.Service.PopulateSecrets(configuration["Redis:ConnectionString"]);
MessageBusBuilder = MessageBusBuilder.Create()
.WithSerializer(new JsonMessageSerializer())
.WithProviderRedis(new RedisMessageBusSettings(connectionString));
MessageBus = new Lazy <RedisMessageBus>(() => (RedisMessageBus)MessageBusBuilder.Build());
}
private IMessageBus BuildMessageBus(IServiceProvider serviceProvider)
{
Log.InfoFormat(CultureInfo.InvariantCulture, "Instanciando the bus");
var messageBusBuilder = MessageBusBuilder.Create()
.SubscribeTo <RecognizeImageRequest>(x => x
.Topic("image-to-recognize")
.Group("workers")
.Instances(3)
.WithSubscriber <RecognizeImageRequestHandler>())
.WithDependencyResolverAsAspNetCore(serviceProvider)
.WithSerializer(new JsonMessageSerializer())
.WithProviderKafka(new KafkaMessageBusSettings(Configuration["Kafka:Brokers"]));
var messageBus = messageBusBuilder.Build();
return(messageBus);
}
public MemoryMessageBusIt()
{
LogManager.Adapter = new DebugLoggerFactoryAdapter();
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
Secrets.Load(@"..\..\..\..\..\secrets.txt");
MessageBusSettings = new MemoryMessageBusSettings();
MessageBusBuilder = MessageBusBuilder.Create()
.WithSerializer(new JsonMessageSerializer())
.WithProviderMemory(MessageBusSettings);
MessageBus = new Lazy <MemoryMessageBus>(() => (MemoryMessageBus)MessageBusBuilder.Build());
}
public KafkaMessageBusIt()
{
_loggerFactory = NullLoggerFactory.Instance;
_logger = _loggerFactory.CreateLogger <KafkaMessageBusIt>();
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
Secrets.Load(@"..\..\..\..\..\secrets.txt");
var kafkaBrokers = configuration["Kafka:Brokers"];
var kafkaUsername = Secrets.Service.PopulateSecrets(configuration["Kafka:Username"]);
var kafkaPassword = Secrets.Service.PopulateSecrets(configuration["Kafka:Password"]);
// Topics on cloudkarafka.com are prefixed with username
TopicPrefix = $"{kafkaUsername}-";
KafkaSettings = new KafkaMessageBusSettings(kafkaBrokers)
{
ProducerConfigFactory = () => AddSsl(kafkaUsername, kafkaPassword, new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "queue.buffering.max.ms", 1 },
{ "socket.nagle.disable", true },
//{"request.required.acks", 0}
}),
ConsumerConfigFactory = (group) => AddSsl(kafkaUsername, kafkaPassword, new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "fetch.error.backoff.ms", 1 },
{ "statistics.interval.ms", 500000 },
{ "socket.nagle.disable", true },
{ KafkaConfigKeys.ConsumerKeys.AutoOffsetReset, KafkaConfigValues.AutoOffsetReset.Earliest }
})
};
MessageBusBuilder = MessageBusBuilder.Create()
.WithLoggerFacory(_loggerFactory)
.WithSerializer(new JsonMessageSerializer())
.WithProviderKafka(KafkaSettings);
MessageBus = new Lazy <KafkaMessageBus>(() => (KafkaMessageBus)MessageBusBuilder.Build());
}
public MemoryMessageBusIt()
{
_loggerFactory = NullLoggerFactory.Instance;
_logger = _loggerFactory.CreateLogger <MemoryMessageBusIt>();
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
Secrets.Load(@"..\..\..\..\..\secrets.txt");
MessageBusSettings = new MemoryMessageBusSettings();
MessageBusBuilder = MessageBusBuilder.Create()
.WithLoggerFacory(_loggerFactory)
.WithSerializer(new JsonMessageSerializer())
.WithProviderMemory(MessageBusSettings);
MessageBus = new Lazy <MemoryMessageBus>(() => (MemoryMessageBus)MessageBusBuilder.Build());
}
public RedisMessageBusIt()
{
_loggerFactory = NullLoggerFactory.Instance;
_logger = _loggerFactory.CreateLogger <RedisMessageBusIt>();
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
Secrets.Load(@"..\..\..\..\..\secrets.txt");
var connectionString = Secrets.Service.PopulateSecrets(configuration["Redis:ConnectionString"]);
MessageBusBuilder = MessageBusBuilder.Create()
.WithLoggerFacory(_loggerFactory)
.WithSerializer(new JsonMessageSerializer())
.WithProviderRedis(new RedisMessageBusSettings(connectionString));
MessageBus = new Lazy <RedisMessageBus>(() => (RedisMessageBus)MessageBusBuilder.Build());
}
private static IMessageBus BuildMessageBus(IConfigurationRoot configuration, IComponentContext x)
{
// unique id across instances of this application (e.g. 1, 2, 3)
var instanceId = configuration["InstanceId"];
var kafkaBrokers = configuration["Kafka:Brokers"];
var instanceGroup = $"worker-{instanceId}";
var sharedGroup = "workers";
var messageBusBuilder = MessageBusBuilder
.Create()
.Handle <GenerateThumbnailRequest, GenerateThumbnailResponse>(s =>
{
s.Topic("thumbnail-generation", t =>
{
t.WithHandler <GenerateThumbnailRequestHandler>()
.Group(sharedGroup)
.Instances(3);
});
})
.WithDependencyResolver(new AutofacMessageBusDependencyResolver(x.Resolve <ILogger <AutofacMessageBusDependencyResolver> >()))
.WithSerializer(new JsonMessageSerializer())
.WithProviderKafka(new KafkaMessageBusSettings(kafkaBrokers)
{
ConsumerConfigFactory = group => new Dictionary <string, object>
{
{ KafkaConfigKeys.ConsumerKeys.AutoCommitEnableMs, 5000 },
{ KafkaConfigKeys.ConsumerKeys.StatisticsIntervalMs, 60000 },
{
"default.topic.config", new Dictionary <string, object>
{
{ KafkaConfigKeys.ConsumerKeys.AutoOffsetReset, KafkaConfigValues.AutoOffsetReset.Latest }
}
}
}
});
var messageBus = messageBusBuilder.Build();
return(messageBus);
}
public KafkaMessageBusIt()
{
LogManager.Adapter = new DebugLoggerFactoryAdapter();
//LogManager.Adapter = new NoOpLoggerFactoryAdapter();
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
Secrets.Load(@"..\..\..\..\..\secrets.txt");
var kafkaBrokers = configuration["Kafka:Brokers"];
var kafkaUsername = Secrets.Service.PopulateSecrets(configuration["Kafka:Username"]);
var kafkaPassword = Secrets.Service.PopulateSecrets(configuration["Kafka:Password"]);
KafkaSettings = new KafkaMessageBusSettings(kafkaBrokers)
{
ProducerConfigFactory = () => new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "queue.buffering.max.ms", 1 },
{ "socket.nagle.disable", true },
//{"request.required.acks", 0}
},
ConsumerConfigFactory = (group) => new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "fetch.error.backoff.ms", 1 },
{ "statistics.interval.ms", 500000 },
{ "socket.nagle.disable", true },
{ KafkaConfigKeys.ConsumerKeys.AutoOffsetReset, KafkaConfigValues.AutoOffsetReset.Earliest }
}
};
MessageBusBuilder = MessageBusBuilder.Create()
.WithSerializer(new JsonMessageSerializer())
.WithProviderKafka(KafkaSettings);
MessageBus = new Lazy <KafkaMessageBus>(() => (KafkaMessageBus)MessageBusBuilder.Build());
}
public IMessageBus CreateMessageBus(IServiceProvider svp)
{
var hybridBusSettings = new HybridMessageBusSettings
{
// Bus 1
["Memory"] = builder =>
{
builder
.Produce <CustomerEmailChangedEvent>(x => x.DefaultTopic(x.Settings.MessageType.Name))
.Consume <CustomerEmailChangedEvent>(x => x.Topic(x.MessageType.Name).WithConsumer <CustomerChangedEventHandler>())
.WithProviderMemory(new MemoryMessageBusSettings {
EnableMessageSerialization = false
});
},
// Bus 2
["AzureSB"] = builder =>
{
var serviceBusConnectionString = Secrets.Service.PopulateSecrets(Configuration["Azure:ServiceBus"]);
builder
.Produce <SendEmailCommand>(x => x.DefaultQueue("test-ping-queue"))
.Consume <SendEmailCommand>(x => x.Queue("test-ping-queue").WithConsumer <SmtpEmailService>())
.WithProviderServiceBus(new ServiceBusMessageBusSettings(serviceBusConnectionString));
}
};
var mbb = MessageBusBuilder.Create()
.WithDependencyResolver(new LookupDependencyResolver(svp.GetRequiredService)) // DI setup will be shared
.WithSerializer(new JsonMessageSerializer()) // serialization setup will be shared between bus 1 and 2
.WithProviderHybrid(hybridBusSettings);
// In summary:
// - The CustomerChangedEvent messages will be going through the SMB Memory provider.
// - The SendEmailCommand messages will be going through the SMB Azure Service Bus provider.
// - Each of the bus providers will serialize messages using JSON and use the same DI to resolve consumers/handlers.
var mb = mbb.Build();
return(mb);
}
public KafkaMessageBusIt()
{
LogManager.Adapter = new DebugLoggerFactoryAdapter();
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
var kafkaBrokers = configuration["Kafka:Brokers"];
KafkaSettings = new KafkaMessageBusSettings(kafkaBrokers)
{
ProducerConfigFactory = () => new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "queue.buffering.max.ms", 1 },
{ "socket.nagle.disable", true },
{ "request.timeout.ms", 2000 }, // when no response within 2 sec of sending a msg, report error
{ "message.timeout.ms", 5000 }
//{"delivery.timeout.ms", 10000 } // when no delivery ack within 10 sek, report error
},
ConsumerConfigFactory = (group) => new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "fetch.error.backoff.ms", 1 },
{ "statistics.interval.ms", 500000 },
{ "socket.nagle.disable", true },
{ KafkaConfigKeys.ConsumerKeys.AutoOffsetReset, KafkaConfigValues.AutoOffsetReset.Earliest }
}
};
MessageBusBuilder = MessageBusBuilder.Create()
.WithSerializer(new JsonMessageSerializer())
.WithProviderKafka(KafkaSettings);
MessageBus = new Lazy <KafkaMessageBus>(() => (KafkaMessageBus)MessageBusBuilder.Build());
}
/**
* Performs IMessageBus creation & configuration
*/
private static IMessageBus CreateMessageBus(IConfiguration configuration, ILoggerFactory loggerFactory)
{
// Choose your provider
var provider = Provider.Kafka;
// Provide your event hub-names OR kafka/service bus topic names
var topicForAddCommand = "add-command";
var topicForMultiplyRequest = "multiply-request";
// Note: Each running instance (node) of ConsoleApp should have its own unique response queue/topic (i.e. responses-{n})
var topicForResponses = "responses";
// Provide consumer group name
var consumerGroup = "consoleapp";
var responseGroup = "consoleapp-1";
if (provider == Provider.Kafka)
{
// Note: We are using the free plan of CloudKarafka to host the Kafka infrastructure. The free plan has a limit on topic you can get free and it requires these topic prefixes.
topicForAddCommand = "4p5ma6io-test-ping";
topicForMultiplyRequest = "4p5ma6io-multiply-request";
topicForResponses = "4p5ma6io-responses";
}
/*
*
* Azure Event Hub setup notes:
* Create 3 event hubs in Azure:
* 1. 'add-command' with 'consoleapp' group consumer
* 2. 'multiply-request' with 'consoleapp' group consumer
* 3. 'responses' with 'consoleapp-1' group consumer
*
* Azure Service Bus setup notes:
* Create 3 topics in Azure:
* 1. 'add-command' with 'consoleapp' subscription
* 2. 'multiply-request' with 'consoleapp' subscription
* 3. 'responses' with 'consoleapp-1' subscription
*
*/
// Create message bus using the fluent builder interface
IMessageBus messageBus = MessageBusBuilder
.Create()
// Pub/Sub example
.Produce <AddCommand>(x => x.DefaultTopic(topicForAddCommand)) // By default AddCommand messages will go to event-hub/topic named 'add-command'
.Consume <AddCommand>(x => x
.Topic(topicForAddCommand)
.WithConsumer <AddCommandConsumer>()
//.WithConsumer<AddCommandConsumer>(nameof(AddCommandConsumer.OnHandle))
//.WithConsumer<AddCommandConsumer>((consumer, message, name) => consumer.OnHandle(message, name))
.Group(consumerGroup) // for Apache Kafka & Azure Event Hub
.SubscriptionName(consumerGroup) // for Azure Service Bus
)
// Req/Resp example
.Produce <MultiplyRequest>(x =>
{
// By default AddCommand messages will go to topic/event-hub named 'multiply-request'
x.DefaultTopic(topicForMultiplyRequest);
if (provider == Provider.AzureServiceBus) // Azure SB specific
{
x.WithModifier((msg, sbMsg) =>
{
// Assign the Azure SB native message properties that you need
sbMsg.PartitionKey = (msg.Left * msg.Right).ToString(CultureInfo.InvariantCulture);
sbMsg.UserProperties["UserId"] = Guid.NewGuid();
});
}
if (provider == Provider.Kafka) // Kafka specific
{
// Message key could be set for the message (this is optional)
x.KeyProvider((request, topic) => Encoding.ASCII.GetBytes((request.Left + request.Right).ToString(CultureInfo.InvariantCulture)));
// Partition selector (this is optional) - assumptions that there are 2 partitions for the topic
// x.PartitionProvider((request, topic) => (request.Left + request.Right) % 2);
}
})
.Handle <MultiplyRequest, MultiplyResponse>(x => x
.Topic(topicForMultiplyRequest) // topic to expect the requests
.WithHandler <MultiplyRequestHandler>()
.Group(consumerGroup) // for Apache Kafka & Azure Event Hub
.SubscriptionName(consumerGroup) // for Azure Service Bus
)
// Configure response message queue (on topic) when using req/resp
.ExpectRequestResponses(x =>
{
x.ReplyToTopic(topicForResponses); // All responses from req/resp will return on this topic (the EventHub name)
x.Group(responseGroup); // for Apache Kafka & Azure Event Hub
x.SubscriptionName(responseGroup); // for Azure Service Bus
x.DefaultTimeout(TimeSpan.FromSeconds(20)); // Timeout request sender if response won't arrive within 10 seconds.
})
.WithLoggerFacory(loggerFactory)
.WithSerializer(new JsonMessageSerializer()) // Use JSON for message serialization
.WithDependencyResolver(new LookupDependencyResolver(type =>
{
// Simulate a dependency container
if (type == typeof(AddCommandConsumer))
{
return(new AddCommandConsumer());
}
if (type == typeof(MultiplyRequestHandler))
{
return(new MultiplyRequestHandler());
}
throw new InvalidOperationException();
}))
.Do(builder =>
{
Console.WriteLine($"Using {provider} as the transport provider");
switch (provider)
{
case Provider.Memory:
builder.WithProviderMemory(new MemoryMessageBusSettings()); // Use Azure Service Bus as provider
break;
case Provider.AzureServiceBus:
// Provide connection string to your Azure SB
var serviceBusConnectionString = Secrets.Service.PopulateSecrets(configuration["Azure:ServiceBus"]);
builder.WithProviderServiceBus(new ServiceBusMessageBusSettings(serviceBusConnectionString)); // Use Azure Service Bus as provider
break;
case Provider.AzureEventHub:
// Provide connection string to your event hub namespace
var eventHubConnectionString = Secrets.Service.PopulateSecrets(configuration["Azure:EventHub:ConnectionString"]);
var storageConnectionString = Secrets.Service.PopulateSecrets(configuration["Azure:EventHub:Storage"]);
var storageContainerName = configuration["Azure:EventHub:ContainerName"];
builder.WithProviderEventHub(new EventHubMessageBusSettings(eventHubConnectionString, storageConnectionString, storageContainerName)); // Use Azure Event Hub as provider
break;
case Provider.Kafka:
// Ensure your Kafka broker is running
var kafkaBrokers = configuration["Kafka:Brokers"];
// If your cluster requires SSL
void AddSsl(ClientConfig c)
{
// cloudkarafka.com uses SSL with SASL authentication
c.SecurityProtocol = SecurityProtocol.SaslSsl;
c.SaslUsername = Secrets.Service.PopulateSecrets(configuration["Kafka:Username"]);
c.SaslPassword = Secrets.Service.PopulateSecrets(configuration["Kafka:Password"]);
c.SaslMechanism = SaslMechanism.ScramSha256;
c.SslCaLocation = "cloudkarafka_2020-12.ca";
}
builder.WithProviderKafka(new KafkaMessageBusSettings(kafkaBrokers)
{
ProducerConfig = (config) =>
{
AddSsl(config);
},
ConsumerConfig = (config) =>
{
AddSsl(config);
}
}); // Or use Apache Kafka as provider
break;
case Provider.Redis:
// Ensure your Kafka broker is running
var redisConnectionString = Secrets.Service.PopulateSecrets(configuration["Redis:ConnectionString"]);
builder.WithProviderRedis(new RedisMessageBusSettings(redisConnectionString)); // Or use Redis as provider
break;
}
})
.Build();
return(messageBus);
}
private static IMessageBus CreateBus(IConfiguration configuration)
{
// Note: remember that Memory provider does not support req-resp yet.
var provider = Provider.Redis;
/*
* var sl = new DictionarySchemaLookupStrategy();
* /// register all your types
* sl.Add(typeof(AddCommand), AddCommand._SCHEMA);
* sl.Add(typeof(MultiplyRequest), MultiplyRequest._SCHEMA);
* sl.Add(typeof(MultiplyResponse), MultiplyResponse._SCHEMA);
*
* var mf = new DictionaryMessageCreationStategy();
* /// register all your types
* mf.Add(typeof(AddCommand), () => new AddCommand());
* mf.Add(typeof(MultiplyRequest), () => new MultiplyRequest());
* mf.Add(typeof(MultiplyResponse), () => new MultiplyResponse());
*
* // longer approach, but should be faster as it's not using reflection
* var avroSerializer = new AvroMessageSerializer(mf, sl);
*/
// alternatively a simpler approach, but using the slower ReflectionMessageCreationStategy and ReflectionSchemaLookupStrategy
var avroSerializer = new AvroMessageSerializer(NullLoggerFactory.Instance);
// Avro serialized using the AvroConvert library - no schema generation neeeded upfront.
var jsonSerializer = new JsonMessageSerializer();
// Note: Certain messages will be serialized by one Avro serializer, other using the Json serializer
var routingSerializer = new HybridMessageSerializer(new Dictionary <IMessageSerializer, Type[]>
{
[jsonSerializer] = new[] { typeof(SubtractCommand) }, // the first one will be the default serializer, no need to declare types here
[avroSerializer] = new[] { typeof(AddCommand), typeof(MultiplyRequest), typeof(MultiplyResponse) },
}, NullLogger <HybridMessageSerializer> .Instance);
return(MessageBusBuilder.Create()
.Produce <AddCommand>(x => x.DefaultTopic("AddCommand"))
.Consume <AddCommand>(x => x.Topic("AddCommand").WithConsumer <AddCommandConsumer>())
.Produce <SubtractCommand>(x => x.DefaultTopic("SubtractCommand"))
.Consume <SubtractCommand>(x => x.Topic("SubtractCommand").WithConsumer <SubtractCommandConsumer>())
.Produce <MultiplyRequest>(x => x.DefaultTopic("MultiplyRequest"))
.Handle <MultiplyRequest, MultiplyResponse>(x => x.Topic("MultiplyRequest").WithHandler <MultiplyRequestHandler>())
.ExpectRequestResponses(x => x.ReplyToTopic("ConsoleApp"))
.WithSerializer(routingSerializer) // Use Avro for message serialization
.WithDependencyResolver(new LookupDependencyResolver(type =>
{
// Simulate a dependency container
if (type == typeof(AddCommandConsumer))
{
return new AddCommandConsumer();
}
if (type == typeof(SubtractCommandConsumer))
{
return new SubtractCommandConsumer();
}
if (type == typeof(MultiplyRequestHandler))
{
return new MultiplyRequestHandler();
}
if (type == typeof(ILoggerFactory))
{
return null;
}
throw new InvalidOperationException();
}))
.Do(builder =>
{
Console.WriteLine($"Using {provider} as the transport provider");
switch (provider)
{
case Provider.Memory:
builder.WithProviderMemory(new MemoryMessageBusSettings {
EnableMessageSerialization = true
});
break;
//case Provider.AzureServiceBus:
// // Provide connection string to your Azure SB
// var serviceBusConnectionString = Secrets.Service.PopulateSecrets(configuration["Azure:ServiceBus"]);
// builder.WithProviderServiceBus(new ServiceBusMessageBusSettings(serviceBusConnectionString)); // Use Azure Service Bus as provider
// break;
//case Provider.AzureEventHub:
// // Provide connection string to your event hub namespace
// var eventHubConnectionString = Secrets.Service.PopulateSecrets(configuration["Azure:EventHub:ConnectionString"]);
// var storageConnectionString = Secrets.Service.PopulateSecrets(configuration["Azure:EventHub:Storage"]);
// var storageContainerName = configuration["Azure:EventHub:ContainerName"];
// builder.WithProviderEventHub(new EventHubMessageBusSettings(eventHubConnectionString, storageConnectionString, storageContainerName)); // Use Azure Event Hub as provider
// break;
//case Provider.Kafka:
// // Ensure your Kafka broker is running
// var kafkaBrokers = configuration["Kafka:Brokers"];
// var kafkaUsername = Secrets.Service.PopulateSecrets(configuration["Kafka:Username"]);
// var kafkaPassword = Secrets.Service.PopulateSecrets(configuration["Kafka:Password"]);
// builder.WithProviderKafka(new KafkaMessageBusSettings(kafkaBrokers)); // Or use Apache Kafka as provider
// break;
case Provider.Redis:
// Ensure your Redis broker is running
var redisConnectionString = Secrets.Service.PopulateSecrets(configuration["Redis:ConnectionString"]);
builder.WithProviderRedis(new RedisMessageBusSettings(redisConnectionString)); // Or use Redis as provider
break;
default:
throw new NotSupportedException();
}
})
.Build());
}
public static void Main(string[] args)
{
string[] pluginFilesArray =
Directory.GetFiles("plugins/", "*.dll");
ArrayList pluginFiles = new ArrayList(pluginFilesArray);
pluginFiles.Add("../Plugins/SensorNodeInputPlugin/build/net5.0/SensorNodeInputPlugin.dll");
pluginFiles.Add("../Plugins/MenuOutputPlugin/build/net5.0/MenuOutputPlugin.dll");
// Use this line to add a plugin that is not within the plugin subfolder
// dlls within the plugin folder are loaded automatically
//pluginFiles.Add(@"D:\Dokumente\Programmieren\UpdateDeutschland\middleware\PluginSrc\BasicInputPlugin\build\net5.0\BasicInputPlugin.dll");
//pluginFiles.Add(@"D:\Dokumente\Programmieren\UpdateDeutschland\middleware\PluginSrc\BasicOutputPlugin\build\net5.0\BasicOutputPlugin.dll");
foreach (string pluginFile in pluginFiles)
{
var DLL = Assembly.LoadFile(Path.GetFullPath(pluginFile));
Console.WriteLine("dll exported types: " + DLL.GetExportedTypes().Length);
foreach (Type type in DLL.GetExportedTypes())
{
try {
var c = Activator.CreateInstance(type);
IPlugin plugin = (IPlugin)c;
plugins.AddFirst(plugin);
Console.WriteLine("found plugin: " + type);
} catch (Exception e) {
if (!(e.GetType() == typeof(MissingMethodException)))
{
Console.WriteLine(e.ToString());
Console.WriteLine("invalid plugin found: " + type);
}
else
{
Console.WriteLine("ignored invalid interface class");
}
}
}
}
MessageBusBuilder builder = MessageBusBuilder.Create()
.WithSerializer(new JsonMessageSerializer())
.WithDependencyResolver(new LookupDependencyResolver(type =>
{
if (handlerResolver.ContainsKey(type))
{
return(handlerResolver[type]);
}
else if (type == typeof(ILoggerFactory))
{
return(null);
}
else
{
Console.WriteLine("other type!! " + type);
}
throw new InvalidOperationException();
}));
foreach (IPlugin plugin in plugins)
{
builder = plugin.initBus(builder);
Dictionary <System.Type, object> pluginHandlers = plugin.getHandlerResolvers();
foreach (KeyValuePair <System.Type, object> entry in pluginHandlers)
{
if (handlerResolver.ContainsKey(entry.Key))
{
Console.WriteLine("ERROR, Handler Resolver already contained!");
throw new Exception("ERROR, Handler Resolver already contained!");
}
handlerResolver[entry.Key] = entry.Value;
}
}
bus = builder
.Do(builder => {
builder.WithProviderMemory(new MemoryMessageBusSettings {
EnableMessageSerialization = true
});
})
.Build();
foreach (IPlugin plugin in plugins)
{
plugin.setMessageBus(bus);
}
foreach (IPlugin plugin in plugins)
{
var pluginTask = Task.Factory.StartNew(plugin.loop, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default);
Console.WriteLine("started plugin loop " + plugin);
}
//TestPlugin.setMessageBus(bus);
//var produceTask = Task.Factory.StartNew(TestPlugin.loop, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default);
//produceTask.Start();
while (true)
{
Thread.Sleep(100);
}
}
/**
* Performs IMessageBus creation & configuration
*/
private static IMessageBus CreateMessageBus(IConfiguration configuration)
{
// Choose your provider
var provider = Provider.AzureServiceBus;
// Provide your event hub-names OR kafka/service bus topic names
var topicForAddCommand = "add-command";
var topicForMultiplyRequest = "multiply-request";
// Note: Each running instance (node) of ConsoleApp should have its own unique response queue/topic (i.e. responses-{n})
var topicForResponses = "responses";
// Provide consumer group name
var consumerGroup = "consoleapp";
var responseGroup = "consoleapp-1";
/*
*
* Azure Event Hub setup notes:
* Create 3 event hubs in Azure:
* 1. 'add-command' with 'consoleapp' group consumer
* 2. 'multiply-request' with 'consoleapp' group consumer
* 3. 'responses' with 'consoleapp-1' group consumer
*
* Azure Service Bus setup notes:
* Create 3 topics in Azure:
* 1. 'add-command' with 'consoleapp' subscription
* 2. 'multiply-request' with 'consoleapp' subscription
* 3. 'responses' with 'consoleapp-1' subscription
*
*/
// Create message bus using the fluent builder interface
IMessageBus messageBus = MessageBusBuilder
.Create()
// Pub/Sub example
.Produce <AddCommand>(x => x.DefaultTopic(topicForAddCommand)) // By default AddCommand messages will go to event-hub/topic named 'add-command'
.Consume <AddCommand>(x => x
.Topic(topicForAddCommand)
.WithConsumer <AddCommandConsumer>()
.Group(consumerGroup) // for Apache Kafka & Azure Event Hub
.SubscriptionName(consumerGroup) // for Azure Service Bus
)
// Req/Resp example
.Produce <MultiplyRequest>(x =>
{
// By default AddCommand messages will go to topic/event-hub named 'multiply-request'
x.DefaultTopic(topicForMultiplyRequest);
if (provider == Provider.AzureServiceBus) // Azure SB specific
{
x.WithModifier((msg, sbMsg) =>
{
// Assign the Azure SB native message properties that you need
sbMsg.PartitionKey = (msg.Left * msg.Right).ToString(CultureInfo.InvariantCulture);
sbMsg.UserProperties["UserId"] = Guid.NewGuid();
});
}
if (provider == Provider.Kafka) // Kafka specific
{
// Message key could be set for the message (this is optional)
x.KeyProvider((request, topic) => Encoding.ASCII.GetBytes((request.Left + request.Right).ToString(CultureInfo.InvariantCulture)));
// Partition selector (this is optional) - assumptions that there are 2 partitions for the topic
// x.PartitionProvider((request, topic) => (request.Left + request.Right) % 2);
}
})
.Handle <MultiplyRequest, MultiplyResponse>(x => x
.Topic(topicForMultiplyRequest) // topic to expect the requests
.WithHandler <MultiplyRequestHandler>()
.Group(consumerGroup) // for Apache Kafka & Azure Event Hub
.SubscriptionName(consumerGroup) // for Azure Service Bus
)
// Configure response message queue (on topic) when using req/resp
.ExpectRequestResponses(x =>
{
x.ReplyToTopic(topicForResponses); // All responses from req/resp will return on this topic (the EventHub name)
x.Group(responseGroup); // for Apache Kafka & Azure Event Hub
x.SubscriptionName(responseGroup); // for Azure Service Bus
x.DefaultTimeout(TimeSpan.FromSeconds(20)); // Timeout request sender if response won't arrive within 10 seconds.
})
.WithSerializer(new JsonMessageSerializer()) // Use JSON for message serialization
.WithDependencyResolver(new LookupDependencyResolver(type =>
{
// Simulate a dependency container
if (type == typeof(AddCommandConsumer))
{
return(new AddCommandConsumer());
}
if (type == typeof(MultiplyRequestHandler))
{
return(new MultiplyRequestHandler());
}
throw new InvalidOperationException();
}))
.Do(builder =>
{
Console.WriteLine($"Using {provider} as the transport provider");
switch (provider)
{
case Provider.AzureServiceBus:
// Provide connection string to your Azure SB
var serviceBusConnectionString = Secrets.Service.PopulateSecrets(configuration["Azure:ServiceBus"]);
builder.WithProviderServiceBus(new ServiceBusMessageBusSettings(serviceBusConnectionString)); // Use Azure Service Bus as provider
break;
case Provider.AzureEventHub:
// Provide connection string to your event hub namespace
var eventHubConnectionString = Secrets.Service.PopulateSecrets(configuration["Azure:EventHub:ConnectionString"]);
var storageConnectionString = Secrets.Service.PopulateSecrets(configuration["Azure:EventHub:Storage"]);
var storageContainerName = configuration["Azure:EventHub:ContainerName"];
builder.WithProviderEventHub(new EventHubMessageBusSettings(eventHubConnectionString, storageConnectionString, storageContainerName)); // Use Azure Event Hub as provider
break;
case Provider.Kafka:
// Ensure your Kafka broker is running
var kafkaBrokers = configuration["Kafka:Brokers"];
builder.WithProviderKafka(new KafkaMessageBusSettings(kafkaBrokers)); // Or use Apache Kafka as provider
break;
}
})
.Build();
return(messageBus);
}
