public KafkaMessageBusIt() { var configuration = new ConfigurationBuilder() .AddJsonFile("appsettings.json") .Build(); var kafkaBrokers = configuration["Kafka:Brokers"]; KafkaSettings = new KafkaMessageBusSettings(kafkaBrokers) { ProducerConfigFactory = () => new Dictionary <string, object> { { "socket.blocking.max.ms", 1 }, { "queue.buffering.max.ms", 1 }, { "socket.nagle.disable", true } }, ConsumerConfigFactory = (group) => new Dictionary <string, object> { { "socket.blocking.max.ms", 1 }, { "fetch.error.backoff.ms", 1 }, { "statistics.interval.ms", 500000 }, { "socket.nagle.disable", true }, { KafkaConfigKeys.Consumer.AutoOffsetReset, KafkaConfigValues.AutoOffsetReset.Earliest } } }; MessageBusBuilder = new MessageBusBuilder() .WithSerializer(new JsonMessageSerializer()) .WithProviderKafka(KafkaSettings); MessageBus = new Lazy <KafkaMessageBus>(() => (KafkaMessageBus)MessageBusBuilder.Build()); }
public KafkaMessageBusIt() { // address to the Kafka broker var kafkaBrokers = "localhost:9092"; KafkaSettings = new KafkaMessageBusSettings(kafkaBrokers) { ProducerConfigFactory = () => new Dictionary <string, object> { { "socket.blocking.max.ms", 1 }, { "queue.buffering.max.ms", 1 }, { "socket.nagle.disable", true } }, ConsumerConfigFactory = (group) => new Dictionary <string, object> { { "socket.blocking.max.ms", 1 }, { "fetch.error.backoff.ms", 1 }, { "statistics.interval.ms", 500000 }, { "socket.nagle.disable", true }, { "auto.offset.reset", "earliest" } } }; MessageBusBuilder = new MessageBusBuilder() .WithSerializer(new JsonMessageSerializer()) .WithProviderKafka(KafkaSettings); MessageBus = new Lazy <KafkaMessageBus>(() => (KafkaMessageBus)MessageBusBuilder.Build()); }
public KafkaMessageBusTest() { MbSettings = new MessageBusSettings { Serializer = new Mock <IMessageSerializer>().Object, DependencyResolver = new Mock <IDependencyResolver>().Object }; KafkaMbSettings = new KafkaMessageBusSettings("host") { ProducerFactory = (producerSettings) => new Mock <Producer>(Enumerable.Empty <KeyValuePair <string, object> >()).Object }; KafkaMb = new Lazy <WrappedKafkaMessageBus>(() => new WrappedKafkaMessageBus(MbSettings, KafkaMbSettings)); }
public KafkaMessageBusIt() { _loggerFactory = NullLoggerFactory.Instance; _logger = _loggerFactory.CreateLogger <KafkaMessageBusIt>(); var configuration = new ConfigurationBuilder() .AddJsonFile("appsettings.json") .Build(); Secrets.Load(@"..\..\..\..\..\secrets.txt"); var kafkaBrokers = configuration["Kafka:Brokers"]; var kafkaUsername = Secrets.Service.PopulateSecrets(configuration["Kafka:Username"]); var kafkaPassword = Secrets.Service.PopulateSecrets(configuration["Kafka:Password"]); // Topics on cloudkarafka.com are prefixed with username TopicPrefix = $"{kafkaUsername}-"; KafkaSettings = new KafkaMessageBusSettings(kafkaBrokers) { ProducerConfig = (config) => { AddSsl(kafkaUsername, kafkaPassword, config); config.LingerMs = 5; // 5ms config.SocketNagleDisable = true; }, ConsumerConfig = (config) => { AddSsl(kafkaUsername, kafkaPassword, config); config.FetchErrorBackoffMs = 1; config.SocketNagleDisable = true; config.StatisticsIntervalMs = 500000; config.AutoOffsetReset = AutoOffsetReset.Earliest; } }; MessageBusBuilder = MessageBusBuilder.Create() .WithLoggerFacory(_loggerFactory) .WithSerializer(new JsonMessageSerializer()) .WithProviderKafka(KafkaSettings); MessageBus = new Lazy <KafkaMessageBus>(() => (KafkaMessageBus)MessageBusBuilder.Build()); }
public KafkaMessageBusIt() { _loggerFactory = NullLoggerFactory.Instance; _logger = _loggerFactory.CreateLogger <KafkaMessageBusIt>(); var configuration = new ConfigurationBuilder() .AddJsonFile("appsettings.json") .Build(); Secrets.Load(@"..\..\..\..\..\secrets.txt"); var kafkaBrokers = configuration["Kafka:Brokers"]; var kafkaUsername = Secrets.Service.PopulateSecrets(configuration["Kafka:Username"]); var kafkaPassword = Secrets.Service.PopulateSecrets(configuration["Kafka:Password"]); // Topics on cloudkarafka.com are prefixed with username TopicPrefix = $"{kafkaUsername}-"; KafkaSettings = new KafkaMessageBusSettings(kafkaBrokers) { ProducerConfigFactory = () => AddSsl(kafkaUsername, kafkaPassword, new Dictionary <string, object> { { "socket.blocking.max.ms", 1 }, { "queue.buffering.max.ms", 1 }, { "socket.nagle.disable", true }, //{"request.required.acks", 0} }), ConsumerConfigFactory = (group) => AddSsl(kafkaUsername, kafkaPassword, new Dictionary <string, object> { { "socket.blocking.max.ms", 1 }, { "fetch.error.backoff.ms", 1 }, { "statistics.interval.ms", 500000 }, { "socket.nagle.disable", true }, { KafkaConfigKeys.ConsumerKeys.AutoOffsetReset, KafkaConfigValues.AutoOffsetReset.Earliest } }) }; MessageBusBuilder = MessageBusBuilder.Create() .WithLoggerFacory(_loggerFactory) .WithSerializer(new JsonMessageSerializer()) .WithProviderKafka(KafkaSettings); MessageBus = new Lazy <KafkaMessageBus>(() => (KafkaMessageBus)MessageBusBuilder.Build()); }
public KafkaMessageBusTest() { var producerMock = new Mock <IProducer <byte[], byte[]> >(); producerMock.SetupGet(x => x.Name).Returns("Producer Name"); var producerBuilderMock = new Mock <ProducerBuilder <byte[], byte[]> >(new ProducerConfig()); producerBuilderMock.Setup(x => x.Build()).Returns(producerMock.Object); MbSettings = new MessageBusSettings { Serializer = new Mock <IMessageSerializer>().Object, DependencyResolver = new Mock <IDependencyResolver>().Object, LoggerFactory = NullLoggerFactory.Instance }; KafkaMbSettings = new KafkaMessageBusSettings("host") { ProducerBuilderFactory = (config) => producerBuilderMock.Object }; KafkaMb = new Lazy <WrappedKafkaMessageBus>(() => new WrappedKafkaMessageBus(MbSettings, KafkaMbSettings)); }
public KafkaMessageBusIt() { LogManager.Adapter = new DebugLoggerFactoryAdapter(); //LogManager.Adapter = new NoOpLoggerFactoryAdapter(); var configuration = new ConfigurationBuilder() .AddJsonFile("appsettings.json") .Build(); Secrets.Load(@"..\..\..\..\..\secrets.txt"); var kafkaBrokers = configuration["Kafka:Brokers"]; var kafkaUsername = Secrets.Service.PopulateSecrets(configuration["Kafka:Username"]); var kafkaPassword = Secrets.Service.PopulateSecrets(configuration["Kafka:Password"]); KafkaSettings = new KafkaMessageBusSettings(kafkaBrokers) { ProducerConfigFactory = () => new Dictionary <string, object> { { "socket.blocking.max.ms", 1 }, { "queue.buffering.max.ms", 1 }, { "socket.nagle.disable", true }, //{"request.required.acks", 0} }, ConsumerConfigFactory = (group) => new Dictionary <string, object> { { "socket.blocking.max.ms", 1 }, { "fetch.error.backoff.ms", 1 }, { "statistics.interval.ms", 500000 }, { "socket.nagle.disable", true }, { KafkaConfigKeys.ConsumerKeys.AutoOffsetReset, KafkaConfigValues.AutoOffsetReset.Earliest } } }; MessageBusBuilder = MessageBusBuilder.Create() .WithSerializer(new JsonMessageSerializer()) .WithProviderKafka(KafkaSettings); MessageBus = new Lazy <KafkaMessageBus>(() => (KafkaMessageBus)MessageBusBuilder.Build()); }
public KafkaMessageBusIt() { LogManager.Adapter = new DebugLoggerFactoryAdapter(); var configuration = new ConfigurationBuilder() .AddJsonFile("appsettings.json") .Build(); var kafkaBrokers = configuration["Kafka:Brokers"]; KafkaSettings = new KafkaMessageBusSettings(kafkaBrokers) { ProducerConfigFactory = () => new Dictionary <string, object> { { "socket.blocking.max.ms", 1 }, { "queue.buffering.max.ms", 1 }, { "socket.nagle.disable", true }, { "request.timeout.ms", 2000 }, // when no response within 2 sec of sending a msg, report error { "message.timeout.ms", 5000 } //{"delivery.timeout.ms", 10000 } // when no delivery ack within 10 sek, report error }, ConsumerConfigFactory = (group) => new Dictionary <string, object> { { "socket.blocking.max.ms", 1 }, { "fetch.error.backoff.ms", 1 }, { "statistics.interval.ms", 500000 }, { "socket.nagle.disable", true }, { KafkaConfigKeys.ConsumerKeys.AutoOffsetReset, KafkaConfigValues.AutoOffsetReset.Earliest } } }; MessageBusBuilder = MessageBusBuilder.Create() .WithSerializer(new JsonMessageSerializer()) .WithProviderKafka(KafkaSettings); MessageBus = new Lazy <KafkaMessageBus>(() => (KafkaMessageBus)MessageBusBuilder.Build()); }
public WrappedKafkaMessageBus(MessageBusSettings settings, KafkaMessageBusSettings kafkaSettings) : base(settings, kafkaSettings) { }
static void Main(string[] args) { SlimMessageBus.IMessageBus _bus; //var testTopic = $"test-ping-{DateTime.Now.Ticks}"; var topic = $"test-ping"; // some unique string across all application instances var instanceId = "1"; // address to your Kafka broker var kafkaBrokers = "172.16.4.241:9092"; var kafkaSettings = new KafkaMessageBusSettings(kafkaBrokers); kafkaSettings.ProducerConfigFactory = () => new Dictionary <string, object> { { "batch.num.messages", 1 }, { "message.max.bytes", 1000 }, { "api.version.request", true }, { "queue.buffering.max.ms", 1 }, { "socket.blocking.max.ms", 1 }, { "fetch.wait.max.ms", 10 }, { "fetch.error.backoff.ms", 10 }, { "fetch.min.bytes", 10 }, { "socket.nagle.disable", true } }; kafkaSettings.ConsumerConfigFactory = (group) => new Dictionary <string, object> { { "fetch.wait.max.ms", 10 }, { "fetch.error.backoff.ms", 10 }, { "queued.min.messages", 1 }, { "api.version.request", true }, { "queue.buffering.max.ms", 1 }, { "socket.blocking.max.ms", 1 }, { "fetch.min.bytes", 10 }, { "statistics.interval.ms", 500000 }, { "socket.nagle.disable", true } }; // conf->set("fetch.min.bytes", "1", errstr); // conf->set("queued.min.messages", "1", errstr); var messageBusBuilder = new MessageBusBuilder() .Publish <PingRequest>(x => { x.DefaultTopic(topic); }) .ExpectRequestResponses(x => { x.ReplyToTopic($"worker-{instanceId}-response"); x.Group($"worker-{instanceId}"); x.DefaultTimeout(TimeSpan.FromSeconds(30)); }) .WithSerializer(new JsonMessageSerializer()) .WithProviderKafka(kafkaSettings); _bus = (KafkaMessageBus)messageBusBuilder.Build(); var b = new Program(); bool cancel = false; Console.CancelKeyPress += delegate { cancel = true; }; Log.Info("Starting worker..."); // using (var container = ContainerSetup.Create()) { // var messagBus = container.Resolve<IMessageBus>(); Log.Info("Worker ready"); while (!cancel) { Log.Info("Sending message"); //var tasks = new List<Task<PingResponse>>(); //for (int i = 0; i < 1; i++) //{ var t = Send(_bus); // tasks.Add(t); // } // Task.WaitAll(tasks.OfType<Task>().ToArray()); // foreach (var t in tasks) { var r = t.Result; Console.WriteLine("{0} {1} {2}", r.Key, r.Timestamp, (DateTime.Now - r.Timestamp).ToString()); } // System.Threading.Thread.Sleep(100); } } Log.Info("Worker stopped"); }
private static IMessageBus BuildMessageBus() { // unique id across instances of this application (e.g. 1, 2, 3) var instanceId = "1"; var kafkaBrokers = "172.16.4.241:9092"; var instanceGroup = $"worker-{instanceId}"; var sharedGroup = $"workers"; var kafkaSettings = new KafkaMessageBusSettings(kafkaBrokers); kafkaSettings.ProducerConfigFactory = () => new Dictionary <string, object> { { "batch.num.messages", 1 }, { "message.max.bytes", 1000 }, { "api.version.request", true }, { "queue.buffering.max.ms", 1 }, { "socket.blocking.max.ms", 1 }, { "fetch.wait.max.ms", 10 }, { "fetch.error.backoff.ms", 10 }, { "fetch.min.bytes", 10 }, { "socket.nagle.disable", true } }; kafkaSettings.ConsumerConfigFactory = (group) => new Dictionary <string, object> { { "fetch.wait.max.ms", 10 }, { "fetch.error.backoff.ms", 10 }, { "queued.min.messages", 1 }, { "api.version.request", true }, { "queue.buffering.max.ms", 1 }, { "socket.blocking.max.ms", 1 }, { "fetch.min.bytes", 10 }, { "statistics.interval.ms", 500000 }, { "socket.nagle.disable", true } }; var messageBusBuilder = new MessageBusBuilder() .Handle <PingRequest, PingResponse>(s => { s.Topic("test-ping", t => { t.Group("ponggroup") .WithHandler <PingHandler>() .Instances(3); //t.Group(sharedGroup) // .WithConsumer<GenerateThumbnailRequestSubscriber>() // .Instances(3); }); }) .WithDependencyResolverAsServiceLocator() .WithSerializer(new JsonMessageSerializer()) .WithProviderKafka(kafkaSettings); var messageBus = messageBusBuilder.Build(); return(messageBus); }