public KafkaMessageBusIt()
{
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
var kafkaBrokers = configuration["Kafka:Brokers"];
KafkaSettings = new KafkaMessageBusSettings(kafkaBrokers)
{
ProducerConfigFactory = () => new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "queue.buffering.max.ms", 1 },
{ "socket.nagle.disable", true }
},
ConsumerConfigFactory = (group) => new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "fetch.error.backoff.ms", 1 },
{ "statistics.interval.ms", 500000 },
{ "socket.nagle.disable", true },
{ KafkaConfigKeys.Consumer.AutoOffsetReset, KafkaConfigValues.AutoOffsetReset.Earliest }
}
};
MessageBusBuilder = new MessageBusBuilder()
.WithSerializer(new JsonMessageSerializer())
.WithProviderKafka(KafkaSettings);
MessageBus = new Lazy <KafkaMessageBus>(() => (KafkaMessageBus)MessageBusBuilder.Build());
}
public KafkaMessageBusIt()
{
// address to the Kafka broker
var kafkaBrokers = "localhost:9092";
KafkaSettings = new KafkaMessageBusSettings(kafkaBrokers)
{
ProducerConfigFactory = () => new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "queue.buffering.max.ms", 1 },
{ "socket.nagle.disable", true }
},
ConsumerConfigFactory = (group) => new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "fetch.error.backoff.ms", 1 },
{ "statistics.interval.ms", 500000 },
{ "socket.nagle.disable", true },
{ "auto.offset.reset", "earliest" }
}
};
MessageBusBuilder = new MessageBusBuilder()
.WithSerializer(new JsonMessageSerializer())
.WithProviderKafka(KafkaSettings);
MessageBus = new Lazy <KafkaMessageBus>(() => (KafkaMessageBus)MessageBusBuilder.Build());
}
public KafkaMessageBusTest()
{
MbSettings = new MessageBusSettings
{
Serializer = new Mock <IMessageSerializer>().Object,
DependencyResolver = new Mock <IDependencyResolver>().Object
};
KafkaMbSettings = new KafkaMessageBusSettings("host")
{
ProducerFactory = (producerSettings) => new Mock <Producer>(Enumerable.Empty <KeyValuePair <string, object> >()).Object
};
KafkaMb = new Lazy <WrappedKafkaMessageBus>(() => new WrappedKafkaMessageBus(MbSettings, KafkaMbSettings));
}
public KafkaMessageBusIt()
{
_loggerFactory = NullLoggerFactory.Instance;
_logger = _loggerFactory.CreateLogger <KafkaMessageBusIt>();
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
Secrets.Load(@"..\..\..\..\..\secrets.txt");
var kafkaBrokers = configuration["Kafka:Brokers"];
var kafkaUsername = Secrets.Service.PopulateSecrets(configuration["Kafka:Username"]);
var kafkaPassword = Secrets.Service.PopulateSecrets(configuration["Kafka:Password"]);
// Topics on cloudkarafka.com are prefixed with username
TopicPrefix = $"{kafkaUsername}-";
KafkaSettings = new KafkaMessageBusSettings(kafkaBrokers)
{
ProducerConfig = (config) =>
{
AddSsl(kafkaUsername, kafkaPassword, config);
config.LingerMs = 5; // 5ms
config.SocketNagleDisable = true;
},
ConsumerConfig = (config) => {
AddSsl(kafkaUsername, kafkaPassword, config);
config.FetchErrorBackoffMs = 1;
config.SocketNagleDisable = true;
config.StatisticsIntervalMs = 500000;
config.AutoOffsetReset = AutoOffsetReset.Earliest;
}
};
MessageBusBuilder = MessageBusBuilder.Create()
.WithLoggerFacory(_loggerFactory)
.WithSerializer(new JsonMessageSerializer())
.WithProviderKafka(KafkaSettings);
MessageBus = new Lazy <KafkaMessageBus>(() => (KafkaMessageBus)MessageBusBuilder.Build());
}
public KafkaMessageBusIt()
{
_loggerFactory = NullLoggerFactory.Instance;
_logger = _loggerFactory.CreateLogger <KafkaMessageBusIt>();
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
Secrets.Load(@"..\..\..\..\..\secrets.txt");
var kafkaBrokers = configuration["Kafka:Brokers"];
var kafkaUsername = Secrets.Service.PopulateSecrets(configuration["Kafka:Username"]);
var kafkaPassword = Secrets.Service.PopulateSecrets(configuration["Kafka:Password"]);
// Topics on cloudkarafka.com are prefixed with username
TopicPrefix = $"{kafkaUsername}-";
KafkaSettings = new KafkaMessageBusSettings(kafkaBrokers)
{
ProducerConfigFactory = () => AddSsl(kafkaUsername, kafkaPassword, new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "queue.buffering.max.ms", 1 },
{ "socket.nagle.disable", true },
//{"request.required.acks", 0}
}),
ConsumerConfigFactory = (group) => AddSsl(kafkaUsername, kafkaPassword, new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "fetch.error.backoff.ms", 1 },
{ "statistics.interval.ms", 500000 },
{ "socket.nagle.disable", true },
{ KafkaConfigKeys.ConsumerKeys.AutoOffsetReset, KafkaConfigValues.AutoOffsetReset.Earliest }
})
};
MessageBusBuilder = MessageBusBuilder.Create()
.WithLoggerFacory(_loggerFactory)
.WithSerializer(new JsonMessageSerializer())
.WithProviderKafka(KafkaSettings);
MessageBus = new Lazy <KafkaMessageBus>(() => (KafkaMessageBus)MessageBusBuilder.Build());
}
public KafkaMessageBusTest()
{
var producerMock = new Mock <IProducer <byte[], byte[]> >();
producerMock.SetupGet(x => x.Name).Returns("Producer Name");
var producerBuilderMock = new Mock <ProducerBuilder <byte[], byte[]> >(new ProducerConfig());
producerBuilderMock.Setup(x => x.Build()).Returns(producerMock.Object);
MbSettings = new MessageBusSettings
{
Serializer = new Mock <IMessageSerializer>().Object,
DependencyResolver = new Mock <IDependencyResolver>().Object,
LoggerFactory = NullLoggerFactory.Instance
};
KafkaMbSettings = new KafkaMessageBusSettings("host")
{
ProducerBuilderFactory = (config) => producerBuilderMock.Object
};
KafkaMb = new Lazy <WrappedKafkaMessageBus>(() => new WrappedKafkaMessageBus(MbSettings, KafkaMbSettings));
}
public KafkaMessageBusIt()
{
LogManager.Adapter = new DebugLoggerFactoryAdapter();
//LogManager.Adapter = new NoOpLoggerFactoryAdapter();
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
Secrets.Load(@"..\..\..\..\..\secrets.txt");
var kafkaBrokers = configuration["Kafka:Brokers"];
var kafkaUsername = Secrets.Service.PopulateSecrets(configuration["Kafka:Username"]);
var kafkaPassword = Secrets.Service.PopulateSecrets(configuration["Kafka:Password"]);
KafkaSettings = new KafkaMessageBusSettings(kafkaBrokers)
{
ProducerConfigFactory = () => new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "queue.buffering.max.ms", 1 },
{ "socket.nagle.disable", true },
//{"request.required.acks", 0}
},
ConsumerConfigFactory = (group) => new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "fetch.error.backoff.ms", 1 },
{ "statistics.interval.ms", 500000 },
{ "socket.nagle.disable", true },
{ KafkaConfigKeys.ConsumerKeys.AutoOffsetReset, KafkaConfigValues.AutoOffsetReset.Earliest }
}
};
MessageBusBuilder = MessageBusBuilder.Create()
.WithSerializer(new JsonMessageSerializer())
.WithProviderKafka(KafkaSettings);
MessageBus = new Lazy <KafkaMessageBus>(() => (KafkaMessageBus)MessageBusBuilder.Build());
}
public KafkaMessageBusIt()
{
LogManager.Adapter = new DebugLoggerFactoryAdapter();
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
var kafkaBrokers = configuration["Kafka:Brokers"];
KafkaSettings = new KafkaMessageBusSettings(kafkaBrokers)
{
ProducerConfigFactory = () => new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "queue.buffering.max.ms", 1 },
{ "socket.nagle.disable", true },
{ "request.timeout.ms", 2000 }, // when no response within 2 sec of sending a msg, report error
{ "message.timeout.ms", 5000 }
//{"delivery.timeout.ms", 10000 } // when no delivery ack within 10 sek, report error
},
ConsumerConfigFactory = (group) => new Dictionary <string, object>
{
{ "socket.blocking.max.ms", 1 },
{ "fetch.error.backoff.ms", 1 },
{ "statistics.interval.ms", 500000 },
{ "socket.nagle.disable", true },
{ KafkaConfigKeys.ConsumerKeys.AutoOffsetReset, KafkaConfigValues.AutoOffsetReset.Earliest }
}
};
MessageBusBuilder = MessageBusBuilder.Create()
.WithSerializer(new JsonMessageSerializer())
.WithProviderKafka(KafkaSettings);
MessageBus = new Lazy <KafkaMessageBus>(() => (KafkaMessageBus)MessageBusBuilder.Build());
}
public WrappedKafkaMessageBus(MessageBusSettings settings, KafkaMessageBusSettings kafkaSettings)
: base(settings, kafkaSettings)
{
}
static void Main(string[] args)
{
SlimMessageBus.IMessageBus _bus;
//var testTopic = $"test-ping-{DateTime.Now.Ticks}";
var topic = $"test-ping";
// some unique string across all application instances
var instanceId = "1";
// address to your Kafka broker
var kafkaBrokers = "172.16.4.241:9092";
var kafkaSettings = new KafkaMessageBusSettings(kafkaBrokers);
kafkaSettings.ProducerConfigFactory = () => new Dictionary <string, object>
{
{ "batch.num.messages", 1 },
{ "message.max.bytes", 1000 },
{ "api.version.request", true },
{ "queue.buffering.max.ms", 1 },
{ "socket.blocking.max.ms", 1 },
{ "fetch.wait.max.ms", 10 },
{ "fetch.error.backoff.ms", 10 },
{ "fetch.min.bytes", 10 },
{ "socket.nagle.disable", true }
};
kafkaSettings.ConsumerConfigFactory = (group) => new Dictionary <string, object>
{
{ "fetch.wait.max.ms", 10 },
{ "fetch.error.backoff.ms", 10 },
{ "queued.min.messages", 1 },
{ "api.version.request", true },
{ "queue.buffering.max.ms", 1 },
{ "socket.blocking.max.ms", 1 },
{ "fetch.min.bytes", 10 },
{ "statistics.interval.ms", 500000 },
{ "socket.nagle.disable", true }
};
// conf->set("fetch.min.bytes", "1", errstr);
// conf->set("queued.min.messages", "1", errstr);
var messageBusBuilder = new MessageBusBuilder()
.Publish <PingRequest>(x =>
{
x.DefaultTopic(topic);
})
.ExpectRequestResponses(x =>
{
x.ReplyToTopic($"worker-{instanceId}-response");
x.Group($"worker-{instanceId}");
x.DefaultTimeout(TimeSpan.FromSeconds(30));
})
.WithSerializer(new JsonMessageSerializer())
.WithProviderKafka(kafkaSettings);
_bus = (KafkaMessageBus)messageBusBuilder.Build();
var b = new Program();
bool cancel = false;
Console.CancelKeyPress += delegate {
cancel = true;
};
Log.Info("Starting worker...");
// using (var container = ContainerSetup.Create())
{
// var messagBus = container.Resolve<IMessageBus>();
Log.Info("Worker ready");
while (!cancel)
{
Log.Info("Sending message");
//var tasks = new List<Task<PingResponse>>();
//for (int i = 0; i < 1; i++)
//{
var t = Send(_bus);
// tasks.Add(t);
// }
// Task.WaitAll(tasks.OfType<Task>().ToArray());
// foreach (var t in tasks)
{
var r = t.Result;
Console.WriteLine("{0} {1} {2}", r.Key, r.Timestamp, (DateTime.Now - r.Timestamp).ToString());
}
// System.Threading.Thread.Sleep(100);
}
}
Log.Info("Worker stopped");
}
private static IMessageBus BuildMessageBus()
{
// unique id across instances of this application (e.g. 1, 2, 3)
var instanceId = "1";
var kafkaBrokers = "172.16.4.241:9092";
var instanceGroup = $"worker-{instanceId}";
var sharedGroup = $"workers";
var kafkaSettings = new KafkaMessageBusSettings(kafkaBrokers);
kafkaSettings.ProducerConfigFactory = () => new Dictionary <string, object>
{
{ "batch.num.messages", 1 },
{ "message.max.bytes", 1000 },
{ "api.version.request", true },
{ "queue.buffering.max.ms", 1 },
{ "socket.blocking.max.ms", 1 },
{ "fetch.wait.max.ms", 10 },
{ "fetch.error.backoff.ms", 10 },
{ "fetch.min.bytes", 10 },
{ "socket.nagle.disable", true }
};
kafkaSettings.ConsumerConfigFactory = (group) => new Dictionary <string, object>
{
{ "fetch.wait.max.ms", 10 },
{ "fetch.error.backoff.ms", 10 },
{ "queued.min.messages", 1 },
{ "api.version.request", true },
{ "queue.buffering.max.ms", 1 },
{ "socket.blocking.max.ms", 1 },
{ "fetch.min.bytes", 10 },
{ "statistics.interval.ms", 500000 },
{ "socket.nagle.disable", true }
};
var messageBusBuilder = new MessageBusBuilder()
.Handle <PingRequest, PingResponse>(s =>
{
s.Topic("test-ping", t =>
{
t.Group("ponggroup")
.WithHandler <PingHandler>()
.Instances(3);
//t.Group(sharedGroup)
// .WithConsumer<GenerateThumbnailRequestSubscriber>()
// .Instances(3);
});
})
.WithDependencyResolverAsServiceLocator()
.WithSerializer(new JsonMessageSerializer())
.WithProviderKafka(kafkaSettings);
var messageBus = messageBusBuilder.Build();
return(messageBus);
}
