/// <summary>Creates new instance <see cref="KafkaProducer"/>.</summary>
public KafkaProducer(ILogger logger, string brokerEndpoints)
{
_logger = logger;
var config = new ProducerConfig
{
BootstrapServers = brokerEndpoints,
ApiVersionRequest = true,
QueueBufferingMaxKbytes = 10240,
//{ "socket.blocking.max.ms", 1 }, // **DEPRECATED * *No longer used.
#if DEBUG
Debug = "msg",
#endif
MessageTimeoutMs = 3000,
};
config.Set("request.required.acks", "-1");
config.Set("queue.buffering.max.ms", "5");
_producer = new ProducerBuilder <Null, string>(config)
.SetKeySerializer(Serializers.Null)
.SetValueSerializer(Serializers.Utf8)
.SetLogHandler(OnLog)
.SetStatisticsHandler((_, json) => Console.WriteLine($"Statistics: {json}"))
.SetErrorHandler(OnError)
.Build();
}
public void ClientNameVersion(string bootstrapServers)
{
LogToFile("start ClientNameVersion");
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
producerConfig.Set("client.software.name", "test");
producerConfig.Set("client.software.version", "1.0");
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000
};
consumerConfig.Set("client.software.name", "test");
consumerConfig.Set("client.software.version", "1.0");
using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build())
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{ }
Assert.Equal(0, Library.HandleCount);
LogToFile("end ClientNameVersion");
}
private static ProducerConfig CreateProducerConfig()
{
var config = new ProducerConfig
{
BootstrapServers = "localhost:9092",
ApiVersionRequest = true,
QueueBufferingMaxKbytes = 10240,
MessageTimeoutMs = 3000
};
config.Set(@"request.required.acks", "-1");
config.Set(@"queue.buffering.max.ms", "5");
return(config);
}
public void Constuctor()
{
// Throw exception if a config value is null and ensure that exception mentions the
// respective config key.
var configWithNullValue = new ProducerConfig();
configWithNullValue.Set("sasl.password", null);
var e = Assert.Throws <ArgumentException>(() => { var c = new Producer(configWithNullValue); });
Assert.Contains("sasl.password", e.Message);
}
/// <summary>Creates new instance <see cref="KafkaTransport"/>. Performs a simplified
/// configuration of the parameters of the manufacturer and the consumer used in this transport.</summary>
/// <param name="rebusLoggerFactory"></param>
/// <param name="asyncTaskFactory"></param>
/// <param name="brokerList">Initial list of brokers as a CSV list of broker host or host:port.</param>
/// <param name="inputQueueName">name of input queue</param>
/// <param name="groupId">Id of group</param>
/// <param name="cancellationToken"></param>
public KafkaTransport(IRebusLoggerFactory rebusLoggerFactory, IAsyncTaskFactory asyncTaskFactory, string brokerList
, string inputQueueName, string groupId = null, CancellationToken cancellationToken = default(CancellationToken))
{
if (string.IsNullOrWhiteSpace(brokerList))
{
throw new NullReferenceException(nameof(brokerList));
}
_producerConfig = new ProducerConfig
{
BootstrapServers = brokerList,
ApiVersionRequest = true,
QueueBufferingMaxKbytes = 10240,
//{ "socket.blocking.max.ms", 1 }, // **DEPRECATED * *No longer used.
#if DEBUG
Debug = "msg",
#endif
MessageTimeoutMs = 3000,
};
_producerConfig.Set("request.required.acks", "-1");
_producerConfig.Set("queue.buffering.max.ms", "5");
if (!string.IsNullOrWhiteSpace(inputQueueName))
{
var maxNameLength = 249;
if (inputQueueName.Length > maxNameLength && _topicRegex.IsMatch(inputQueueName))
{
throw new ArgumentException("Недопустимые символы или длинна топика (файла)", nameof(inputQueueName));
}
Address = inputQueueName;
_queueSubscriptionStorage = new KafkaSubscriptionStorage(rebusLoggerFactory, asyncTaskFactory, brokerList
, inputQueueName, groupId, cancellationToken);
}
_log = rebusLoggerFactory.GetLogger <KafkaTransport>();
_asyncTaskFactory = asyncTaskFactory ?? throw new ArgumentNullException(nameof(asyncTaskFactory));
_cancellationToken = cancellationToken;
}
static void Main(string[] args)
{
var producerConfig = new ProducerConfig
{
//BootstrapServers = , //will be set from the general parameter
ApiVersionRequest = true,
QueueBufferingMaxKbytes = 10240,
#if DEBUG
Debug = "msg",
#endif
MessageTimeoutMs = 3000,
};
producerConfig.Set("request.required.acks", "-1");
producerConfig.Set("queue.buffering.max.ms", "5");
var consumerConfig = new ConsumerConfig
{
//BootstrapServers = , //will be set from the general parameter
ApiVersionRequest = true,
//GroupId = // will be set random
EnableAutoCommit = false,
FetchWaitMaxMs = 5,
FetchErrorBackoffMs = 5,
QueuedMinMessages = 1000,
SessionTimeoutMs = 6000,
//StatisticsIntervalMs = 5000,
#if DEBUG
TopicMetadataRefreshIntervalMs = 20000, // Otherwise it runs maybe five minutes
Debug = "msg",
#endif
AutoOffsetReset = AutoOffsetReset.Latest,
EnablePartitionEof = true
};
consumerConfig.Set("fetch.message.max.bytes", "10240");
IContainer container;
var builder = new ContainerBuilder();
builder.RegisterInstance(new Counter(ItemCount)).As <Counter>().SingleInstance();
builder.RegisterType <ConfirmationHandler>().As(typeof(IHandleMessages <>).MakeGenericType(typeof(Confirmation)));
builder.RegisterRebus((configurer, context) => configurer
.Logging(l => l.ColoredConsole(Rebus.Logging.LogLevel.Info))
.Transport(t => t.UseKafka(_kafkaEndpoint
, "scaleout.producer", producerConfig, consumerConfig))
.Routing(r => r.TypeBased().Map <TestMessage>("scaleout.consumers"))
);
using (container = builder.Build())
using (IBus bus = container.Resolve <IBus>())
{
bus.Subscribe <Confirmation>().Wait();
char key;
do
{
var sw = Stopwatch.StartNew();
var sendAmount = 0;
var messages = Enumerable.Range(1, ItemCount)
.Select(i =>
{
sendAmount = sendAmount + i;
return(bus.Publish(new TestMessage {
MessageNumber = i
}));
}).ToArray();
Task.WaitAll(messages);
Console.WriteLine($"Send: {sendAmount} for {sw.ElapsedMilliseconds / 1000f:N3}c");
Console.WriteLine("Press any key to exit or 'r' to repeat.");
key = Console.ReadKey().KeyChar;
} while (key == 'r' || key == 'к');
bus.Unsubscribe <Confirmation>().Wait(); // only for test
}
}
/// <summary>Initializes the transport by ensuring that the input queue has been created</summary>
public void Initialize()
{
// ToDo: Allow configuring transport options via Rebus
var producerConfig = new ProducerConfig
{
BootstrapServers = _brokerList,
ApiVersionRequest = true,
QueueBufferingMaxKbytes = 10240,
//{ "socket.blocking.max.ms", 1 }, // **DEPRECATED * *No longer used.
#if DEBUG
Debug = "msg",
#endif
MessageTimeoutMs = 3000,
};
producerConfig.Set("request.required.acks", "-1");
producerConfig.Set("queue.buffering.max.ms", "5");
var builder = new ProducerBuilder <Ignore, TransportMessage>(producerConfig)
.SetKeySerializer(new IgnoreSerializer())
.SetValueSerializer(new TransportMessageSerializer())
.SetLogHandler(ProducerOnLog)
.SetStatisticsHandler(ProducerOnStatistics)
.SetErrorHandler(ProducerOnError);
try
{
_producer = builder.Build();
}
catch (DllNotFoundException)
{ // Try loading librdkafka.dll
if (!Library.IsLoaded)
{
string directory = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().GetName().CodeBase.Substring(8));
var pathToLibrd = System.IO.Path.Combine(directory, $"librdkafka/{(Environment.Is64BitOperatingSystem ? "x64" : "x86")}/librdkafka.dll");
_log.Info($"librdkafka is not loaded. Trying to load {pathToLibrd}");
Confluent.Kafka.Library.Load(pathToLibrd);
_log.Info($"Using librdkafka version: {Library.Version}");
}
_producer = builder.Build();
}
// ToDo: Allow configuring transport options
var config = new ConsumerConfig
{
BootstrapServers = _brokerList,
ApiVersionRequest = true,
GroupId = !string.IsNullOrEmpty(_groupId) ? _groupId : Guid.NewGuid().ToString("N"),
EnableAutoCommit = false,
FetchWaitMaxMs = 5,
FetchErrorBackoffMs = 5,
QueuedMinMessages = 1000,
SessionTimeoutMs = 6000,
//StatisticsIntervalMs = 5000,
#if DEBUG
Debug = "msg",
#endif
AutoOffsetReset = AutoOffsetReset.Latest,
EnablePartitionEof = true
};
config.Set("fetch.message.max.bytes", "10240");
// Note: If a key or value deserializer is not set (as is the case below), the
// deserializer corresponding to the appropriate type from Confluent.Kafka.Serdes
// will be used automatically (where available). The default deserializer for string
// is UTF8. The default deserializer for Ignore returns null for all input data
// (including non-null data).
_consumer = new ConsumerBuilder <Ignore, TransportMessage>(config)
.SetKeyDeserializer(Deserializers.Ignore)
.SetValueDeserializer(new TransportMessageDeserializer())
.SetLogHandler(ConsumerOnLog)
.SetErrorHandler(ConsumerOnError)
.SetStatisticsHandler(ConsumerOnStatistics)
.SetRebalanceHandler(ConsumerOnRebalance)
.Build();
_consumer.Subscribe(_knownRoutes.Values);
}