private static IConsumer <Ignore, byte[]> CreateConsumer(KafkaMessageFlowReceiverSettings settings, ITracer tracer)
{
var config = new ConsumerConfig((IDictionary <string, string>)settings.Config)
{
// help kafka server logs to identify node
ClientId = Environment.MachineName,
// manually store offsets
// https://github.com/edenhill/librdkafka/wiki/FAQ#why-committing-each-message-is-slow
// предполагаются что заданы дефолтные значения для параметров
// enable.auto.commit=true
EnableAutoOffsetStore = false,
// не ждём, пока наберётся необходимый batchsize
// если дошли до конца partition, то возвращаем столько сколько можем
EnablePartitionEof = true
};
// включить отладку
//config.Set("debug", "all");
var consumer = new ConsumerBuilder <Ignore, byte[]>(config)
.SetLogHandler((_, x) => OnLog(tracer, x))
.SetErrorHandler((_, x) => OnError(tracer, x))
.SetPartitionsAssignedHandler((_, x) => OnPartitionsAssigned(tracer, x, settings.TopicPartitionOffsets))
.SetPartitionsRevokedHandler((_, x) => OnPartitionsRevoked(tracer, x))
.SetOffsetsCommittedHandler((_, x) => OnOffsetsCommitted(tracer, x))
.Build();
consumer.Subscribe(settings.TopicPartitionOffsets.Select(x => x.Topic));
tracer.Info("KafkaAudit. Topic consumer created");
return(consumer);
}
private static KafkaMessageFlowReceiverSettings ParseConnectionString(string connectionString)
{
const string Topic = "topic";
const string PollTimeout = "pollTimeout";
var settings = new KafkaMessageFlowReceiverSettings
{
Config = JsonConvert.DeserializeObject <Dictionary <string, string> >(connectionString)
};
// Topic (required)
if (!settings.Config.TryGetValue(Topic, out var rawTopic))
{
throw new InvalidOperationException($"Required parameter \"{Topic}\" was not found. ConnectionString: {connectionString}");
}
settings.TopicPartitionOffset = ParseTopicPartitionOffset(rawTopic);
settings.Config.Remove(Topic);
// PollTimeout (optional)
if (settings.Config.TryGetValue(PollTimeout, out var rawPollTimeout))
{
settings.PollTimeout = TimeSpan.Parse(rawPollTimeout);
settings.Config.Remove(PollTimeout);
}
return(settings);
}
public KafkaMessageFlowReceiver(KafkaMessageFlowReceiverSettings settings, ITracer tracer)
{
_pollTimeout = settings.PollTimeout;
_tracer = tracer;
_consumer = CreateConsumer(settings, _tracer);
}
