private static IConsumer <Ignore, byte[]> CreateConsumer(KafkaMessageFlowReceiverSettings settings, ITracer tracer) { var config = new ConsumerConfig((IDictionary <string, string>)settings.Config) { // help kafka server logs to identify node ClientId = Environment.MachineName, // manually store offsets // https://github.com/edenhill/librdkafka/wiki/FAQ#why-committing-each-message-is-slow // предполагаются что заданы дефолтные значения для параметров // enable.auto.commit=true EnableAutoOffsetStore = false, // не ждём, пока наберётся необходимый batchsize // если дошли до конца partition, то возвращаем столько сколько можем EnablePartitionEof = true }; // включить отладку //config.Set("debug", "all"); var consumer = new ConsumerBuilder <Ignore, byte[]>(config) .SetLogHandler((_, x) => OnLog(tracer, x)) .SetErrorHandler((_, x) => OnError(tracer, x)) .SetPartitionsAssignedHandler((_, x) => OnPartitionsAssigned(tracer, x, settings.TopicPartitionOffsets)) .SetPartitionsRevokedHandler((_, x) => OnPartitionsRevoked(tracer, x)) .SetOffsetsCommittedHandler((_, x) => OnOffsetsCommitted(tracer, x)) .Build(); consumer.Subscribe(settings.TopicPartitionOffsets.Select(x => x.Topic)); tracer.Info("KafkaAudit. Topic consumer created"); return(consumer); }
private static KafkaMessageFlowReceiverSettings ParseConnectionString(string connectionString) { const string Topic = "topic"; const string PollTimeout = "pollTimeout"; var settings = new KafkaMessageFlowReceiverSettings { Config = JsonConvert.DeserializeObject <Dictionary <string, string> >(connectionString) }; // Topic (required) if (!settings.Config.TryGetValue(Topic, out var rawTopic)) { throw new InvalidOperationException($"Required parameter \"{Topic}\" was not found. ConnectionString: {connectionString}"); } settings.TopicPartitionOffset = ParseTopicPartitionOffset(rawTopic); settings.Config.Remove(Topic); // PollTimeout (optional) if (settings.Config.TryGetValue(PollTimeout, out var rawPollTimeout)) { settings.PollTimeout = TimeSpan.Parse(rawPollTimeout); settings.Config.Remove(PollTimeout); } return(settings); }
public KafkaMessageFlowReceiver(KafkaMessageFlowReceiverSettings settings, ITracer tracer) { _pollTimeout = settings.PollTimeout; _tracer = tracer; _consumer = CreateConsumer(settings, _tracer); }