public IConsumer <Ignore, byte[]> CreateConsumer( KafkaQueueConfiguration config, Action <IConsumer <Ignore, byte[]>, LogMessage> logHandler = null, Action <IConsumer <Ignore, byte[]>, Error> errorHandler = null) { config.ThrowIfNull(nameof(config)); var builder = new ConsumerBuilder <Ignore, byte[]>(new ConsumerConfig { GroupId = config.GroupId, BootstrapServers = $"{config.Server}", AutoOffsetReset = config.AutoOffsetReset, SaslKerberosKeytab = config.KeyTab, SaslKerberosPrincipal = config.User, SaslKerberosServiceName = config.ServiceName, SecurityProtocol = config.Protocol, SaslMechanism = config.Mechanism, Debug = config.Debug }); if (logHandler != null) { builder.SetLogHandler(logHandler); } if (errorHandler != null) { builder.SetErrorHandler(errorHandler); } return(builder.Build()); }
public IConsumer <byte[], byte[]> GetConsumer(ConsumerConfig config, IConsumerRebalanceListener rebalanceListener) { ConsumerBuilder <byte[], byte[]> builder = builderKafkaHandler.GetConsumerBuilder(config); if (rebalanceListener != null) { builder.SetPartitionsAssignedHandler((c, p) => rebalanceListener.PartitionsAssigned(c, p)); builder.SetPartitionsRevokedHandler((c, p) => rebalanceListener.PartitionsRevoked(c, p)); builder.SetLogHandler(loggerAdapter.LogConsume); builder.SetErrorHandler(loggerAdapter.ErrorConsume); if (exposeLibrdKafka) { // TODO : test librdkafka statistics with IntegrationTest (WIP see #82) var consumerStatisticsHandler = new ConsumerStatisticsHandler( config.ClientId, streamConfig.ApplicationId, (config as StreamizConsumerConfig)?.ThreadId); consumerStatisticsHandler.Register(MetricsRegistry); builder.SetStatisticsHandler((c, stat) => { var statistics = JsonConvert.DeserializeObject <Statistics>(stat); consumerStatisticsHandler.Publish(statistics); }); } } return(builder.Build()); }
public void ConsumeMessage(string topicName) { String ErrorReason = String.Empty; var consumeBldr = new ConsumerBuilder <Ignore, string>(_config.ConsumerConfig()); { //Added Error handler consumeBldr.SetErrorHandler(ConsumerErrorHandler); consumeBldr.SetLogHandler(LogHandler); var consumer = consumeBldr.Build(); try { consumer.Subscribe(topicName); ReadMessages(consumer); } catch (Exception ex) { // if this exception occured then ErrorReason = $"Error : { ex.Message } Type of { ex.GetType().Name } not handled and closing the consumer. { ex.InnerException?.Message } Source: { ex?.Source } .Stack :{ ex?.StackTrace}"; _worker.UnProcessedMessage("Empty Message Found", ErrorReason, SyslogLevel.Error); consumer.Close();//TBD consumer?.Dispose(); } } }
public IConsumer <byte[], byte[]> GetRestoreConsumer(ConsumerConfig config) { ConsumerBuilder <byte[], byte[]> builder = builderKafkaHandler.GetConsumerBuilder(config); builder.SetLogHandler(loggerAdapter.LogConsume); builder.SetErrorHandler(loggerAdapter.ErrorConsume); return(builder.Build()); }
public IConsumer <byte[], byte[]> GetGlobalConsumer(ConsumerConfig config) { ConsumerBuilder <byte[], byte[]> builder = new ConsumerBuilder <byte[], byte[]>(config); // TOOD : Finish builder.SetLogHandler(loggerAdapter.LogConsume); builder.SetErrorHandler(loggerAdapter.ErrorConsume); return(builder.Build()); }
public IConsumer <byte[], byte[]> GetGlobalConsumer(ConsumerConfig config) { config.AutoOffsetReset = AutoOffsetReset.Earliest; ConsumerBuilder <byte[], byte[]> builder = builderKafkaHandler.GetConsumerBuilder(config); builder.SetLogHandler(loggerAdapter.LogConsume); builder.SetErrorHandler(loggerAdapter.ErrorConsume); return(builder.Build()); }
public IConsumer <byte[], byte[]> GetConsumer(ConsumerConfig config, IConsumerRebalanceListener rebalanceListener) { ConsumerBuilder <byte[], byte[]> builder = new ConsumerBuilder <byte[], byte[]>(config); if (rebalanceListener != null) { builder.SetPartitionsAssignedHandler((c, p) => rebalanceListener.PartitionsAssigned(c, p)); builder.SetPartitionsRevokedHandler((c, p) => rebalanceListener.PartitionsRevoked(c, p)); builder.SetLogHandler(loggerAdapter.LogConsume); builder.SetErrorHandler(loggerAdapter.ErrorConsume); } return(builder.Build()); }
/// <summary> /// 同步运行状态 /// </summary> /// <returns></returns> Task <bool> IMessageReceiver.LoopBegin() { //KafkaOption.Instance.Consumer.Set(ConfigPropertyNames.Consumer.ConsumeResultFields, "all"); builder = new ConsumerBuilder <byte[], string>(KafkaOption.Instance.Consumer); //var value = KafkaOption.Instance.Consumer.Get(ConfigPropertyNames.Consumer.ConsumeResultFields); builder.SetErrorHandler(OnError); builder.SetLogHandler(OnLog); consumer = builder.Build(); consumer.Subscribe(Service.ServiceName); if (KafkaOption.Instance.Message.Concurrency > 0) { ConcurrencySemaphore = new SemaphoreSlim(KafkaOption.Instance.Message.Concurrency, KafkaOption.Instance.Message.Concurrency); } return(Task.FromResult(true)); }
/// <inheritdoc cref="IConfluentConsumerBuilder.Build" /> public IConsumer <byte[]?, byte[]?> Build() { if (_config == null) { throw new InvalidOperationException("SetConfig must be called to provide the consumer configuration."); } var builder = new ConsumerBuilder <byte[]?, byte[]?>(_config); if (_statisticsHandler != null) { builder.SetStatisticsHandler(_statisticsHandler); } if (_errorHandler != null) { builder.SetErrorHandler(_errorHandler); } if (_partitionsAssignedHandler != null) { builder.SetPartitionsAssignedHandler(_partitionsAssignedHandler); } if (_partitionsRevokedHandler != null) { builder.SetPartitionsRevokedHandler(_partitionsRevokedHandler); } if (_offsetsCommittedHandler != null) { builder.SetOffsetsCommittedHandler(_offsetsCommittedHandler); } if (_logHandler != null) { builder.SetLogHandler(_logHandler); } return(builder.Build()); }
public static void ReadDatas(string brokerList, string topic, int partition, int offsetMin, int offsetMax) { try { Console.WriteLine("Start read datas"); var config = new ConsumerConfig { BootstrapServers = brokerList, GroupId = "csharp-consumer", EnableAutoCommit = false, SessionTimeoutMs = 6000, // AutoOffsetReset = AutoOffsetReset.Earliest, EnablePartitionEof = true }; var consumerBuilder = new ConsumerBuilder <string, string>(config); consumerBuilder.SetErrorHandler(MyErrorHandler); consumerBuilder.SetLogHandler(MyLogHandler); var consumer = consumerBuilder.Build(); // Read datas from consumer.Assign(new TopicPartitionOffset(topic, new Partition(partition), new Offset(offsetMin))); ConsumeResult <string, string> result; var sw = new Stopwatch(); do { sw.Restart(); result = consumer.Consume(10_000); Console.WriteLine(sw.ElapsedMilliseconds); if (result != null) { Console.WriteLine($"{result.IsPartitionEOF} {result.TopicPartitionOffset}: '{result.Message?.Key}' | '{result.Message?.Value}'"); } }while (result != null && !result.IsPartitionEOF && result.Offset < offsetMax); } catch (Exception e) { Console.WriteLine(e); } }
private void CreateConsumer(ConsumerConfig config, string[] topics) { try { var builder = new ConsumerBuilder <Ignore, string>(config); builder.SetLogHandler((producer, message) => { // add logging }); consumer = builder.Build(); } catch (Exception ex) { BeginInvoke(new MethodInvoker(() => { status.Text = $"Could not create consumer: {ex.Message}"; })); return; } if (topics.Length == 1) { consumer.Subscribe(topics[0]); } else { consumer.Subscribe(topics); } BeginInvoke(new MethodInvoker(() => { status.Text = $"Consuming"; })); consumerCancellationTS = new CancellationTokenSource(); while (true) { try { var cr = consumer.Consume(consumerCancellationTS.Token); BeginInvoke(new MethodInvoker(() => { output.AppendText($"Consumed message at '{cr.TopicPartitionOffset}' on {DateTime.Now}:"); output.AppendText(Environment.NewLine); output.AppendText($"{cr.Value}"); output.AppendText(Environment.NewLine); output.AppendText(Environment.NewLine); })); } catch (ConsumeException ex) { BeginInvoke(new MethodInvoker(() => { status.Text = $"Error occured: {ex.Error.Reason}"; })); } catch (OperationCanceledException) { // Ensure the consumer leaves the group cleanly and final offsets are committed. consumer.Close(); break; } } consumer.Dispose(); consumer = null; }
public void Start(string instanceId, CancellationToken cancellationToken = default(CancellationToken)) { funcExecSemaphore = new Semaphore(MaxOutstanding, MaxOutstanding); CancellationTokenSource errorCts = new CancellationTokenSource(); CancellationTokenSource compositeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, errorCts.Token); CancellationToken compositeCancellationToken = compositeCts.Token; bool aMessageHasBeenProcessed = false; var cConfig = new ConsumerConfig { ClientId = $"{Name}-consumer-{instanceId}", GroupId = $"{Name}-group", BootstrapServers = BootstrapServers, EnableAutoCommit = true, EnableAutoOffsetStore = false, AutoOffsetReset = AutoOffsetReset.Latest }; if (DebugContext != null) { cConfig.Debug = DebugContext; } var cBuilder = new ConsumerBuilder <TInKey, TInValue>(cConfig); if (InKeyDeserializer != null) { cBuilder.SetKeyDeserializer(InKeyDeserializer); } if (InValueDeserializer != null) { cBuilder.SetValueDeserializer(InValueDeserializer); } if (Logger != null) { cBuilder.SetLogHandler((_, m) => { Logger(m); }); } cBuilder.SetErrorHandler((c, e) => { if (e.Code == ErrorCode.Local_AllBrokersDown || e.Code == ErrorCode.Local_Authentication) { if (!aMessageHasBeenProcessed) { // Logger.Log(e); errorCts.Cancel(); return; } } if (Logger != null) { Logger(new LogMessage(c.Name, SyslogLevel.Error, "unknown", e.Reason)); } }); var pConfig = new ProducerConfig { ClientId = $"{Name}-producer-{instanceId}", BootstrapServers = BootstrapServers, EnableIdempotence = true, LingerMs = 5, DeliveryReportFields = "none" }; if (DebugContext != null) { pConfig.Debug = DebugContext; } var pBuilder = new ProducerBuilder <TOutKey, TOutValue>(pConfig); if (OutKeySerializer != null) { pBuilder.SetKeySerializer(OutKeySerializer); } if (OutValueSerializer != null) { pBuilder.SetValueSerializer(OutValueSerializer); } if (Logger != null) { pBuilder.SetLogHandler((_, m) => { Logger(m); }); } pBuilder.SetErrorHandler((p, e) => { if (e.IsFatal) { errorCts.Cancel(); return; } if (e.Code == ErrorCode.Local_AllBrokersDown || e.Code == ErrorCode.Local_Authentication) { if (!aMessageHasBeenProcessed) { errorCts.Cancel(); return; } } if (Logger != null) { Logger(new LogMessage(p.Name, SyslogLevel.Error, "unknown", e.Reason)); } }); var partitionState = new Dictionary <TopicPartition, PartitionState>(); using (var producer = pBuilder.Build()) using (var consumer = cBuilder.Build()) { consumer.Subscribe(InputTopic); try { while (true) { ConsumeResult <TInKey, TInValue> cr; try { cr = consumer.Consume(compositeCancellationToken); } catch (ConsumeException ex) { if (ex.Error.Code == ErrorCode.Local_ValueDeserialization) { // For an in-depth discussion of what to do in the event of deserialization errors, refer to: // https://www.confluent.io/blog/kafka-connect-deep-dive-error-handling-dead-letter-queues if (ConsumeErrorTolerance == ErrorTolerance.All) { continue; } errorCts.Cancel(); // no error tolerance. } Thread.Sleep(TimeSpan.FromSeconds(10)); // ?? if not fail fast, do we want to sleep and why? continue; } if (!partitionState.ContainsKey(cr.TopicPartition)) { partitionState.Add(cr.TopicPartition, new PartitionState(this)); } partitionState[cr.TopicPartition].HandleConsumedMessage(cr, consumer, producer, funcExecSemaphore, errorCts); aMessageHasBeenProcessed = true; } } catch (OperationCanceledException) { } } if (errorCts.IsCancellationRequested) { throw new Exception("error occured, and we're failing fast."); } }
/// <summary> /// 创建消费者生成器 /// </summary> private void CreateConsumerBuilder() { if (disposed) { throw new ObjectDisposedException(nameof(KafkaConsumer)); } if (builder == null) { lock (this) { if (builder == null) { ConsumerConfig config = new ConsumerConfig(); config.BootstrapServers = BootstrapServers; config.GroupId = GroupId; config.AutoOffsetReset = AutoOffsetReset.Earliest; config.EnableAutoCommit = EnableAutoCommit; if (!string.IsNullOrEmpty(SaslUsername)) { config.SaslUsername = SaslUsername; config.SaslPassword = SaslPassword; config.SaslMechanism = SaslMechanism.Plain; config.SecurityProtocol = SecurityProtocol.SaslPlaintext; } //config.EnableAutoOffsetStore = true; //config.IsolationLevel = IsolationLevel.ReadCommitted; //config.MaxPollIntervalMs = 10000; //List<KeyValuePair<string, string>> config = new List<KeyValuePair<string, string>>(); //config.Add(new KeyValuePair<string, string>("bootstrap.servers", BootstrapServers)); //config.Add(new KeyValuePair<string, string>("group.id", GroupId)); //config.Add(new KeyValuePair<string, string>("auto.offset.reset", "earliest")); //config.Add(new KeyValuePair<string, string>("enable.auto.commit", EnableAutoCommit.ToString().ToLower())); //if (!string.IsNullOrEmpty(SaslUsername)) //{ // config.Add(new KeyValuePair<string, string>("security.protocol", "SASL_PLAINTEXT")); // config.Add(new KeyValuePair<string, string>("sasl.mechanism", "PLAIN")); // config.Add(new KeyValuePair<string, string>("sasl.username", SaslUsername)); // config.Add(new KeyValuePair<string, string>("sasl.password", SaslPassword)); //} //config.Add(new KeyValuePair<string, string>("max.poll.interval.ms", "10000")); //config.Add(new KeyValuePair<string, string>("session.timeout.ms", "10000")); //config.Add(new KeyValuePair<string, string>("isolation.level", "read_uncommitted")); builder = new ConsumerBuilder <string, object>(config); Action <Delegate, object> tryCatchWrap = (@delegate, arg) => { try { @delegate?.DynamicInvoke(arg); } catch { } }; builder.SetErrorHandler((p, e) => tryCatchWrap(ErrorHandler, new Exception(e.Reason))); builder.SetStatisticsHandler((p, e) => tryCatchWrap(StatisticsHandler, e)); builder.SetLogHandler((p, e) => tryCatchWrap(LogHandler, new KafkaLogMessage(e))); builder.SetValueDeserializer(new KafkaConverter()); } } } }