public IConsumer <Ignore, byte[]> CreateConsumer(
KafkaQueueConfiguration config,
Action <IConsumer <Ignore, byte[]>, LogMessage> logHandler = null,
Action <IConsumer <Ignore, byte[]>, Error> errorHandler = null)
{
config.ThrowIfNull(nameof(config));
var builder = new ConsumerBuilder <Ignore, byte[]>(new ConsumerConfig
{
GroupId = config.GroupId,
BootstrapServers = $"{config.Server}",
AutoOffsetReset = config.AutoOffsetReset,
SaslKerberosKeytab = config.KeyTab,
SaslKerberosPrincipal = config.User,
SaslKerberosServiceName = config.ServiceName,
SecurityProtocol = config.Protocol,
SaslMechanism = config.Mechanism,
Debug = config.Debug
});
if (logHandler != null)
{
builder.SetLogHandler(logHandler);
}
if (errorHandler != null)
{
builder.SetErrorHandler(errorHandler);
}
return(builder.Build());
}
public IConsumer <byte[], byte[]> GetConsumer(ConsumerConfig config, IConsumerRebalanceListener rebalanceListener)
{
ConsumerBuilder <byte[], byte[]> builder = builderKafkaHandler.GetConsumerBuilder(config);
if (rebalanceListener != null)
{
builder.SetPartitionsAssignedHandler((c, p) => rebalanceListener.PartitionsAssigned(c, p));
builder.SetPartitionsRevokedHandler((c, p) => rebalanceListener.PartitionsRevoked(c, p));
builder.SetLogHandler(loggerAdapter.LogConsume);
builder.SetErrorHandler(loggerAdapter.ErrorConsume);
if (exposeLibrdKafka)
{
// TODO : test librdkafka statistics with IntegrationTest (WIP see #82)
var consumerStatisticsHandler = new ConsumerStatisticsHandler(
config.ClientId,
streamConfig.ApplicationId,
(config as StreamizConsumerConfig)?.ThreadId);
consumerStatisticsHandler.Register(MetricsRegistry);
builder.SetStatisticsHandler((c, stat) =>
{
var statistics = JsonConvert.DeserializeObject <Statistics>(stat);
consumerStatisticsHandler.Publish(statistics);
});
}
}
return(builder.Build());
}
public void ConsumeMessage(string topicName)
{
String ErrorReason = String.Empty;
var consumeBldr = new ConsumerBuilder <Ignore, string>(_config.ConsumerConfig());
{
//Added Error handler
consumeBldr.SetErrorHandler(ConsumerErrorHandler);
consumeBldr.SetLogHandler(LogHandler);
var consumer = consumeBldr.Build();
try
{
consumer.Subscribe(topicName);
ReadMessages(consumer);
}
catch (Exception ex)
{
// if this exception occured then
ErrorReason = $"Error : { ex.Message } Type of { ex.GetType().Name } not handled and closing the consumer. { ex.InnerException?.Message } Source: { ex?.Source } .Stack :{ ex?.StackTrace}";
_worker.UnProcessedMessage("Empty Message Found", ErrorReason, SyslogLevel.Error);
consumer.Close();//TBD
consumer?.Dispose();
}
}
}
public IConsumer <byte[], byte[]> GetRestoreConsumer(ConsumerConfig config)
{
ConsumerBuilder <byte[], byte[]> builder = builderKafkaHandler.GetConsumerBuilder(config);
builder.SetLogHandler(loggerAdapter.LogConsume);
builder.SetErrorHandler(loggerAdapter.ErrorConsume);
return(builder.Build());
}
public IConsumer <byte[], byte[]> GetGlobalConsumer(ConsumerConfig config)
{
ConsumerBuilder <byte[], byte[]> builder = new ConsumerBuilder <byte[], byte[]>(config);
// TOOD : Finish
builder.SetLogHandler(loggerAdapter.LogConsume);
builder.SetErrorHandler(loggerAdapter.ErrorConsume);
return(builder.Build());
}
public IConsumer <byte[], byte[]> GetGlobalConsumer(ConsumerConfig config)
{
config.AutoOffsetReset = AutoOffsetReset.Earliest;
ConsumerBuilder <byte[], byte[]> builder = builderKafkaHandler.GetConsumerBuilder(config);
builder.SetLogHandler(loggerAdapter.LogConsume);
builder.SetErrorHandler(loggerAdapter.ErrorConsume);
return(builder.Build());
}
public IConsumer <byte[], byte[]> GetConsumer(ConsumerConfig config, IConsumerRebalanceListener rebalanceListener)
{
ConsumerBuilder <byte[], byte[]> builder = new ConsumerBuilder <byte[], byte[]>(config);
if (rebalanceListener != null)
{
builder.SetPartitionsAssignedHandler((c, p) => rebalanceListener.PartitionsAssigned(c, p));
builder.SetPartitionsRevokedHandler((c, p) => rebalanceListener.PartitionsRevoked(c, p));
builder.SetLogHandler(loggerAdapter.LogConsume);
builder.SetErrorHandler(loggerAdapter.ErrorConsume);
}
return(builder.Build());
}
/// <summary>
/// 同步运行状态
/// </summary>
/// <returns></returns>
Task <bool> IMessageReceiver.LoopBegin()
{
//KafkaOption.Instance.Consumer.Set(ConfigPropertyNames.Consumer.ConsumeResultFields, "all");
builder = new ConsumerBuilder <byte[], string>(KafkaOption.Instance.Consumer);
//var value = KafkaOption.Instance.Consumer.Get(ConfigPropertyNames.Consumer.ConsumeResultFields);
builder.SetErrorHandler(OnError);
builder.SetLogHandler(OnLog);
consumer = builder.Build();
consumer.Subscribe(Service.ServiceName);
if (KafkaOption.Instance.Message.Concurrency > 0)
{
ConcurrencySemaphore = new SemaphoreSlim(KafkaOption.Instance.Message.Concurrency, KafkaOption.Instance.Message.Concurrency);
}
return(Task.FromResult(true));
}
/// <inheritdoc cref="IConfluentConsumerBuilder.Build" />
public IConsumer <byte[]?, byte[]?> Build()
{
if (_config == null)
{
throw new InvalidOperationException("SetConfig must be called to provide the consumer configuration.");
}
var builder = new ConsumerBuilder <byte[]?, byte[]?>(_config);
if (_statisticsHandler != null)
{
builder.SetStatisticsHandler(_statisticsHandler);
}
if (_errorHandler != null)
{
builder.SetErrorHandler(_errorHandler);
}
if (_partitionsAssignedHandler != null)
{
builder.SetPartitionsAssignedHandler(_partitionsAssignedHandler);
}
if (_partitionsRevokedHandler != null)
{
builder.SetPartitionsRevokedHandler(_partitionsRevokedHandler);
}
if (_offsetsCommittedHandler != null)
{
builder.SetOffsetsCommittedHandler(_offsetsCommittedHandler);
}
if (_logHandler != null)
{
builder.SetLogHandler(_logHandler);
}
return(builder.Build());
}
public static void ReadDatas(string brokerList, string topic, int partition, int offsetMin, int offsetMax)
{
try
{
Console.WriteLine("Start read datas");
var config = new ConsumerConfig
{
BootstrapServers = brokerList,
GroupId = "csharp-consumer",
EnableAutoCommit = false,
SessionTimeoutMs = 6000,
// AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = true
};
var consumerBuilder = new ConsumerBuilder <string, string>(config);
consumerBuilder.SetErrorHandler(MyErrorHandler);
consumerBuilder.SetLogHandler(MyLogHandler);
var consumer = consumerBuilder.Build();
// Read datas from
consumer.Assign(new TopicPartitionOffset(topic, new Partition(partition), new Offset(offsetMin)));
ConsumeResult <string, string> result;
var sw = new Stopwatch();
do
{
sw.Restart();
result = consumer.Consume(10_000);
Console.WriteLine(sw.ElapsedMilliseconds);
if (result != null)
{
Console.WriteLine($"{result.IsPartitionEOF} {result.TopicPartitionOffset}: '{result.Message?.Key}' | '{result.Message?.Value}'");
}
}while (result != null && !result.IsPartitionEOF && result.Offset < offsetMax);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
private void CreateConsumer(ConsumerConfig config, string[] topics)
{
try
{
var builder = new ConsumerBuilder <Ignore, string>(config);
builder.SetLogHandler((producer, message) =>
{
// add logging
});
consumer = builder.Build();
}
catch (Exception ex)
{
BeginInvoke(new MethodInvoker(() =>
{
status.Text = $"Could not create consumer: {ex.Message}";
}));
return;
}
if (topics.Length == 1)
{
consumer.Subscribe(topics[0]);
}
else
{
consumer.Subscribe(topics);
}
BeginInvoke(new MethodInvoker(() =>
{
status.Text = $"Consuming";
}));
consumerCancellationTS = new CancellationTokenSource();
while (true)
{
try
{
var cr = consumer.Consume(consumerCancellationTS.Token);
BeginInvoke(new MethodInvoker(() =>
{
output.AppendText($"Consumed message at '{cr.TopicPartitionOffset}' on {DateTime.Now}:");
output.AppendText(Environment.NewLine);
output.AppendText($"{cr.Value}");
output.AppendText(Environment.NewLine);
output.AppendText(Environment.NewLine);
}));
}
catch (ConsumeException ex)
{
BeginInvoke(new MethodInvoker(() =>
{
status.Text = $"Error occured: {ex.Error.Reason}";
}));
}
catch (OperationCanceledException)
{
// Ensure the consumer leaves the group cleanly and final offsets are committed.
consumer.Close();
break;
}
}
consumer.Dispose();
consumer = null;
}
public void Start(string instanceId, CancellationToken cancellationToken = default(CancellationToken))
{
funcExecSemaphore = new Semaphore(MaxOutstanding, MaxOutstanding);
CancellationTokenSource errorCts = new CancellationTokenSource();
CancellationTokenSource compositeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, errorCts.Token);
CancellationToken compositeCancellationToken = compositeCts.Token;
bool aMessageHasBeenProcessed = false;
var cConfig = new ConsumerConfig
{
ClientId = $"{Name}-consumer-{instanceId}",
GroupId = $"{Name}-group",
BootstrapServers = BootstrapServers,
EnableAutoCommit = true,
EnableAutoOffsetStore = false,
AutoOffsetReset = AutoOffsetReset.Latest
};
if (DebugContext != null)
{
cConfig.Debug = DebugContext;
}
var cBuilder = new ConsumerBuilder <TInKey, TInValue>(cConfig);
if (InKeyDeserializer != null)
{
cBuilder.SetKeyDeserializer(InKeyDeserializer);
}
if (InValueDeserializer != null)
{
cBuilder.SetValueDeserializer(InValueDeserializer);
}
if (Logger != null)
{
cBuilder.SetLogHandler((_, m) =>
{
Logger(m);
});
}
cBuilder.SetErrorHandler((c, e) =>
{
if (e.Code == ErrorCode.Local_AllBrokersDown ||
e.Code == ErrorCode.Local_Authentication)
{
if (!aMessageHasBeenProcessed)
{
// Logger.Log(e);
errorCts.Cancel();
return;
}
}
if (Logger != null)
{
Logger(new LogMessage(c.Name, SyslogLevel.Error, "unknown", e.Reason));
}
});
var pConfig = new ProducerConfig
{
ClientId = $"{Name}-producer-{instanceId}",
BootstrapServers = BootstrapServers,
EnableIdempotence = true,
LingerMs = 5,
DeliveryReportFields = "none"
};
if (DebugContext != null)
{
pConfig.Debug = DebugContext;
}
var pBuilder = new ProducerBuilder <TOutKey, TOutValue>(pConfig);
if (OutKeySerializer != null)
{
pBuilder.SetKeySerializer(OutKeySerializer);
}
if (OutValueSerializer != null)
{
pBuilder.SetValueSerializer(OutValueSerializer);
}
if (Logger != null)
{
pBuilder.SetLogHandler((_, m) =>
{
Logger(m);
});
}
pBuilder.SetErrorHandler((p, e) =>
{
if (e.IsFatal)
{
errorCts.Cancel();
return;
}
if (e.Code == ErrorCode.Local_AllBrokersDown ||
e.Code == ErrorCode.Local_Authentication)
{
if (!aMessageHasBeenProcessed)
{
errorCts.Cancel();
return;
}
}
if (Logger != null)
{
Logger(new LogMessage(p.Name, SyslogLevel.Error, "unknown", e.Reason));
}
});
var partitionState = new Dictionary <TopicPartition, PartitionState>();
using (var producer = pBuilder.Build())
using (var consumer = cBuilder.Build())
{
consumer.Subscribe(InputTopic);
try
{
while (true)
{
ConsumeResult <TInKey, TInValue> cr;
try
{
cr = consumer.Consume(compositeCancellationToken);
}
catch (ConsumeException ex)
{
if (ex.Error.Code == ErrorCode.Local_ValueDeserialization)
{
// For an in-depth discussion of what to do in the event of deserialization errors, refer to:
// https://www.confluent.io/blog/kafka-connect-deep-dive-error-handling-dead-letter-queues
if (ConsumeErrorTolerance == ErrorTolerance.All)
{
continue;
}
errorCts.Cancel(); // no error tolerance.
}
Thread.Sleep(TimeSpan.FromSeconds(10)); // ?? if not fail fast, do we want to sleep and why?
continue;
}
if (!partitionState.ContainsKey(cr.TopicPartition))
{
partitionState.Add(cr.TopicPartition, new PartitionState(this));
}
partitionState[cr.TopicPartition].HandleConsumedMessage(cr, consumer, producer, funcExecSemaphore, errorCts);
aMessageHasBeenProcessed = true;
}
}
catch (OperationCanceledException) { }
}
if (errorCts.IsCancellationRequested)
{
throw new Exception("error occured, and we're failing fast.");
}
}
/// <summary>
/// 创建消费者生成器
/// </summary>
private void CreateConsumerBuilder()
{
if (disposed)
{
throw new ObjectDisposedException(nameof(KafkaConsumer));
}
if (builder == null)
{
lock (this)
{
if (builder == null)
{
ConsumerConfig config = new ConsumerConfig();
config.BootstrapServers = BootstrapServers;
config.GroupId = GroupId;
config.AutoOffsetReset = AutoOffsetReset.Earliest;
config.EnableAutoCommit = EnableAutoCommit;
if (!string.IsNullOrEmpty(SaslUsername))
{
config.SaslUsername = SaslUsername;
config.SaslPassword = SaslPassword;
config.SaslMechanism = SaslMechanism.Plain;
config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
}
//config.EnableAutoOffsetStore = true;
//config.IsolationLevel = IsolationLevel.ReadCommitted;
//config.MaxPollIntervalMs = 10000;
//List<KeyValuePair<string, string>> config = new List<KeyValuePair<string, string>>();
//config.Add(new KeyValuePair<string, string>("bootstrap.servers", BootstrapServers));
//config.Add(new KeyValuePair<string, string>("group.id", GroupId));
//config.Add(new KeyValuePair<string, string>("auto.offset.reset", "earliest"));
//config.Add(new KeyValuePair<string, string>("enable.auto.commit", EnableAutoCommit.ToString().ToLower()));
//if (!string.IsNullOrEmpty(SaslUsername))
//{
// config.Add(new KeyValuePair<string, string>("security.protocol", "SASL_PLAINTEXT"));
// config.Add(new KeyValuePair<string, string>("sasl.mechanism", "PLAIN"));
// config.Add(new KeyValuePair<string, string>("sasl.username", SaslUsername));
// config.Add(new KeyValuePair<string, string>("sasl.password", SaslPassword));
//}
//config.Add(new KeyValuePair<string, string>("max.poll.interval.ms", "10000"));
//config.Add(new KeyValuePair<string, string>("session.timeout.ms", "10000"));
//config.Add(new KeyValuePair<string, string>("isolation.level", "read_uncommitted"));
builder = new ConsumerBuilder <string, object>(config);
Action <Delegate, object> tryCatchWrap = (@delegate, arg) =>
{
try
{
@delegate?.DynamicInvoke(arg);
}
catch { }
};
builder.SetErrorHandler((p, e) => tryCatchWrap(ErrorHandler, new Exception(e.Reason)));
builder.SetStatisticsHandler((p, e) => tryCatchWrap(StatisticsHandler, e));
builder.SetLogHandler((p, e) => tryCatchWrap(LogHandler, new KafkaLogMessage(e)));
builder.SetValueDeserializer(new KafkaConverter());
}
}
}
}