/// <summary>
/// Initializes a new instance of the <see cref="KafkaReceiver"/> class.
/// </summary>
/// <param name="name">The name of the receiver.</param>
/// <param name="topic">
/// The topic to subscribe to. A regex can be specified to subscribe to the set of
/// all matching topics (which is updated as topics are added / removed from the
/// cluster). A regex must be front anchored to be recognized as a regex. e.g. ^myregex
/// </param>
/// <param name="consumerConfig">The configuration used in creation of the Kafka consumer.</param>
public KafkaReceiver(string name, string topic, ConsumerConfig consumerConfig)
: base(name)
{
if (consumerConfig is null)
{
throw new ArgumentNullException(nameof(consumerConfig));
}
if (consumerConfig.EnableAutoCommit is false)
{
throw new ArgumentOutOfRangeException(nameof(consumerConfig), "The 'EnableAutoCommit' setting must be true.");
}
Topic = topic ?? throw new ArgumentNullException(nameof(topic));
GroupId = consumerConfig.GroupId;
BootstrapServers = consumerConfig.BootstrapServers;
EnableAutoOffsetStore = consumerConfig.EnableAutoOffsetStore;
AutoOffsetReset = consumerConfig.AutoOffsetReset;
var builder = new ConsumerBuilder <string, byte[]>(consumerConfig);
builder.SetErrorHandler(OnError);
_consumer = new Lazy <IConsumer <string, byte[]> >(() => builder.Build());
_pollingThread = new Lazy <Thread>(() => new Thread(PollForMessages)
{
IsBackground = true
});
_trackingThread = new Lazy <Thread>(() => new Thread(TrackMessageHandling)
{
IsBackground = true
});
}
public IConsumer <byte[], byte[]> GetConsumer(ConsumerConfig config, IConsumerRebalanceListener rebalanceListener)
{
ConsumerBuilder <byte[], byte[]> builder = builderKafkaHandler.GetConsumerBuilder(config);
if (rebalanceListener != null)
{
builder.SetPartitionsAssignedHandler((c, p) => rebalanceListener.PartitionsAssigned(c, p));
builder.SetPartitionsRevokedHandler((c, p) => rebalanceListener.PartitionsRevoked(c, p));
builder.SetLogHandler(loggerAdapter.LogConsume);
builder.SetErrorHandler(loggerAdapter.ErrorConsume);
if (exposeLibrdKafka)
{
// TODO : test librdkafka statistics with IntegrationTest (WIP see #82)
var consumerStatisticsHandler = new ConsumerStatisticsHandler(
config.ClientId,
streamConfig.ApplicationId,
(config as StreamizConsumerConfig)?.ThreadId);
consumerStatisticsHandler.Register(MetricsRegistry);
builder.SetStatisticsHandler((c, stat) =>
{
var statistics = JsonConvert.DeserializeObject <Statistics>(stat);
consumerStatisticsHandler.Publish(statistics);
});
}
}
return(builder.Build());
}
/// <summary>
/// Initializes a new instance of the <see cref="KafkaReceiver"/> class.
/// </summary>
/// <param name="name">The name of the receiver.</param>
/// <param name="topic">
/// The topic to subscribe to. A regex can be specified to subscribe to the set of
/// all matching topics (which is updated as topics are added / removed from the
/// cluster). A regex must be front anchored to be recognized as a regex. e.g. ^myregex
/// </param>
/// <param name="groupId">
/// Client group id string. All clients sharing the same group.id belong to the same group.
/// </param>
/// <param name="bootstrapServers">
/// List of brokers as a CSV list of broker host or host:port.
/// </param>
/// <param name="enableAutoOffsetStore">
/// Whether to automatically store offset of last message provided to application.
/// </param>
/// <param name="autoOffsetReset">
/// Action to take when there is no initial offset in offset store or the desired
/// offset is out of range: 'smallest','earliest' - automatically reset the offset
/// to the smallest offset, 'largest','latest' - automatically reset the offset to
/// the largest offset, 'error' - trigger an error which is retrieved by consuming
/// messages and checking 'message->err'.
/// </param>
public KafkaReceiver(string name, string topic, string groupId, string bootstrapServers,
bool enableAutoOffsetStore = false, AutoOffsetReset autoOffsetReset = Confluent.Kafka.AutoOffsetReset.Latest)
: base(name)
{
Topic = topic ?? throw new ArgumentNullException(nameof(topic));
GroupId = groupId ?? throw new ArgumentNullException(nameof(groupId));
BootstrapServers = bootstrapServers ?? throw new ArgumentNullException(nameof(bootstrapServers));
EnableAutoOffsetStore = enableAutoOffsetStore;
AutoOffsetReset = autoOffsetReset;
var config = GetConsumerConfig(groupId, bootstrapServers, enableAutoOffsetStore, autoOffsetReset);
var builder = new ConsumerBuilder <string, byte[]>(config);
builder.SetErrorHandler(OnError);
_consumer = new Lazy <IConsumer <string, byte[]> >(() => builder.Build());
_pollingThread = new Lazy <Thread>(() => new Thread(PollForMessages)
{
IsBackground = true
});
_trackingThread = new Lazy <Thread>(() => new Thread(TrackMessageHandling)
{
IsBackground = true
});
}
public override Action Connection(IEnumerable <KeyValuePair <string, object> > options)
{
string borkerList = "";
var list = options.GetEnumerator();
while (list.MoveNext())
{
if ("BorkerList".Equals(list.Current.Key))
{
borkerList = list.Current.Value.ToString();
}
}
var cConfig = new ConsumerConfig
{
BootstrapServers = borkerList,
BrokerVersionFallback = "0.10.0.0",
ApiVersionFallbackMs = 0,
//SaslMechanism = SaslMechanism.Plain,
//SecurityProtocol = SecurityProtocol.SaslSsl,
//SslCaLocation = "/usr/local/etc/openssl/cert.pem", // suitable configuration for linux, osx.
// SslCaLocation = "c:\\path\\to\\cacert.pem", // windows
//SaslUsername = "******",
//SaslPassword = "******",
GroupId = Guid.NewGuid().ToString(),
AutoOffsetReset = AutoOffsetReset.Earliest
};
return(() =>
{
var consumerBuilder = new ConsumerBuilder <string, MessageBase>(cConfig);
consumerBuilder.SetErrorHandler(OnConnectionException);
_consumerClient = consumerBuilder.Build();
});
}
public IConsumer <Ignore, byte[]> CreateConsumer(
KafkaQueueConfiguration config,
Action <IConsumer <Ignore, byte[]>, LogMessage> logHandler = null,
Action <IConsumer <Ignore, byte[]>, Error> errorHandler = null)
{
config.ThrowIfNull(nameof(config));
var builder = new ConsumerBuilder <Ignore, byte[]>(new ConsumerConfig
{
GroupId = config.GroupId,
BootstrapServers = $"{config.Server}",
AutoOffsetReset = config.AutoOffsetReset,
SaslKerberosKeytab = config.KeyTab,
SaslKerberosPrincipal = config.User,
SaslKerberosServiceName = config.ServiceName,
SecurityProtocol = config.Protocol,
SaslMechanism = config.Mechanism,
Debug = config.Debug
});
if (logHandler != null)
{
builder.SetLogHandler(logHandler);
}
if (errorHandler != null)
{
builder.SetErrorHandler(errorHandler);
}
return(builder.Build());
}
/// <summary>
/// Initializes a new instance of the <see cref="KafkaReceiver"/> class.
/// </summary>
/// <param name="name">The name of the receiver.</param>
/// <param name="topic">
/// The topic to subscribe to. A regex can be specified to subscribe to the set of
/// all matching topics (which is updated as topics are added / removed from the
/// cluster). A regex must be front anchored to be recognized as a regex. e.g. ^myregex
/// </param>
/// <param name="groupId">
/// Client group id string. All clients sharing the same group.id belong to the same
/// group.
/// </param>
/// <param name="bootstrapServers">
/// List of brokers as a CSV list of broker host or host:port.
/// </param>
/// <param name="config">
/// A collection of librdkafka configuration parameters (refer to
/// https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) and parameters
/// specific to this client (refer to: Confluent.Kafka.ConfigPropertyNames).
/// </param>
public KafkaReceiver(string name, string topic, string groupId, string bootstrapServers, ConsumerConfig config = null)
: base(name)
{
Topic = topic ?? throw new ArgumentNullException(nameof(topic));
Config = config ?? new ConsumerConfig();
Config.GroupId = groupId ?? throw new ArgumentNullException(nameof(groupId));
Config.BootstrapServers = bootstrapServers ?? throw new ArgumentNullException(nameof(bootstrapServers));
Config.EnableAutoCommit = Config.EnableAutoCommit ?? false;
var consumerBuilder = new ConsumerBuilder <Ignore, string>(Config);
consumerBuilder.SetErrorHandler(OnError);
_consumer = new Lazy <IConsumer <Ignore, string> >(() => consumerBuilder.Build());
_pollingThread = new Lazy <Thread>(() => new Thread(PollForMessages)
{
IsBackground = true
});
_trackingThread = new Lazy <Thread>(() => new Thread(TrackMessageHandling)
{
IsBackground = true
});
}
private static void AddKafkaConsumer(this IServiceCollection services)
{
services.AddTransient(p =>
{
var logger = p.GetService <ILogger <KafkaConsumerService> >();
var kafkaOptions = p.GetService <IOptions <KafkaOptions> >();
var consumerConfig = new ConsumerConfig(kafkaOptions.Value.Configuration)
{
EnablePartitionEof = kafkaOptions.Value.EnablePartitionEof,
StatisticsIntervalMs = kafkaOptions.Value.StatisticsIntervalMs
};
var consumerBuilder = new ConsumerBuilder <string, string>(consumerConfig);
var consumer = consumerBuilder
.SetErrorHandler((_, e) => logger?.LogError($"Error: {e.Reason}", e))
.SetStatisticsHandler((_, json) => logger?.LogDebug($"Statistics: {json}"))
.SetPartitionsAssignedHandler((c, partitions) =>
{
logger?.LogInformation($"Assigned partitions: [{string.Join(", ", partitions)}]");
})
.SetPartitionsRevokedHandler((c, partitions) =>
{
logger?.LogInformation($"Revoking assignment: [{string.Join(", ", partitions)}]");
})
.Build();
return(consumer);
});
}
public void ConsumeMessage(string topicName)
{
String ErrorReason = String.Empty;
var consumeBldr = new ConsumerBuilder <Ignore, string>(_config.ConsumerConfig());
{
//Added Error handler
consumeBldr.SetErrorHandler(ConsumerErrorHandler);
consumeBldr.SetLogHandler(LogHandler);
var consumer = consumeBldr.Build();
try
{
consumer.Subscribe(topicName);
ReadMessages(consumer);
}
catch (Exception ex)
{
// if this exception occured then
ErrorReason = $"Error : { ex.Message } Type of { ex.GetType().Name } not handled and closing the consumer. { ex.InnerException?.Message } Source: { ex?.Source } .Stack :{ ex?.StackTrace}";
_worker.UnProcessedMessage("Empty Message Found", ErrorReason, SyslogLevel.Error);
consumer.Close();//TBD
consumer?.Dispose();
}
}
}
public IConsumer <string, string> Create()
{
var config = new ConsumerConfig(_configuration.GetConsumerConfiguration());
var builder = new ConsumerBuilder <string, string>(config);
builder.SetErrorHandler(OnKafkaError);
return(builder.Build());
}
public IConsumer <byte[], byte[]> GetRestoreConsumer(ConsumerConfig config)
{
ConsumerBuilder <byte[], byte[]> builder = builderKafkaHandler.GetConsumerBuilder(config);
builder.SetLogHandler(loggerAdapter.LogConsume);
builder.SetErrorHandler(loggerAdapter.ErrorConsume);
return(builder.Build());
}
public IConsumer <byte[], byte[]> GetGlobalConsumer(ConsumerConfig config)
{
ConsumerBuilder <byte[], byte[]> builder = new ConsumerBuilder <byte[], byte[]>(config);
// TOOD : Finish
builder.SetLogHandler(loggerAdapter.LogConsume);
builder.SetErrorHandler(loggerAdapter.ErrorConsume);
return(builder.Build());
}
public IConsumer <byte[], byte[]> GetGlobalConsumer(ConsumerConfig config)
{
config.AutoOffsetReset = AutoOffsetReset.Earliest;
ConsumerBuilder <byte[], byte[]> builder = builderKafkaHandler.GetConsumerBuilder(config);
builder.SetLogHandler(loggerAdapter.LogConsume);
builder.SetErrorHandler(loggerAdapter.ErrorConsume);
return(builder.Build());
}
/// <summary>
/// 订阅回调模式-消费(持续订阅)
/// </summary>
/// <param name="Func">回调函数,若配置为非自动提交(默认为否),则通过回调函数的返回值判断是否提交</param>
/// <param name="Topic">主题</param>
public void Consume(Func <ConsumeResult <TKey, TValue>, bool> Func, string Topic)
{
Task.Factory.StartNew(() =>
{
var builder = new ConsumerBuilder <TKey, TValue>(ConsumerConfig);
//设置反序列化方式
builder.SetValueDeserializer(new KafkaDConverter <TValue>());
builder.SetErrorHandler((_, e) =>
{
Logger.Error(LoggerType.KafkaException, null, null, $"Error:{e.Reason}");
}).SetStatisticsHandler((_, json) =>
{
Console.WriteLine($"-{DateTime.Now:yyyy-MM-dd HH:mm:ss} > 消息监听中..");
}).SetPartitionsAssignedHandler((c, partitions) =>
{
string partitionsStr = string.Join(", ", partitions);
Console.WriteLine($"-分配的kafka分区:{partitionsStr}");
}).SetPartitionsRevokedHandler((c, partitions) =>
{
string partitionsStr = string.Join(", ", partitions);
Console.WriteLine($"-回收了kafka的分区:{partitionsStr}");
});
using var consumer = builder.Build();
consumer.Subscribe(Topic);
while (AppSetting.Kafka.IsConsumerSubscribe) //true
{
ConsumeResult <TKey, TValue> result = null;
try
{
result = consumer.Consume();
if (result.IsPartitionEOF)
{
continue;
}
if (Func(result))
{
if (!(bool)ConsumerConfig.EnableAutoCommit)
{
//手动提交，如果上面的EnableAutoCommit=true表示自动提交，则无需调用Commit方法
consumer.Commit(result);
}
}
}
catch (ConsumeException ex)
{
Logger.Error(LoggerType.KafkaException, $"Topic:{Topic},{ex.Error.Reason}", null, ex.Message + ex.StackTrace);
}
catch (Exception ex)
{
Logger.Error(LoggerType.KafkaException, $"Topic:{result.Topic}", null, ex.Message + ex.StackTrace);
}
}
});
}
private static void StartConsuming <T>()
{
bool consuming = true;
ConsumerBuilder <Ignore, T> consumerBuilder = new ConsumerBuilder <Ignore, T>(Configuration.ConsumerConfig);
consumerBuilder.SetErrorHandler((s, e) =>
{
consuming = !e.IsFatal;
Log(e.ToString());
});
using (IConsumer <Ignore, T> consumer = consumerBuilder.Build())
{
ConsumerConfig consumerConfig = Configuration.ConsumerConfig;
Log($"Consumer for group: '{consumerConfig.GroupId}' was created. Data type: '{Configuration.Type}'.");
consumer.Subscribe(new[] { Configuration.TopicBlocks, Configuration.TopicTransactions, Configuration.TopicReceipts });
Log($"Subscribed to topics: '{Configuration.TopicBlocks}', {Configuration.TopicTransactions}, '{Configuration.TopicReceipts}'.");
while (consuming)
{
try
{
ConsumeResult <Ignore, T> consumeResult = consumer.Consume();
Type type = typeof(T);
if (type == typeof(string))
{
ConsumeAsJson(consumeResult as ConsumeResult <Ignore, string>);
}
else if (type == typeof(byte[]))
{
ConsumeAsUtf8Json(consumeResult as ConsumeResult <Ignore, byte[]>);
}
else
{
Log($"Unknown data type: {type.Name}");
continue;
}
Log($"Consumed value at '{consumeResult.TopicPartitionOffset}'.");
}
catch (ConsumeException exception)
{
Log($"Consumer error occured: {exception.Error.Reason}");
}
catch (Exception exception)
{
Log(exception.Message);
}
}
consumer.Close();
}
}
public IConsumer <byte[], byte[]> GetConsumer(ConsumerConfig config, IConsumerRebalanceListener rebalanceListener)
{
ConsumerBuilder <byte[], byte[]> builder = new ConsumerBuilder <byte[], byte[]>(config);
if (rebalanceListener != null)
{
builder.SetPartitionsAssignedHandler((c, p) => rebalanceListener.PartitionsAssigned(c, p));
builder.SetPartitionsRevokedHandler((c, p) => rebalanceListener.PartitionsRevoked(c, p));
builder.SetLogHandler(loggerAdapter.LogConsume);
builder.SetErrorHandler(loggerAdapter.ErrorConsume);
}
return(builder.Build());
}
public JT809_Same_Consumer(
IOptions <JT809SameConsumerConfig> consumerConfigAccessor,
ILoggerFactory loggerFactory)
{
logger = loggerFactory.CreateLogger("JT809_Same_Consumer");
JT809ConsumerConfig = consumerConfigAccessor.Value;
TopicName = JT809ConsumerConfig.TopicName;
ConsumerBuilder <string, byte[]> consumerBuilder = new ConsumerBuilder <string, byte[]>(consumerConfigAccessor.Value);
consumerBuilder.SetErrorHandler((consumer, error) =>
{
logger.LogError(error.Reason);
});
Consumer = consumerBuilder.Build();
}
public IConsumer <T, TV> GetConsumerBuilder <T, TV>(string consumerId) where T : class where TV : class
{
if (string.IsNullOrEmpty(consumerId))
{
throw new ConsumerNotFoundException($"The consumerId param can not be null or empty");
}
var consumerOptions = KafkaOptions.Consumers.FirstOrDefault(p => p.ConsumerId == consumerId);
if (consumerOptions == null)
{
throw new ConsumerNotFoundException($"Could not find consumer configuration with ConsumerId {consumerId}");
}
var configuration = GetDefaultKafkaConsumerConfiguration(consumerOptions);
var consumer = new ConsumerBuilder <T, TV>(configuration);
IConsumer <T, TV> result = null;
try
{
consumer.SetErrorHandler((_, e) => Devon4NetLogger.Error(new ConsumerException($"Error code {e.Code} : {e.Reason}")));
consumer.SetStatisticsHandler((_, json) => Devon4NetLogger.Information($"Statistics: {json}"));
consumer.SetPartitionsAssignedHandler((c, partitions) =>
{
Devon4NetLogger.Information($"Assigned partitions: [{string.Join(", ", partitions)}]");
});
consumer.SetPartitionsRevokedHandler((c, partitions) =>
{
Devon4NetLogger.Information($"Revoking assignment: [{string.Join(", ", partitions)}]");
});
result = consumer.Build();
if (!string.IsNullOrEmpty(consumerOptions.Topics))
{
result.Subscribe(consumerOptions.GetTopics());
}
}
catch (InvalidOperationException ex)
{
Devon4NetLogger.Error(ex);
}
return(result);
}
protected virtual IList <IConsumer <string, T> > CreateConsumers()
{
List <IConsumer <string, T> > consumers = new List <IConsumer <string, T> >();
foreach (var topicPartition in topicPartitionList)
{
ConsumerBuilder <string, T> consumerBuilder = new ConsumerBuilder <string, T>(ConsumerConfig);
consumerBuilder.SetErrorHandler((consumer, error) =>
{
logger.LogError(error.Reason);
});
if (Deserializer != null)
{
consumerBuilder.SetValueDeserializer(Deserializer);
}
consumers.Add(consumerBuilder.Build());
}
return(consumers);
}
/// <summary>
/// 同步运行状态
/// </summary>
/// <returns></returns>
Task <bool> IMessageReceiver.LoopBegin()
{
//KafkaOption.Instance.Consumer.Set(ConfigPropertyNames.Consumer.ConsumeResultFields, "all");
builder = new ConsumerBuilder <byte[], string>(KafkaOption.Instance.Consumer);
//var value = KafkaOption.Instance.Consumer.Get(ConfigPropertyNames.Consumer.ConsumeResultFields);
builder.SetErrorHandler(OnError);
builder.SetLogHandler(OnLog);
consumer = builder.Build();
consumer.Subscribe(Service.ServiceName);
if (KafkaOption.Instance.Message.Concurrency > 0)
{
ConcurrencySemaphore = new SemaphoreSlim(KafkaOption.Instance.Message.Concurrency, KafkaOption.Instance.Message.Concurrency);
}
return(Task.FromResult(true));
}
protected JT809Consumer(
IOptions <JT809TopicOptions> topicOptionsAccessor,
IOptions <ConsumerConfig> consumerConfigAccessor,
ILoggerFactory loggerFactory)
: base(topicOptionsAccessor.Value.TopicName, consumerConfigAccessor.Value)
{
logger = loggerFactory.CreateLogger("JT809Consumer");
Consumers = new List <IConsumer <string, T> >();
ConsumerBuilder <string, T> consumerBuilder = new ConsumerBuilder <string, T>(ConsumerConfig);
consumerBuilder.SetErrorHandler((consumer, error) =>
{
logger.LogError(error.Reason);
});
if (Deserializer != null)
{
consumerBuilder.SetValueDeserializer(Deserializer);
}
Consumers.Add(consumerBuilder.Build());
}
/// <summary>
/// Initializes a new instance of the <see cref="KafkaReceiver"/> class.
/// </summary>
/// <param name="name">The name of the receiver.</param>
/// <param name="topic">
/// The topic to subscribe to. A regex can be specified to subscribe to the set of
/// all matching topics (which is updated as topics are added / removed from the
/// cluster). A regex must be front anchored to be recognized as a regex. e.g. ^myregex
/// </param>
/// <param name="groupId">
/// Client group id string. All clients sharing the same group.id belong to the same group.
/// </param>
/// <param name="bootstrapServers">
/// List of brokers as a CSV list of broker host or host:port.
/// </param>
/// <param name="enableAutoOffsetStore">
/// Whether to automatically store offset of last message provided to application.
/// </param>
/// <param name="autoOffsetReset">
/// Action to take when there is no initial offset in offset store or the desired
/// offset is out of range: 'smallest','earliest' - automatically reset the offset
/// to the smallest offset, 'largest','latest' - automatically reset the offset to
/// the largest offset, 'error' - trigger an error which is retrieved by consuming
/// messages and checking 'message->err'.
/// </param>
/// <param name="replayEngine">
/// The <see cref="IReplayEngine"/> used to replay messages. If <see langword="null"/>,
/// then a <see cref="DefaultReplayEngine"/> is used.
/// </param>
public KafkaReceiver(string name, string topic, string groupId, string bootstrapServers,
bool enableAutoOffsetStore = false, AutoOffsetReset autoOffsetReset = AutoOffsetReset.Latest,
IReplayEngine replayEngine = null)
: base(name)
{
if (string.IsNullOrEmpty(topic))
{
throw new ArgumentNullException(nameof(topic));
}
if (string.IsNullOrEmpty(groupId))
{
throw new ArgumentNullException(nameof(groupId));
}
if (string.IsNullOrEmpty(bootstrapServers))
{
throw new ArgumentNullException(nameof(bootstrapServers));
}
Topic = topic;
GroupId = groupId;
BootstrapServers = bootstrapServers;
EnableAutoOffsetStore = enableAutoOffsetStore;
AutoOffsetReset = autoOffsetReset;
ReplayEngine = replayEngine ?? _defaultReplayEngine.Value;
var config = GetConsumerConfig(GroupId, BootstrapServers, EnableAutoOffsetStore, AutoOffsetReset);
var builder = new ConsumerBuilder <string, byte[]>(config);
builder.SetErrorHandler(OnError);
_consumer = new Lazy <IConsumer <string, byte[]> >(() => builder.Build());
_pollingThread = new Lazy <Thread>(() => new Thread(PollForMessages)
{
IsBackground = true
});
_trackingThread = new Lazy <Thread>(() => new Thread(TrackMessageHandling)
{
IsBackground = true
});
}
/// <inheritdoc cref="IConfluentConsumerBuilder.Build" />
public IConsumer <byte[]?, byte[]?> Build()
{
if (_config == null)
{
throw new InvalidOperationException("SetConfig must be called to provide the consumer configuration.");
}
var builder = new ConsumerBuilder <byte[]?, byte[]?>(_config);
if (_statisticsHandler != null)
{
builder.SetStatisticsHandler(_statisticsHandler);
}
if (_errorHandler != null)
{
builder.SetErrorHandler(_errorHandler);
}
if (_partitionsAssignedHandler != null)
{
builder.SetPartitionsAssignedHandler(_partitionsAssignedHandler);
}
if (_partitionsRevokedHandler != null)
{
builder.SetPartitionsRevokedHandler(_partitionsRevokedHandler);
}
if (_offsetsCommittedHandler != null)
{
builder.SetOffsetsCommittedHandler(_offsetsCommittedHandler);
}
if (_logHandler != null)
{
builder.SetLogHandler(_logHandler);
}
return(builder.Build());
}
public static void ReadDatas(string brokerList, string topic, int partition, int offsetMin, int offsetMax)
{
try
{
Console.WriteLine("Start read datas");
var config = new ConsumerConfig
{
BootstrapServers = brokerList,
GroupId = "csharp-consumer",
EnableAutoCommit = false,
SessionTimeoutMs = 6000,
// AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = true
};
var consumerBuilder = new ConsumerBuilder <string, string>(config);
consumerBuilder.SetErrorHandler(MyErrorHandler);
consumerBuilder.SetLogHandler(MyLogHandler);
var consumer = consumerBuilder.Build();
// Read datas from
consumer.Assign(new TopicPartitionOffset(topic, new Partition(partition), new Offset(offsetMin)));
ConsumeResult <string, string> result;
var sw = new Stopwatch();
do
{
sw.Restart();
result = consumer.Consume(10_000);
Console.WriteLine(sw.ElapsedMilliseconds);
if (result != null)
{
Console.WriteLine($"{result.IsPartitionEOF} {result.TopicPartitionOffset}: '{result.Message?.Key}' | '{result.Message?.Value}'");
}
}while (result != null && !result.IsPartitionEOF && result.Offset < offsetMax);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
private ConsumerBuilder <K, T> CreateKafkaConsumerBuilder()
{
var kafkaConsumerBuilder = new ConsumerBuilder <K, T>(consumerConfig);
kafkaConsumerBuilder.SetKeyDeserializer(keyDeserializer);
kafkaConsumerBuilder.SetValueDeserializer(valueDeserializer);
kafkaConsumerBuilder.SetErrorHandler((_, e) => OnError?.Invoke(new StreamingError {
IsFatal = e.IsFatal, Reason = e.Reason
}));
kafkaConsumerBuilder.SetStatisticsHandler((_, statistics) => OnStatistics?.Invoke(statistics));
if (partitionsAssignedHandle != null && CommitEnable)
{
throw new ArgumentException("The partition assigned handle can not been setted if " +
"'CommitEnable' property was setted to true.");
}
else if (partitionsAssignedHandle != null)
{
kafkaConsumerBuilder.SetPartitionsAssignedHandler(partitionsAssignedHandle);
}
return(kafkaConsumerBuilder);
}
public async Task ConsumeMessages(CancellationToken stoppingToken)
{
var kafkaConfig = new ConsumerConfig
{
// Note: The AutoOffsetReset property determines the start offset in the event
// there are not yet any committed offsets for the consumer group for the
// topic/partitions of interest. By default, offsets are committed
// automatically, so in this example, consumption will only start from the
// earliest message in the topic 'my-topic' the first time you run the program.
AutoOffsetReset = AutoOffsetReset.Earliest,
GroupId = ConsumerGroupId,
BootstrapServers = Kafka
};
var consumer = new ConsumerBuilder <Key, Event>(kafkaConfig);
consumer.SetErrorHandler(ConsumerErrorHandler);
consumer.SetStatisticsHandler(ConsumerStatsHandler);
using (var schemaRegistry = new CachedSchemaRegistryClient(_schemaRegistryConfig))
{
consumer.SetKeyDeserializer(new AvroDeserializer <Key>(schemaRegistry).AsSyncOverAsync());
consumer.SetValueDeserializer(new AvroDeserializer <Event>(schemaRegistry).AsSyncOverAsync());
using (var c = consumer.Build())
{
c.Subscribe(_topics.FirstOrDefault());
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true; // prevent the process from terminating.
};
try
{
while (!stoppingToken.IsCancellationRequested)
{
try
{
var cr = c.Consume(stoppingToken);
if (!cr.IsPartitionEOF)
{
ProcessMessage(cr);
}
}
catch (ConsumeException e)
{
_logger.LogError($"Error occured in Kafka Consumer service: {e.Error.Reason + Environment.NewLine + e.StackTrace}");
}
await Task.Delay(VerificationDelay, stoppingToken);
}
}
catch (OperationCanceledException)
{
// Ensure the consumer leaves the group cleanly and final offsets are committed.
c.Close();
}
}
}
}
/// <summary>
/// 创建消费者生成器
/// </summary>
private void CreateConsumerBuilder()
{
if (disposed)
{
throw new ObjectDisposedException(nameof(KafkaConsumer));
}
if (builder == null)
{
lock (this)
{
if (builder == null)
{
ConsumerConfig config = new ConsumerConfig();
config.BootstrapServers = BootstrapServers;
config.GroupId = GroupId;
config.AutoOffsetReset = AutoOffsetReset.Earliest;
config.EnableAutoCommit = EnableAutoCommit;
if (!string.IsNullOrEmpty(SaslUsername))
{
config.SaslUsername = SaslUsername;
config.SaslPassword = SaslPassword;
config.SaslMechanism = SaslMechanism.Plain;
config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
}
//config.EnableAutoOffsetStore = true;
//config.IsolationLevel = IsolationLevel.ReadCommitted;
//config.MaxPollIntervalMs = 10000;
//List<KeyValuePair<string, string>> config = new List<KeyValuePair<string, string>>();
//config.Add(new KeyValuePair<string, string>("bootstrap.servers", BootstrapServers));
//config.Add(new KeyValuePair<string, string>("group.id", GroupId));
//config.Add(new KeyValuePair<string, string>("auto.offset.reset", "earliest"));
//config.Add(new KeyValuePair<string, string>("enable.auto.commit", EnableAutoCommit.ToString().ToLower()));
//if (!string.IsNullOrEmpty(SaslUsername))
//{
// config.Add(new KeyValuePair<string, string>("security.protocol", "SASL_PLAINTEXT"));
// config.Add(new KeyValuePair<string, string>("sasl.mechanism", "PLAIN"));
// config.Add(new KeyValuePair<string, string>("sasl.username", SaslUsername));
// config.Add(new KeyValuePair<string, string>("sasl.password", SaslPassword));
//}
//config.Add(new KeyValuePair<string, string>("max.poll.interval.ms", "10000"));
//config.Add(new KeyValuePair<string, string>("session.timeout.ms", "10000"));
//config.Add(new KeyValuePair<string, string>("isolation.level", "read_uncommitted"));
builder = new ConsumerBuilder <string, object>(config);
Action <Delegate, object> tryCatchWrap = (@delegate, arg) =>
{
try
{
@delegate?.DynamicInvoke(arg);
}
catch { }
};
builder.SetErrorHandler((p, e) => tryCatchWrap(ErrorHandler, new Exception(e.Reason)));
builder.SetStatisticsHandler((p, e) => tryCatchWrap(StatisticsHandler, e));
builder.SetLogHandler((p, e) => tryCatchWrap(LogHandler, new KafkaLogMessage(e)));
builder.SetValueDeserializer(new KafkaConverter());
}
}
}
}
public static void Main(string[] args)
{
var configuration = GetConfiguration(args);
try
{
var prometheusConfig = configuration.GetSection("prometheusMetrics").Get <PrometheusConfig>();
MetricServer metricServer = null;
if (prometheusConfig.Enabled)
{
metricServer = new MetricServer(port: prometheusConfig.Port);
metricServer.Start();
}
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
var consumerConf = configuration.GetSection("consumerConf").Get <ConsumerConfig>();
consumerConf.GroupId = Guid.NewGuid().ToString();
ConsumerBuilder <Null, string> builder = new ConsumerBuilder <Null, string>(consumerConf);
builder.SetErrorHandler((_, error) =>
{
Console.WriteLine($"An error ocurred consuming the event: {error.Reason}");
if (error.IsFatal)
{
Environment.Exit(-1);
}
});
builder.HandleStatistics(new PrometheusConsumerStatisticsHandler(new string[] { "application" }, new string[] { "test-consumer-statistics" }));
builder.SetKeyDeserializer(Deserializers.Null);
builder.SetValueDeserializer(Deserializers.Utf8);
using (var kafkaConsumer = builder.Build())
{
kafkaConsumer.Subscribe(configuration.GetValue <string>("topic"));
while (!cancellationTokenSource.IsCancellationRequested)
{
ConsumeResult <Null, string> consumedResult;
try
{
consumedResult = kafkaConsumer.Consume(cancellationTokenSource.Token);
if (null != consumedResult)
{
Console.WriteLine($"Received message: {consumedResult.Value}");
}
}
catch (Exception ex)
{
Console.WriteLine("An error occurred consuming the event.", ex);
Environment.Exit(-2);
}
}
}
Console.WriteLine("Exit requested. Gracefully exiting...");
}
catch (Exception ex)
{
Console.WriteLine("An error occurred while starting up the test.", ex);
Environment.Exit(-2);
}
}
private static void StartConsumingAvro <T>(string topic) where T : ISpecificRecord
{
bool consuming = true;
ConsumerBuilder <Ignore, T> consumerBuilder = new ConsumerBuilder <Ignore, T>(Configuration.ConsumerConfig);
consumerBuilder.SetErrorHandler((s, e) =>
{
consuming = !e.IsFatal;
Log(e.ToString());
});
CachedSchemaRegistryClient schemaRegistry = new CachedSchemaRegistryClient(new[]
{
new KeyValuePair <string, string>(SchemaRegistryConfig.PropertyNames.SchemaRegistryUrl, Configuration.SchemaRegistryUrl)
});
var deserializer = new AvroDeserializer <T>(schemaRegistry).AsSyncOverAsync();
consumerBuilder.SetValueDeserializer(deserializer);
IConsumer <Ignore, T> consumer = consumerBuilder.Build();
using (consumer)
{
ConsumerConfig consumerConfig = Configuration.ConsumerConfig;
Log($"Consumer for group: '{consumerConfig.GroupId}' was created. Data type: '{Configuration.Type}'.");
consumer.Subscribe(new[] { topic });
Log($"Subscribed to topic: '{topic}'.");
while (consuming)
{
try
{
ConsumeResult <Ignore, T> consumeResult = consumer.Consume();
Type type = typeof(T);
if (type == typeof(Avro.Models.Block))
{
ConsumeResult <Ignore, Avro.Models.Block> result = consumeResult as ConsumeResult <Ignore, Avro.Models.Block>;
Avro.Models.Block block = result.Value;
Log($"Block: {block.blockNumber} {block.blockHash}");
}
else if (type == typeof(Avro.Models.FullTransaction))
{
ConsumeResult <Ignore, FullTransaction> result = consumeResult as ConsumeResult <Ignore, Avro.Models.FullTransaction>;
FullTransaction transaction = result.Value;
Log($"Transaction for block: {transaction.blockNumber} {transaction.receipt.blockHash}");
}
else
{
Log($"Unknown data type: {type.Name}");
continue;
}
Log($"Consumed value at '{consumeResult.TopicPartitionOffset}'.");
}
catch (ConsumeException exception)
{
Log($"Consumer error occured: {exception.Error.Reason}");
}
catch (Exception exception)
{
Log(exception.Message);
}
}
consumer.Close();
}
}
public void Start(string instanceId, CancellationToken cancellationToken = default(CancellationToken))
{
funcExecSemaphore = new Semaphore(MaxOutstanding, MaxOutstanding);
CancellationTokenSource errorCts = new CancellationTokenSource();
CancellationTokenSource compositeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, errorCts.Token);
CancellationToken compositeCancellationToken = compositeCts.Token;
bool aMessageHasBeenProcessed = false;
var cConfig = new ConsumerConfig
{
ClientId = $"{Name}-consumer-{instanceId}",
GroupId = $"{Name}-group",
BootstrapServers = BootstrapServers,
EnableAutoCommit = true,
EnableAutoOffsetStore = false,
AutoOffsetReset = AutoOffsetReset.Latest
};
if (DebugContext != null)
{
cConfig.Debug = DebugContext;
}
var cBuilder = new ConsumerBuilder <TInKey, TInValue>(cConfig);
if (InKeyDeserializer != null)
{
cBuilder.SetKeyDeserializer(InKeyDeserializer);
}
if (InValueDeserializer != null)
{
cBuilder.SetValueDeserializer(InValueDeserializer);
}
if (Logger != null)
{
cBuilder.SetLogHandler((_, m) =>
{
Logger(m);
});
}
cBuilder.SetErrorHandler((c, e) =>
{
if (e.Code == ErrorCode.Local_AllBrokersDown ||
e.Code == ErrorCode.Local_Authentication)
{
if (!aMessageHasBeenProcessed)
{
// Logger.Log(e);
errorCts.Cancel();
return;
}
}
if (Logger != null)
{
Logger(new LogMessage(c.Name, SyslogLevel.Error, "unknown", e.Reason));
}
});
var pConfig = new ProducerConfig
{
ClientId = $"{Name}-producer-{instanceId}",
BootstrapServers = BootstrapServers,
EnableIdempotence = true,
LingerMs = 5,
DeliveryReportFields = "none"
};
if (DebugContext != null)
{
pConfig.Debug = DebugContext;
}
var pBuilder = new ProducerBuilder <TOutKey, TOutValue>(pConfig);
if (OutKeySerializer != null)
{
pBuilder.SetKeySerializer(OutKeySerializer);
}
if (OutValueSerializer != null)
{
pBuilder.SetValueSerializer(OutValueSerializer);
}
if (Logger != null)
{
pBuilder.SetLogHandler((_, m) =>
{
Logger(m);
});
}
pBuilder.SetErrorHandler((p, e) =>
{
if (e.IsFatal)
{
errorCts.Cancel();
return;
}
if (e.Code == ErrorCode.Local_AllBrokersDown ||
e.Code == ErrorCode.Local_Authentication)
{
if (!aMessageHasBeenProcessed)
{
errorCts.Cancel();
return;
}
}
if (Logger != null)
{
Logger(new LogMessage(p.Name, SyslogLevel.Error, "unknown", e.Reason));
}
});
var partitionState = new Dictionary <TopicPartition, PartitionState>();
using (var producer = pBuilder.Build())
using (var consumer = cBuilder.Build())
{
consumer.Subscribe(InputTopic);
try
{
while (true)
{
ConsumeResult <TInKey, TInValue> cr;
try
{
cr = consumer.Consume(compositeCancellationToken);
}
catch (ConsumeException ex)
{
if (ex.Error.Code == ErrorCode.Local_ValueDeserialization)
{
// For an in-depth discussion of what to do in the event of deserialization errors, refer to:
// https://www.confluent.io/blog/kafka-connect-deep-dive-error-handling-dead-letter-queues
if (ConsumeErrorTolerance == ErrorTolerance.All)
{
continue;
}
errorCts.Cancel(); // no error tolerance.
}
Thread.Sleep(TimeSpan.FromSeconds(10)); // ?? if not fail fast, do we want to sleep and why?
continue;
}
if (!partitionState.ContainsKey(cr.TopicPartition))
{
partitionState.Add(cr.TopicPartition, new PartitionState(this));
}
partitionState[cr.TopicPartition].HandleConsumedMessage(cr, consumer, producer, funcExecSemaphore, errorCts);
aMessageHasBeenProcessed = true;
}
}
catch (OperationCanceledException) { }
}
if (errorCts.IsCancellationRequested)
{
throw new Exception("error occured, and we're failing fast.");
}
}