private void KafkaAutoCommittedOffsets(object state)
{
ConsumerSetting setting = state as ConsumerSetting;
var config = new Dictionary <string, object>
{
{ "bootstrap.servers", setting.Broker },
{ "group.id", setting.GroupID },
{ "enable.auto.commit", true }, // this is the default
{ "auto.commit.interval.ms", 5000 },
{ "statistics.interval.ms", 60000 },
{ "session.timeout.ms", 6000 },
{ "auto.offset.reset", "smallest" }
};
using (var consumer = new Consumer <Ignore, string>(config, null, new StringDeserializer(Encoding.UTF8)))
{
if (setting.Action != null)
{
consumer.OnMessage += (_, message) =>
{
ConsumerResult messageResult = new ConsumerResult();
messageResult.Broker = setting.Broker;
messageResult.Topic = message.Topic;
messageResult.Partition = message.Partition;
messageResult.Offset = message.Offset.Value;
messageResult.Message = message.Value;
//执行外界自定义的方法
setting.Action(messageResult);
};
}
consumer.Subscribe(setting.Topic);
while (!setting.CancelSource.IsCancellationRequested)
{
consumer.Poll(TimeSpan.FromMilliseconds(100));
}
}
}
/// <summary>
/// offsets are manually committed.
/// no extra thread is created for the Poll loop.
/// </summary>
public void Consume(string broker, string topic, string groupId, CancellationTokenSource cancelSource, Action <ConsumerResult> callbackAction = null)
{
if (string.IsNullOrEmpty(broker) || string.IsNullOrWhiteSpace(broker) || broker.Length <= 0)
{
throw new ArgumentNullException("broker argument for consumer can't be null.");
}
if (string.IsNullOrWhiteSpace(topic) || string.IsNullOrEmpty(topic) || topic.Length <= 0)
{
throw new ArgumentNullException("topic argument for consumer can't be null.");
}
if (string.IsNullOrWhiteSpace(groupId) || string.IsNullOrEmpty(groupId) || groupId.Length <= 0)
{
throw new ArgumentNullException("message argument for consumer can't be null.");
}
var config = new Dictionary <string, object>
{
{ "bootstrap.servers", broker },
{ "group.id", groupId },
{ "enable.auto.commit", false },
{ "statistics.interval.ms", 60000 },
{ "session.timeout.ms", 6000 },
{ "auto.offset.reset", "smallest" }
};
using (var consumer = new Consumer <Ignore, string>(config, new IgnoreDeserializer(), new StringDeserializer(Encoding.UTF8)))
{
// Note: All event handlers are called on the main thread.
consumer.OnPartitionEOF += (_, end)
=> Console.WriteLine($"Reached end of topic {end.Topic} partition {end.Partition}, next message will be at offset {end.Offset}");
consumer.OnError += (_, error)
=> Console.WriteLine($"Error: {error}");
// Raised on deserialization errors or when a consumed message has an error != NoError.
consumer.OnConsumeError += (_, error)
=> Console.WriteLine($"Consume error: {error}");
// Raised when the consumer is assigned a new set of partitions.
consumer.OnPartitionsAssigned += (_, partitions) =>
{
Console.WriteLine($"Assigned partitions: [{string.Join(", ", partitions)}], member id: {consumer.MemberId}");
// If you don't add a handler to the OnPartitionsAssigned event,
// the below .Assign call happens automatically. If you do, you
// must call .Assign explicitly in order for the consumer to
// start consuming messages.
consumer.Assign(partitions);
};
// Raised when the consumer's current assignment set has been revoked.
consumer.OnPartitionsRevoked += (_, partitions) =>
{
Console.WriteLine($"Revoked partitions: [{string.Join(", ", partitions)}]");
// If you don't add a handler to the OnPartitionsRevoked event,
// the below .Unassign call happens automatically. If you do,
// you must call .Unassign explicitly in order for the consumer
// to stop consuming messages from it's previously assigned
// partitions.
consumer.Unassign();
};
consumer.OnStatistics += (_, json)
=> Console.WriteLine($"Statistics: {json}");
consumer.Subscribe(topic);
Console.WriteLine($"Started consumer, Ctrl-C to stop consuming");
while (!cancelSource.IsCancellationRequested)
{
if (!consumer.Consume(out Message <Ignore, string> msg, TimeSpan.FromMilliseconds(100)))
{
continue;
}
if (msg != null)
{
ConsumerResult msgResult = new ConsumerResult();
msgResult.Broker = broker;
msgResult.Topic = msg.Topic;
msgResult.Partition = msg.Partition;
msgResult.Offset = msg.Offset;
msgResult.Message = msg.Value;
callbackAction?.Invoke(msgResult);
if (msg.Offset % 5 == 0)
{
var committedOffsets = consumer.CommitAsync(msg).Result;
Console.WriteLine($"Committed offset: {committedOffsets}");
}
}
}
}
}