async Task CommitOffsets()
{
List <TopicPartitionOffset> offSetsToCommit = new List <TopicPartitionOffset>();
List <TopicPartitionOffset> offSetsToRemove = new List <TopicPartitionOffset>();
var partitions = from offset in OffsetsReceived.Keys
group offset by new
{
offset.Topic,
offset.Partition
} into keys
select new
{
offsetNames = keys.Key.Topic + keys.Key.Partition,
offsetValues = keys.ToList()
};
foreach (var k in partitions)
{
var orderedOffsets = k.offsetValues.OrderBy(p => p.Offset);
TopicPartitionOffset maxOffset = new TopicPartitionOffset();
var offsetFound = false;
foreach (var o in orderedOffsets)
{
maxOffset.Offset = o.Offset;
maxOffset.Partition = o.Partition;
maxOffset.Topic = o.Topic;
offsetFound = true;
offSetsToRemove.Add(o);
}
if (offsetFound)
{
maxOffset.Offset = maxOffset.Offset + 1;
offSetsToCommit.Add(maxOffset);
}
}
if (offSetsToCommit.Count > 0)
{
await consumer.Commit(offSetsToCommit).ConfigureAwait(false);
}
foreach (var offsetToRemove in offSetsToRemove)
{
bool aux;
if (!OffsetsReceived.TryRemove(offsetToRemove, out aux))
{
Logger.Info("offset received could not be removed from list");
return;
}
}
}
private async Task Handle(Message kafkaMessage, string type, JObject message)
{
await _handler(type, message);
await _consumer.Commit(kafkaMessage).ConfigureAwait(false);
}
public void SubscribeTopic <T, TH>(Func <TH> handler, string borkerList, string groupID, List <string> topics)
where TH : IProcessMessage <T>
where T : MessageBase
{
var containsKey = _subsManager.HasSubscriptionsForEvent <T>();
if (!containsKey)
{
Task.Run(() =>
{
try
{
if (string.IsNullOrWhiteSpace(borkerList))
{
borkerList = ConnectionBuilder.getBorkerList();
}
bool enableAutoCommit = false;
var config = new Config()
{
GroupId = groupID,
EnableAutoCommit = enableAutoCommit,
StatisticsInterval = TimeSpan.FromSeconds(60)
};
EventConsumer consumer = new EventConsumer(config, borkerList);
consumer.OnMessage += async(obj, msgs) => {
string text = Encoding.UTF8.GetString(msgs.Payload, 0, msgs.Payload.Length);
_logger.LogInformation("Topic: {0} Partition: {1} Offset: {2} {3}", msgs.Topic, msgs.Partition, msgs.Offset, text);
MessageBase mb = new MessageBase()
{
MessageBodyByte = msgs.Payload,
MessageRouter = msgs.Topic
};
await ProcessEvent(mb.MessageRouter, mb);
if (!enableAutoCommit && msgs.Offset % 10 == 0)
{
_logger.LogInformation($"Committing offset");
consumer.Commit(msgs).Wait();
_logger.LogInformation($"Committed offset");
}
};
consumer.OnConsumerError += (obj, errorCode) =>
{
_logger.LogWarning("Consumer Error: {0}", errorCode);
};
consumer.OnEndReached += (obj, end) => {
_logger.LogInformation("Reached end of topic {0} partition {1}, next message will be at offset {2}", end.Topic, end.Partition, end.Offset);
};
consumer.OnError += (obj, error) => {
_logger.LogWarning("Error: {0} {1}", error.ErrorCode, error.Reason);
};
if (enableAutoCommit)
{
consumer.OnOffsetCommit += (obj, commit) => {
if (commit.Error != ErrorCode.NO_ERROR)
{
_logger.LogInformation("Failed to commit offsets: {0}", commit.Error);
}
_logger.LogInformation("Successfully committed offsets: [{0}]", string.Join(", ", commit.Offsets));
};
}
consumer.OnPartitionsAssigned += (obj, partitions) => {
_logger.LogInformation("Assigned partitions: [{0}], member id: {1}", string.Join(", ", partitions), consumer.MemberId);
consumer.Assign(partitions);
};
consumer.OnPartitionsRevoked += (obj, partitions) => {
_logger.LogInformation("Revoked partitions: [{0}]", string.Join(", ", partitions));
consumer.Unassign();
};
consumer.OnStatistics += (obj, json) => {
_logger.LogInformation("Statistics: {0}", json);
};
consumer.Subscribe(topics);
consumer.Start();
_logger.LogInformation("Assigned to: [{0}]", string.Join(", ", consumer.Assignment));
_logger.LogInformation("Subscribed to: [{0}]", string.Join(", ", consumer.Subscription));
_logger.LogInformation("Started consumer, press enter to stop consuming");
while (true)
{
Thread.Sleep(1);
}
// Console.ReadLine();
}
catch (Exception ex)
{
_logger.LogError("SubscribeTopicAsync", ex);
return(ex.Message);
}
});//----------end run
}//-----------end if
_subsManager.AddSubscription <T, TH>(handler);
}//-----------end SubscribeTopic
public static void Run(string brokerList, List <string> topics)
{
bool enableAutoCommit = false;
var config = new Config()
{
GroupId = "advanced-csharp-consumer",
EnableAutoCommit = enableAutoCommit,
StatisticsInterval = TimeSpan.FromSeconds(60)
};
using (var consumer = new EventConsumer(config, brokerList))
{
consumer.OnMessage += (obj, msg) =>
{
string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length);
Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
Console.WriteLine("1 Response: Partition {0},Offset {1} : {2}",
msg.Partition, msg.Offset, text);
if (!enableAutoCommit && msg.Offset % 10 == 0)
{
Console.WriteLine("Committing offset");
consumer.Commit(msg).Wait();
Console.WriteLine("Committed offset");
}
};
consumer.OnConsumerError += (obj, errorCode) =>
{
Console.WriteLine($"Consumer Error: {errorCode}");
};
consumer.OnEndReached += (obj, end) =>
{
Console.WriteLine($"Reached end of topic {end.Topic} partition {end.Partition}, next message will be at offset {end.Offset}");
};
consumer.OnError += (obj, error) =>
{
Console.WriteLine($"Error: {error.ErrorCode} {error.Reason}");
};
if (enableAutoCommit)
{
consumer.OnOffsetCommit += (obj, commit) =>
{
if (commit.Error != ErrorCode.NO_ERROR)
{
Console.WriteLine($"Failed to commit offsets: {commit.Error}");
}
Console.WriteLine($"Successfully committed offsets: [{string.Join(", ", commit.Offsets)}]");
};
}
consumer.OnPartitionsAssigned += (obj, partitions) =>
{
Console.WriteLine($"Assigned partitions: [{string.Join(", ", partitions)}], member id: {consumer.MemberId}");
consumer.Assign(partitions);
};
consumer.OnPartitionsRevoked += (obj, partitions) =>
{
Console.WriteLine($"Revoked partitions: [{string.Join(", ", partitions)}]");
consumer.Unassign();
};
consumer.OnStatistics += (obj, json) =>
{
Console.WriteLine($"Statistics: {json}");
};
consumer.Subscribe(topics);
consumer.Start();
Console.WriteLine($"Assigned to: [{string.Join(", ", consumer.Assignment)}]");
Console.WriteLine($"Subscribed to: [{string.Join(", ", consumer.Subscription)}]");
Console.WriteLine($"Started consumer, press enter to stop consuming");
Console.ReadLine();
}
}