static async Task Main(string[] args)
{
_logger = Core.Logger.Logger.BuildLogger();
var cts = new CancellationTokenSource();
_logger.Information("Consumindo mensagens do Kafka");
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true;
cts.Cancel();
};
var config = new ConsumerConfig
{
BootstrapServers = Constants.BootstrapServers,
GroupId = Constants.GroupId,
AutoOffsetReset = AutoOffsetReset.Earliest,
Acks = Acks.All,
EnableAutoCommit = true,
EnableAutoOffsetStore = false
};
try
{
using (var consumer = new ConsumerBuilder <Ignore, string>(config)
.SetStatisticsHandler((_, json) => _logger.Information($"Statistic: {json}"))
.SetLogHandler((_, log) => _logger.Information($"Log: {log}"))
.Build())
{
consumer.Subscribe(Constants.TopicName);
try
{
while (true)
{
var commitMessage = true;
var cr = consumer.Consume(cts.Token);
var message = JsonConvert.DeserializeObject <EmailMessage>(cr.Message.Value);
var topicPartitions = new List <TopicPartition>()
{
{ cr.TopicPartition }
};
try
{
var now = DateTime.Now;
_logger.Information($"===========================================================");
_logger.Information($"Hora atual: {now}");
if (DateTime.Compare(now, message.Timestamp) == -1)
{
var waitTime = ((TimeSpan)(message.Timestamp - now)).TotalMilliseconds;
_logger.Warning($"Hora que deve processar: {message.Timestamp}. Esperar: {waitTime} ms");
consumer.Pause(topicPartitions);
await Task.Delay(TimeSpan.FromMilliseconds(waitTime));
consumer.Resume(topicPartitions);
//commitMessage = await SendMessageToRetryTopic(message);
}
await ProcessMessage(message);
}
catch (MyKafkaException kex)
{
_logger.Error($"Erro interno: {kex.Message}");
_logger.Information("Enviando para retry topic.");
commitMessage = await SendMessageToRetryTopic(message, true);
}
finally
{
if (commitMessage)
{
consumer.StoreOffset(cr);
_logger.Information($"Mensagem '{message.Id}' commitada.");
//consumer.Pause(topicPartitions);
//_logger.Information($"Consumer paused. {cr.TopicPartition.Topic} / {cr.TopicPartition.Partition}");
//await Task.Delay(TimeSpan.FromSeconds(10));
//consumer.Resume(topicPartitions);
//_logger.Information($"Consumer resumed. {cr.TopicPartition.Topic} / {cr.TopicPartition.Partition}");
}
}
}
}
catch (OperationCanceledException)
{
consumer.Close();
}
}
}
catch (Exception ex)
{
_logger.Error($"Erro: {ex.Message}");
}
}
public void Consumer_Pause_Resume(string bootstrapServers)
{
LogToFile("start Consumer_Pause_Resume");
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
AutoOffsetReset = AutoOffsetReset.Latest
};
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
IEnumerable <TopicPartition> assignment = null;
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var producer = new ProducerBuilder <byte[], byte[]>(producerConfig).Build())
using (var consumer =
new ConsumerBuilder <byte[], byte[]>(consumerConfig)
.SetPartitionsAssignedHandler((c, partitions) => { assignment = partitions; })
.Build())
{
consumer.Subscribe(topic.Name);
while (assignment == null)
{
consumer.Consume(TimeSpan.FromSeconds(2));
}
ConsumeResult <byte[], byte[]> record = consumer.Consume(TimeSpan.FromSeconds(2));
Assert.Null(record);
producer.ProduceAsync(topic.Name, new Message <byte[], byte[]> {
Value = Serializers.Utf8.Serialize("test value", SerializationContext.Empty)
}).Wait();
record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record?.Message);
Assert.Equal(0, record?.Offset);
consumer.Pause(assignment);
producer.ProduceAsync(topic.Name, new Message <byte[], byte[]> {
Value = Serializers.Utf8.Serialize("test value 2", SerializationContext.Empty)
}).Wait();
record = consumer.Consume(TimeSpan.FromSeconds(2));
Assert.Null(record);
consumer.Resume(assignment);
record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record?.Message);
Assert.Equal(1, record?.Offset);
// check that these don't throw.
consumer.Pause(new List <TopicPartition>());
consumer.Resume(new List <TopicPartition>());
consumer.Close();
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Consumer_Pause_Resume");
}
public static void Consumer_Pause_Resume(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start Consumer_Pause_Resume");
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
AutoOffsetReset = AutoOffsetReset.Latest
};
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
IEnumerable <TopicPartition> assignment = null;
using (var producer = new ProducerBuilder <byte[], byte[]>(producerConfig).Build())
using (var consumer =
new ConsumerBuilder <byte[], byte[]>(consumerConfig)
.SetRebalanceHandler((c, e) =>
{
if (e.IsAssignment)
{
c.Assign(e.Partitions);
assignment = e.Partitions;
}
})
.Build())
{
consumer.Subscribe(singlePartitionTopic);
while (assignment == null)
{
consumer.Consume(TimeSpan.FromSeconds(10));
}
ConsumeResult <byte[], byte[]> record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Null(record);
producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> {
Value = Serializers.Utf8.Serialize("test value", true, null, null)
}).Wait();
record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record?.Message);
consumer.Pause(assignment);
producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> {
Value = Serializers.Utf8.Serialize("test value 2", true, null, null)
}).Wait();
record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Null(record);
consumer.Resume(assignment);
record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record?.Message);
// check that these don't throw.
consumer.Pause(new List <TopicPartition>());
consumer.Resume(new List <TopicPartition>());
consumer.Close();
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Consumer_Pause_Resume");
}