protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var kafkaConsumer = default(IConsumer <int, MyMessage>);
try
{
var config = new ConsumerConfig
{
BootstrapServers = string.Join(',', bootstrapServers),
// In order to guarantee only processed messages are committed, auto commit
// must be disabled.
EnableAutoCommit = false,
// The following settings are just for testing purposes to allow us to run the
// consumer multiple times without having to produce new messages every time.
GroupId = Guid.NewGuid().ToString(),
AutoOffsetReset = AutoOffsetReset.Earliest,
};
kafkaConsumer = new ConsumerBuilder <int, MyMessage>(config)
.SetValueDeserializer(new JsonSerializer <MyMessage>())
.SetPartitionsAssignedHandler(OnPartitionAssigned)
.SetPartitionsRevokedHandler(OnPartitionRevoked)
.Build();
kafkaConsumer.Subscribe("resilience-tests");
var consumer = new MyConsumer(kafkaConsumer, this.processor);
do
{
stoppingToken.ThrowIfCancellationRequested();
await consumer.ConsumeAsync(stoppingToken);
}while (true);
}
catch (OperationCanceledException)
{
try
{
kafkaConsumer?.Close();
}
catch
{
// Ignore close exceptions.
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
public static void Run(IServiceProvider serviceProvider)
{
// get exchange settings
var settings = serviceProvider.GetRequiredService <IOptions <ExchangeSettings> >().Value;
// get the user manager & email sender
var userManager = serviceProvider.GetRequiredService <UserManager <ApplicationUser> >();
var context = serviceProvider.GetRequiredService <ApplicationDbContext>();
var emailSender = serviceProvider.GetRequiredService <IEmailSender>();
var conf = new ConsumerConfig
{
GroupId = "order-updates-group",
BootstrapServers = settings.KafkaHost,
// Note: The AutoOffsetReset property determines the start offset in the event
// there are not yet any committed offsets for the consumer group for the
// topic/partitions of interest. By default, offsets are committed
// automatically, so in this example, consumption will only start from the
// earliest message in the topic 'my-topic' the first time you run the program.
AutoOffsetReset = AutoOffsetReset.Earliest,
};
using (var c = new ConsumerBuilder <Ignore, string>(conf).Build())
{
c.Subscribe(new string[] { OrdersTopic, DealsTopic });
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) => {
e.Cancel = true; // prevent the process from terminating.
cts.Cancel();
};
try
{
while (true)
{
try
{
var cr = c.Consume(cts.Token);
Console.WriteLine($"Consumed message at: '{cr.TopicPartitionOffset}'.");
switch (cr.Topic)
{
case OrdersTopic:
ProcessOrder(context, userManager, settings, emailSender, cr.Message.Value);
break;
case DealsTopic:
ProcessDeal(context, userManager, emailSender, cr.Message.Value);
break;
default:
break;
}
// commit queue offsets
c.Commit(cr);
}
catch (ConsumeException e)
{
Console.WriteLine($"Error occured: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
// Ensure the consumer leaves the group cleanly and final offsets are committed.
c.Close();
}
}
}
public void Consume(string bootstrapServers, string schemaRegistryUrl, string topicName)
{
var schemaRegistryRequestTimeoutMs = 30000;
var schemaRegistryMaxCachedSchemas = 1000;
var groupID = Guid.NewGuid().ToString();
var bufferBytes = 1024;
var autoRegisterSchema = true;
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
SchemaRegistryUrl = schemaRegistryUrl,
// optional schema registry client properties:
SchemaRegistryRequestTimeoutMs = schemaRegistryRequestTimeoutMs,
SchemaRegistryMaxCachedSchemas = schemaRegistryMaxCachedSchemas
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
AutoOffsetReset = AutoOffsetReset.Earliest,
GroupId = groupID // "Test" //Guid.NewGuid().ToString()
};
var avroSerializerConfig = new AvroSerializerConfig
{
// optional Avro serializer properties:
BufferBytes = bufferBytes,
AutoRegisterSchemas = autoRegisterSchema
};
NewConstructionAddressEvent addr = new NewConstructionAddressEvent();
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <string, NewConstructionAddressEvent>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(new AvroDeserializer <NewConstructionAddressEvent>(schemaRegistry).AsSyncOverAsync())
.SetErrorHandler((_, e) => logger.LogError($"Error: {e.Reason}"))
.Build())
{
try
{
logger.LogInformation($"Starting consumer.subscribe.");
consumer.Subscribe(topicName);
while (true)
{
try
{
logger.LogInformation($"Starting: consumer.Consume");
var consumeResult = consumer.Consume();
string k = consumeResult.Key;
logger.LogInformation($"BusMessage: {consumeResult.Message}, constructionAddressId: {consumeResult.Value.constructionAddressId}");
}
catch (OperationCanceledException)
{
logger.LogInformation($"OperationCancelled for consumer.Consume");
break;
}
catch (ConsumeException e)
{
logger.LogInformation(e, $"Consume error: {e.Error.Reason}");
break;
}
}
}
catch (Exception ex)
{
logger.LogError(ex, $"Consume error: {ex.Message}");
}
finally
{
consumer.Close();
}
}
}
public void Consume()
{
var config = new ConsumerConfig
{
BootstrapServers = _kafkaSettings.BrokerList,
GroupId = _kafkaSettings.GroupId, //"csharp-consumer",
EnableAutoCommit = false,
StatisticsIntervalMs = 5000,
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = true
};
const int commitPeriod = 5;
// Note: If a key or value deserializer is not set (as is the case below), the
// deserializer corresponding to the appropriate type from Confluent.Kafka.Deserializers
// will be used automatically (where available). The default deserializer for string
// is UTF8. The default deserializer for Ignore returns null for all input data
// (including non-null data).
using (var consumer = new ConsumerBuilder <Ignore, string>(config)
// Note: All handlers are called on the main .Consume thread.
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.SetStatisticsHandler((_, json) => Console.WriteLine($"Statistics: {json}"))
.SetPartitionsAssignedHandler((c, partitions) =>
{
Console.WriteLine($"Assigned partitions: [{string.Join(", ", partitions)}]");
// possibly manually specify start offsets or override the partition assignment provided by
// the consumer group by returning a list of topic/partition/offsets to assign to, e.g.:
//
// return partitions.Select(tp => new TopicPartitionOffset(tp, externalOffsets[tp]));
})
.SetPartitionsRevokedHandler((c, partitions) =>
{
Console.WriteLine($"Revoking assignment: [{string.Join(", ", partitions)}]");
})
.Build())
{
consumer.Subscribe(_kafkaSettings.Topics);
try
{
while (true)
{
try
{
var consumeResult = consumer.Consume(_kafkaSettings.CancellationToken);
if (consumeResult.IsPartitionEOF)
{
Console.WriteLine(
$"Reached end of topic {consumeResult.Topic}, partition {consumeResult.Partition}, offset {consumeResult.Offset}.");
continue;
}
Console.WriteLine($"Received message at {consumeResult.TopicPartitionOffset}: {consumeResult.Value}");
var consumed = ReceiveMessage?.Invoke(consumeResult.Value.ConvertToMessage(), _kafkaSettings.CancellationToken);
if ((consumed.HasValue && consumed.Value) &&
consumeResult.Offset % commitPeriod == 0)
{
// The Commit method sends a "commit offsets" request to the Kafka
// cluster and synchronously waits for the response. This is very
// slow compared to the rate at which the consumer is capable of
// consuming messages. A high performance application will typically
// commit offsets relatively infrequently and be designed handle
// duplicate messages in the event of failure.
try
{
consumer.Commit(consumeResult);
}
catch (KafkaException e)
{
Console.WriteLine($"Commit error: {e.Error.Reason}");
}
}
}
catch (ConsumeException e)
{
Console.WriteLine($"Consume error: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
Console.WriteLine("Closing consumer.");
consumer.Close();
}
}
}
public async Task Subscribe <T>(Action <T> callback) where T : IEvent
{
var conf = new ConsumerConfig
{
GroupId = $"{Dns.GetHostName()}-{Assembly.GetExecutingAssembly()}",
BootstrapServers = _parameters.GetStringParameter("KAFKA_BOOTSTRAP_SERVERS"),
AutoOffsetReset = AutoOffsetReset.Earliest
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = _parameters.GetStringParameter("KAFKA_SCHEMA_REGISTRY_URLS")
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var c = new ConsumerBuilder <Ignore, T>(conf)
.SetValueDeserializer(new AvroDeserializer <T>(schemaRegistry).AsSyncOverAsync())
.SetErrorHandler((_, e) => _logger.LogError($"Exception has occured when trying to consume an event : {e.Reason}"))
.Build())
{
c.Subscribe(GetTopicName <T>());
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) => {
e.Cancel = true; // prevent the process from terminating.
cts.Cancel();
};
await Task.Factory.StartNew(() =>
{
try
{
while (true)
{
try
{
var cr = c.Consume(cts.Token);
if (cr.Message.Value == null)
{
continue;
}
_logger.LogDebug($"Consumed message '{cr.Message.Value}' at: '{cr.TopicPartitionOffset}'.");
try
{
callback(cr.Message.Value);
}
catch (Exception e)
{
_logger.LogError($"Event {cr.Message.Value} was not processed \n\a" + e.GetFullMessage());
}
}
catch (ConsumeException e)
{
_logger.LogError($"Error occured: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
c.Close();
}
}, cts.Token);
}
}
public async Task <EventsData> GetSampleEvents(int seconds)
{
var config = new ConsumerConfig
{
BootstrapServers = _brokerList,
GroupId = _consumerGroup,
EnableAutoCommit = false,
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Latest
};
// Set the authentication for EventHub Kafka
if (_inputType == Constants.InputType_KafkaEventHub)
{
config.SecurityProtocol = SecurityProtocol.SaslSsl;
config.SaslMechanism = SaslMechanism.Plain;
config.SaslUsername = "******";
config.SaslPassword = _connectionString;
config.SslCaLocation = _cacertLocation;
}
StartTimer(seconds);
const int commitPeriod = 5;
using (var consumer = new ConsumerBuilder <Ignore, string>(config).Build())
{
EventsData eventsData = new EventsData();
consumer.Subscribe(_topics);
try
{
while (true)
{
try
{
var consumeResult = consumer.Consume(new TimeSpan(0, 0, seconds));
if (_timeout)
{
_logger.LogInformation($"Closing consumer");
consumer.Close();
return(await Task.FromResult(eventsData).ConfigureAwait(false));
}
if (consumeResult.IsPartitionEOF)
{
_logger.LogInformation($"Reached end of topic {consumeResult.Topic}, partition {consumeResult.Partition}, offset {consumeResult.Offset}.");
continue;
}
_logger.LogInformation($"Received message at {consumeResult.TopicPartitionOffset}: {consumeResult.Value}");
// Get raw data
EventRaw er = new EventRaw
{
Raw = consumeResult.Value,
Properties = new Dictionary <string, string>()
{
{ "HeadersCount", consumeResult.Headers.Count.ToString() }
},
};
// Set properties (using the Headers)
if (consumeResult.Headers != null && consumeResult.Headers.Count > 0)
{
for (int i = 0; i < consumeResult.Headers.Count; i++)
{
string key = consumeResult.Headers[i].Key;
string val = System.Text.Encoding.UTF8.GetString(consumeResult.Headers[i].GetValueBytes());
er.Properties.Add(key, val);
}
}
// Set the SystemProperties
er.SystemProperties.Add("Topic", consumeResult.Topic);
er.SystemProperties.Add("Partition", consumeResult.Partition.Value.ToString());
er.SystemProperties.Add("Offset", consumeResult.Offset.Value.ToString());
er.SystemProperties.Add("UtcDateTime", consumeResult.Timestamp.UtcDateTime.ToString());
er.SystemProperties.Add("UnixTimestampMs", consumeResult.Timestamp.UnixTimestampMs.ToString());
er.Json = JsonConvert.SerializeObject(er);
eventsData.EventsJson += er.Json + "\r\n";
eventsData.Events.Add(er);
if (consumeResult.Offset % commitPeriod == 0)
{
try
{
consumer.Commit(consumeResult);
}
catch (KafkaException e)
{
_logger.LogError($"Commit error: {e.Error.Reason}\n{e.ToString()}");
}
}
}
catch (ConsumeException e)
{
_logger.LogError($"Consume error: {e.Error.Reason}\n{e.ToString()}");
}
}
}
catch (OperationCanceledException e)
{
_logger.LogInformation($"Closing consumer");
consumer.Close();
return(await Task.FromResult(eventsData).ConfigureAwait(false));
}
}
}
private static void Main(string[] args)
{
var config = new ConsumerConfig
{
GroupId = "messages-consumers",
BootstrapServers = "localhost:9092",
AutoOffsetReset = AutoOffsetReset.Earliest
};
Console.Write("Would you like to see message content? Y/N: ");
var selection = Console.ReadLine();
var result = string.Empty;
switch (selection)
{
case "Y":
result = selection;
break;
case "N":
result = selection;
break;
default:
Console.WriteLine("Invalid input. Messages content no will be shown.");
result = "N";
break;
}
Console.WriteLine();
Console.WriteLine("Press Ctrl+C to exit.");
Console.WriteLine("Listening...");
Console.WriteLine();
using (var consumer = new ConsumerBuilder <Ignore, byte[]>(config).Build())
{
consumer.Subscribe(topicName);
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true;
cts.Cancel();
};
try
{
while (true)
{
try
{
var message = consumer.Consume(cts.Token);
if (result == "Y")
{
Console.WriteLine($"{message.Message.Timestamp.UtcDateTime.ToLocalTime()}: " +
$"{Serializer.DeserializeMessage(message.Message.Value)}");
Console.WriteLine();
}
else
{
Console.WriteLine($"{message.Message.Timestamp.UtcDateTime.ToLocalTime()}: message received.");
}
}
catch (ConsumeException e)
{
Console.WriteLine($"Error occured: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
consumer.Close();
}
}
Console.WriteLine("Press any key to exit...");
Console.ReadKey();
}
public static void Main(string[] args)
{
provider.FromXmlString(File.ReadAllText("C:\\Projects\\keys\\key.xml"));
//// buffer to write byte sequence of the given block_size
blockSize = (provider.KeySize / 8);
rijAlg.Mode = CipherMode.ECB;
rijAlg.Padding = PaddingMode.ISO10126;
var conf = new ConsumerConfig
{
GroupId = "test-consumer-group",
BootstrapServers = "localhost:9093",
StatisticsIntervalMs = 10000,
// QueuedMinMessages = 1000,
FetchWaitMaxMs = 5,
FetchErrorBackoffMs = 10,
EnableAutoCommit = true,
SocketReceiveBufferBytes = 1048576,
SecurityProtocol = SecurityProtocol.Ssl,
EnableSslCertificateVerification = true,
SslCaLocation = "C:\\Projects\\keys\\cert-signed",
// Debug="consumer,topic",
AutoCommitIntervalMs = 10000,
// Note: The AutoOffsetReset property determines the start offset in the event
// there are not yet any committed offsets for the consumer group for the
// topic/partitions of interest. By default, offsets are committed
// automatically, so in this example, consumption will only start from the
// earliest message in the topic 'my-topic' the first time you run the program.
// AutoOffsetReset = AutoOffsetReset.Earliest
};
var commitPeriod = 10; // in ms ( for manual commit operation )
ThreadPool.SetMinThreads(8, 5);
ThreadPool.SetMaxThreads(10, 10);
using (var c = new ConsumerBuilder <Ignore, string>(conf)
.SetStatisticsHandler((_, json) => {
if (totalMessages > 0 && !isTestingInProgress)
{
var ellapsedTime = ((statisticsEndTime < lastMessageReceivedTime ? lastMessageReceivedTime : statisticsEndTime) - statisticsStartTime).TotalMilliseconds;
Console.WriteLine($"Statistics: Aggregate time ellapsed - {totalEllapsed}ms, Total Messages - {totalMessages}, Average response - {totalEllapsed / totalMessages}ms");
Console.WriteLine($"Statistics: Total time elapsed - {ellapsedTime}, Throughput - {totalMessages / (ellapsedTime / 1000)} messages/sec.,");
Console.WriteLine($"Statistics: Total messages above {maxDelayThreshold}ms - {totalMessageAboveThreshold}");
Console.WriteLine($"Last mesasge received time - {lastMessageReceivedTime}, Statistics ended time - {statisticsEndTime}");
Console.WriteLine($"Maximum delay - {maxDelay}ms");
totalEllapsed = 0;
totalMessages = 0;
totalMessageAboveThreshold = 0;
maxDelay = 0;
}
})
.Build())
{
c.Subscribe("my-topic");
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true; // prevent the process from terminating.
cts.Cancel();
};
try
{
while (true)
{
try
{
var cr = c.Consume(cts.Token);
if (cr.IsPartitionEOF)
{
Console.WriteLine(
$"Reached end of topic {cr.Topic}, partition {cr.Partition}, offset {cr.Offset}.");
continue;
}
var data = cr.Message.Value;
ThreadPool.QueueUserWorkItem(ThreadExecutionHandler, data);
lastMessageReceivedTime = DateTime.Now;
//ThreadExecutionHandler(data);
// Console.WriteLine($"Consumed message at: '{cr.TopicPartitionOffset}'.");
//if (cr.Offset % commitPeriod == 0)
//{
// try
// {
// c.Commit(cr);
// }
// catch (Exception ex)
// {
// Console.WriteLine(ex);
// }
//}
}
catch (ConsumeException e)
{
Console.WriteLine($"Error occured: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
// Ensure the consumer leaves the group cleanly and final offsets are committed.
c.Close();
}
}
}
protected void Run(string topics, int concurrence, CancellationToken cancellationToken)
{
int nPart = -1;
for (int i = 0; i < concurrence; i++)
{
var ts = new ThreadStart(() =>
{
using (var consumer = new ConsumerBuilder <Ignore, string>(consumerConfig).Build())
{
consumer.Subscribe(topics);
while (!cancellationToken.IsCancellationRequested)
{
ConsumeResult <Ignore, string> consumeResult = null;
try
{
consumeResult = consumer.Consume(1000);
// Loggar partições conectadas
if (consumer.Assignment.Count != nPart)
{
nPart = consumer.Assignment.Count;
consumer.Assignment.ForEach((tp) =>
{
logger.LogDebug($"Assinado para partição [{tp.Partition}] para Thread [{Thread.CurrentThread.ManagedThreadId}]");
});
}
if (consumeResult == null)
{
continue;
}
this.Handler(consumeResult);
}
catch (Exception ex)
{
logger.LogError(ex.Message);
//Retry
//DLQ
Thread.Sleep(1000);
}
finally
{
if (consumeResult != null)
{
consumer.Commit(consumeResult);
}
}
}
consumer.Close();
}
});
new Thread(ts).Start();
}
}
public override async Task Read(Dictionary <string, string> properties)
{
var groupId = EndPointConfig.Properties["GroupId"];
var servers = EndPointConfig.Properties["BootstrapServers"];
var topic = EndPointConfig.Properties["Topic"];
var topics = new List <string>()
{
topic
};
var config = new ConsumerConfig
{
BootstrapServers = servers,
GroupId = groupId,
EnableAutoCommit = false,
StatisticsIntervalMs = 5000,
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = true
};
const int commitPeriod = 5;
// Note: If a key or value deserializer is not set (as is the case below), the
// deserializer corresponding to the appropriate type from Confluent.Kafka.Deserializers
// will be used automatically (where available). The default deserializer for string
// is UTF8. The default deserializer for Ignore returns null for all input data
// (including non-null data).
using (var consumer = new ConsumerBuilder <Ignore, string>(config)
// Note: All handlers are called on the main .Consume thread.
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.SetStatisticsHandler((_, json) => Console.WriteLine($"Statistics: {json}"))
.SetPartitionsAssignedHandler((c, partitions) =>
{
_logger.Info($"Assigned partitions: [{0}]", string.Join(", ", partitions));
// possibly manually specify start offsets or override the partition assignment provided by
// the consumer group by returning a list of topic/partition/offsets to assign to, e.g.:
//
// return partitions.Select(tp => new TopicPartitionOffset(tp, externalOffsets[tp]));
})
.SetPartitionsRevokedHandler((c, partitions) =>
{
_logger.Info($"Revoking assignment: [{0}]", string.Join(", ", partitions));
})
.Build())
{
consumer.Subscribe(topics);
try
{
while (!Token.IsCancellationRequested)
{
try
{
var consumeResult = consumer.Consume(Token);
if (consumeResult.IsPartitionEOF)
{
_logger.Info($"Reached end of topic {0}, partition {1}, offset {2}.",
consumeResult.Topic, consumeResult.Partition, consumeResult.Offset);
continue;
}
_logger.Info($"Received message at {0}: {1}", consumeResult.TopicPartitionOffset, consumeResult.Value);
var nameValues = (Dictionary <string, string>)JsonConvert.DeserializeObject(consumeResult.Value, typeof(Dictionary <string, string>));
var entityCollection = new EntityCollection();
foreach (var key in nameValues.Keys)
{
entityCollection.Entities.Add(key, nameValues[key]);
}
Data.Add(entityCollection);
await SignalHandler(new Dictionary <string, string>());
if (consumeResult.Offset % commitPeriod == 0)
{
// The Commit method sends a "commit offsets" request to the Kafka
// cluster and synchronously waits for the response. This is very
// slow compared to the rate at which the consumer is capable of
// consuming messages. A high performance application will typically
// commit offsets relatively infrequently and be designed handle
// duplicate messages in the event of failure.
try
{
consumer.Commit(consumeResult);
}
catch (KafkaException e)
{
_logger.Error($"Commit error: {0}", e.Error.Reason);
}
}
}
catch (ConsumeException e)
{
_logger.Error($"Consume error: {0}", e.Error.Reason);
}
}
}
catch (OperationCanceledException)
{
_logger.Debug("Closing consumer.");
consumer.Close();
}
}
}
public static void ProduceConsume(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = true
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
SchemaRegistryUrl = schemaRegistryServers
};
var adminClientConfig = new AdminClientConfig
{
BootstrapServers = bootstrapServers
};
string topic = Guid.NewGuid().ToString();
using (var adminClient = new AdminClientBuilder(adminClientConfig).Build())
{
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topic, NumPartitions = 1, ReplicationFactor = 1
}
}).Wait();
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, User>(producerConfig)
.SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
.SetValueSerializer(new AvroSerializer <User>(schemaRegistry))
.Build())
{
for (int i = 0; i < 100; ++i)
{
var user = new User
{
name = i.ToString(),
favorite_number = i,
favorite_color = "blue"
};
producer
.ProduceAsync(topic, new Message <string, User> {
Key = user.name, Value = user
})
.Wait();
}
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <string, User>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(new AvroDeserializer <User>(schemaRegistry).AsSyncOverAsync())
.SetErrorHandler((_, e) => Assert.True(false, e.Reason))
.Build())
{
consumer.Subscribe(topic);
int i = 0;
while (true)
{
var record = consumer.Consume(TimeSpan.FromMilliseconds(100));
if (record == null)
{
continue;
}
if (record.IsPartitionEOF)
{
break;
}
Assert.Equal(i.ToString(), record.Message.Key);
Assert.Equal(i.ToString(), record.Message.Value.name);
Assert.Equal(i, record.Message.Value.favorite_number);
Assert.Equal("blue", record.Message.Value.favorite_color);
i += 1;
}
Assert.Equal(100, i);
consumer.Close();
}
}
private void Subscribe()
{
if (init == 1)
{
return;
}
if (_option.ConsumerConfig == null)
{
throw new ArgumentNullException(nameof(_option.ConsumerConfig), nameof(KafkaOptions));
}
if (Interlocked.Exchange(ref init, 1) == 1)
{
return;
}
try
{
_log.LogInformation($"Subscribe to topic {_option.ResponseTopic}");
Task.Run(async() =>
{
using var consumer = new ConsumerBuilder <byte[], byte[]>(_option.ConsumerConfig).Build();
consumer.Subscribe(_option.ResponseTopic);
try
{
while (true)
{
var message = consumer.Consume(_cancellationTokenSource.Token);
try
{
await Handle(message);
}
catch (Exception e)
{
_log.LogError(e, $"Unprocessed exception,topic: {message.Topic}, partition: {message.Partition.Value} offset: {message.Offset.Value}");
}
}
}
catch (OperationCanceledException e)
{
_log.LogInformation(e, "Kafka client cancelled. Close consumer");
consumer.Close();
}
}, _cancellationTokenSource.Token).ContinueWith(task =>
{
if (task.Exception != null)
{
_log.LogError(task.Exception, "Error subscribe kafka ");
}
else
{
_log.LogInformation($"Kafka client subscribe started. {task.Status}");
}
}, _cancellationTokenSource.Token);
}
catch (Exception)
{
Interlocked.Exchange(ref init, 0);
throw;
}
}
public override void Run(string brokerList, List <string> topics, CancellationToken cancellationToken)
{
var config = new ConsumerConfig
{
// the group.id property must be specified when creating a consumer, even
// if you do not intend to use any consumer group functionality.
GroupId = new Guid().ToString(),
BootstrapServers = brokerList,
// partition offsets can be committed to a group even by consumers not
// subscribed to the group. in this example, auto commit is disabled
// to prevent this from occurring.
EnableAutoCommit = true
};
Stopwatch sw = new Stopwatch();
sw.Start();
Console.WriteLine($"sw.ElapsedMilliseconds start");
using (var consumer =
new ConsumerBuilder <int, string>(config)
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.Build())
{
consumer.Assign(topics.Select(topic => new TopicPartitionOffset(topic, 0, Offset.Beginning)).ToList());
try
{
while (true)
{
try
{
var result = consumer.ConsumeBatch <int, string>(1000, 10);
if (result.Messages.Count > 0)
{
var connString = Utility.GetConnectionString("ConnectionStrings:DefaultConnectionEF");
using var conn = new NpgsqlConnection(connString);
conn.Open();
var valuesTableSql = string.Join(",", Enumerable.Range(0, result.Messages.Count).Select(i => $"(@p1{i}, @p2{i},@p3{i},@p4{i},@p5{i})"));
using (var tx = conn.BeginTransaction())
{
using (var cmd = new NpgsqlCommand($"INSERT INTO \"Messages\" (\"Id\", \"Created\", \"Content\", \"IsReceived\", \"ReceivedTimestamp\") VALUES {valuesTableSql};", conn))
{
for (int i = 0; i < result.Messages.Count; ++i)
{
cmd.Parameters.AddWithValue($"p1{i}", result.Messages.ElementAt(i).Key);
cmd.Parameters.AddWithValue($"p2{i}", DateTime.Now);
cmd.Parameters.AddWithValue($"p3{i}", result.Messages.ElementAt(i).Value);
cmd.Parameters.AddWithValue($"p4{i}", false);
cmd.Parameters.AddWithValue($"p5{i}", DateTime.Now);
}
cmd.ExecuteNonQuery();
}
tx.Commit();
}
Console.WriteLine($"after database: {sw.ElapsedMilliseconds}, MC: {result.MessagesConsumed}, WM: {result.BatchWindowMilliseconds}");
}
}
catch (ConsumeException e)
{
Console.WriteLine($"Consume error: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
Console.WriteLine("Closing consumer.");
consumer.Close();
}
}
sw.Stop();
Console.WriteLine($"sw.ElapsedMilliseconds: {sw.ElapsedMilliseconds}");
}
void Listen()
{
var config = new ConsumerConfig
{
BootstrapServers = BrokerList,
GroupId = GroupId, //"csharp-consumer",
EnableAutoCommit = false,
StatisticsIntervalMs = 5000,
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = true,
// A good introduction to the CooperativeSticky assignor and incremental rebalancing:
// https://www.confluent.io/blog/cooperative-rebalancing-in-kafka-streams-consumer-ksqldb/
PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky
};
const int commitPeriod = 1;
// Note: If a key or value deserializer is not set (as is the case below), the
// deserializer corresponding to the appropriate type from Confluent.Kafka.Deserializers
// will be used automatically (where available). The default deserializer for string
// is UTF8. The default deserializer for Ignore returns null for all input data
// (including non-null data).
using (var consumer = new ConsumerBuilder <Ignore, string>(config)
// Note: All handlers are called on the main .Consume thread.
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
//.SetStatisticsHandler((_, json) => Console.WriteLine($"Statistics: {json}"))
.SetPartitionsAssignedHandler((c, partitions) =>
{
// Since a cooperative assignor (CooperativeSticky) has been configured, the
// partition assignment is incremental (adds partitions to any existing assignment).
Console.WriteLine($"Incremental partition assignment: [{string.Join(", ", partitions)}]");
// Possibly manually specify start offsets by returning a list of topic/partition/offsets
// to assign to, e.g.:
// return partitions.Select(tp => new TopicPartitionOffset(tp, externalOffsets[tp]));
})
.SetPartitionsRevokedHandler((c, partitions) =>
{
// Since a cooperative assignor (CooperativeSticky) has been configured, the revoked
// assignment is incremental (may remove only some partitions of the current assignment).
Console.WriteLine($"Incremental partition revokation: [{string.Join(", ", partitions)}]");
})
.SetPartitionsLostHandler((c, partitions) =>
{
// The lost partitions handler is called when the consumer detects that it has lost ownership
// of its assignment (fallen out of the group).
Console.WriteLine($"Partitions were lost: [{string.Join(", ", partitions)}]");
})
.Build())
{
consumer.Subscribe(Topics);
try
{
while (true)
{
try
{
var consumeResult = consumer.Consume(CancellationToken);
if (consumeResult.IsPartitionEOF)
{
Console.WriteLine(
$"Reached end of topic {consumeResult.Topic}, partition {consumeResult.Partition}, offset {consumeResult.Offset}.");
continue;
}
OnMessageReceived(consumeResult.Message.Value);
Console.WriteLine($"Received message at {consumeResult.TopicPartitionOffset}: {consumeResult.Message.Value}");
if (consumeResult.Offset % commitPeriod == 0)
{
// The Commit method sends a "commit offsets" request to the Kafka
// cluster and synchronously waits for the response. This is very
// slow compared to the rate at which the consumer is capable of
// consuming messages. A high performance application will typically
// commit offsets relatively infrequently and be designed handle
// duplicate messages in the event of failure.
try
{
consumer.Commit(consumeResult);
}
catch (KafkaException e)
{
Console.WriteLine($"Commit error: {e.Error.Reason}");
}
}
}
catch (ConsumeException e)
{
Console.WriteLine($"Consume error: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
Console.WriteLine("Closing consumer.");
consumer.Close();
}
}
}
public Task Process(CancellationToken cancellationToken)
{
const int commitPeriod = 5;
var config = new ConsumerConfig {
BootstrapServers = "127.0.0.1:9092", GroupId = "test-consumer"
};
// Create the consumer
using (var consumer = new ConsumerBuilder <Null, string>(config).Build())
{
Console.WriteLine($"Subscribing to Kafka Topic {"testtopic"}");
// Subscribe to the Kafka topic
consumer.Subscribe(new List <string>()
{
"testtopic"
});
try
{
while (true)
{
try
{
var consumeResult = consumer.Consume(cancellationToken);
if (consumeResult.IsPartitionEOF)
{
Console.WriteLine(
$"Reached end of topic {consumeResult.Topic}, partition {consumeResult.Partition}, offset {consumeResult.Offset}.");
continue;
}
Console.WriteLine($"Received message at {consumeResult.TopicPartitionOffset}: {consumeResult.Message.Value}");
if (consumeResult.Offset % commitPeriod == 0)
{
try
{
consumer.Commit(consumeResult);
}
catch (KafkaException e)
{
Console.WriteLine($"Commit error: {e.Error.Reason}");
}
}
}
catch (ConsumeException e)
{
Console.WriteLine($"Consume error: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
Console.WriteLine("Closing consumer.");
consumer.Close();
}
}
return(Task.Run(() => true));
}
public static void Consumer_AutoCommit(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start Consumer_AutoCommit");
int N = 2;
var firstProduced = Util.ProduceNullStringMessages(bootstrapServers, singlePartitionTopic, 100, N);
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000,
AutoCommitIntervalMs = 1000,
EnableAutoCommit = false,
EnablePartitionEof = true
};
using (var consumer =
new ConsumerBuilder <Null, string>(consumerConfig)
.SetRebalanceHandler((c, e) =>
{
if (e.IsAssignment)
{
Assert.Single(e.Partitions);
c.Assign(new TopicPartitionOffset(singlePartitionTopic, firstProduced.Partition, firstProduced.Offset));
}
})
.Build())
{
consumer.Subscribe(singlePartitionTopic);
int msgCnt = 0;
while (true)
{
var record = consumer.Consume(TimeSpan.FromMilliseconds(100));
if (record == null)
{
continue;
}
if (record.IsPartitionEOF)
{
break;
}
msgCnt += 1;
}
Assert.Equal(msgCnt, N);
Thread.Sleep(TimeSpan.FromSeconds(3));
var committed = consumer.Committed(new [] { new TopicPartition(singlePartitionTopic, 0) }, TimeSpan.FromSeconds(10));
// if this was committing, would expect the committed offset to be first committed offset + N
// (don't need to subtract 1 since the next message to be consumed is the value that is committed).
Assert.NotEqual(firstProduced.Offset + N, committed[0].Offset);
consumer.Close();
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Consumer_AutoCommit");
}
static void Main(string[] args)
{
Console.WriteLine($"PatientService({instanceId}) - starting");
var conf = new ConsumerConfig
{
GroupId = consumer_group,
BootstrapServers = kafkaBrokers,
// Note: The AutoOffsetReset property determines the start offset in the event
// there are not yet any committed offsets for the consumer group for the
// topic/partitions of interest. By default, offsets are committed
// automatically, so in this example, consumption will only start from the
// earliest message in the topic 'my-topic' the first time you run the program.
AutoOffsetReset = AutoOffsetReset.Earliest
};
var jsonOptions = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
};
try
{
using (var consumer = new ConsumerBuilder <Ignore, string>(conf).Build())
{
consumer.Subscribe(topic);
try
{
while (true)
{
try
{
var response = consumer.Consume();
var rxEvent = JsonSerializer.Deserialize(response.Value, typeof(RxPrescribedEvent), jsonOptions) as RxPrescribedEvent;
if (rxEvent.Medication == null)
{
Console.WriteLine("***Throwing away junk ***");
continue;
}
Console.WriteLine($"PatientService({instanceId}) - Medication {rxEvent.Medication.DrugName} prescribed notification sent to patient {rxEvent.Patient.LastName}, {rxEvent.Patient.FirstName}\n");
}
catch (ConsumeException e)
{
Console.WriteLine($"PatientService({instanceId}) - Event consuming error occured: {e.Error.Reason}");
}
catch (JsonException e)
{
Console.WriteLine($"PatientService({instanceId}) - JSON error occured: {e.Message}");
}
}
}
catch (OperationCanceledException)
{
// Ensure the consumer leaves the group cleanly and final offsets are committed.
consumer.Close();
}
}
}
catch (Exception ex)
{
Console.WriteLine($"PatientService({instanceId}) - unhandled exception: {ex}");
}
}
public async Task KafkaConsumer()
{
List <string> topicListFromDB = chatsService.GetTopics();
ConsumerConfig config = JsonConvert.DeserializeObject <ConsumerConfig>(System.IO.File.ReadAllText(Constant.ConsumerConfigFilePath));
var topicListFromServer = AdminClient.RemoveAdminTopics(AdminClient.getMetadata(Constant.BrokerIP));
foreach (string topic in topicListFromServer)
{
if (!topicListFromDB.Contains(topic))
{
Chat chat = new Chat(topic);
chatsService.Create(chat);
}
}
using (var consumer = new ConsumerBuilder <Ignore, byte[]>(config).Build())
{
string fileName = "";
consumer.Subscribe(topicListFromServer);
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true; // prevent the process from terminating.
cts.Cancel();
};
try
{
while (true)
{
try
{
var cr = consumer.Consume(cts.Token);
Console.WriteLine(cr.Topic);
// store the image
if (cr.Topic.Equals("FileContent"))
{
if (fileName != "")
{
Console.WriteLine(cr.Value);
FileStream myFile = System.IO.File.Open(@B3Image + fileName, FileMode.Create, FileAccess.Write);
BinaryWriter myWriter = new BinaryWriter(myFile);
myWriter.Write(cr.Value);
myWriter.Close();
myFile.Close();
fileName = "";
}
}
else if (cr.Topic.Equals("FileName"))
{
fileName = Encoding.Default.GetString(cr.Value);
}
else
{
var incomming = new Chat();
try
{
ChatLine chatPackage = JsonConvert.DeserializeObject <List <ChatLine> >(Encoding.Default.GetString(cr.Value)).First();
incomming = new Chat(cr.Topic, chatPackage);
}
catch (Exception)
{
Console.WriteLine("Deserialize failed, check MongoDB if it is delivered from CLI.");
ChatLine chatPackage = new ChatLine(Encoding.Default.GetString(cr.Value), "CLTestUser");
incomming = new Chat(cr.Topic, chatPackage);
}
if (topicListFromDB.Contains(cr.Topic))
{
//Add new line to DB
chatsService.UpdateByTopic(cr.Topic, incomming);
}
else
{
//create new Topic
chatsService.Create(incomming);
topicListFromDB.Add(cr.Topic);
}
var allChats = JsonConvert.SerializeObject
(
chatsService.Get(),
new JsonSerializerSettings {
ContractResolver = new CamelCasePropertyNamesContractResolver()
}
);
await chatHub.SendMessage(allChats);
}
}
catch (ConsumeException e)
{
Console.WriteLine($"Error occured: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
// Ensure the consumer leaves the group cleanly and final offsets are committed.
consumer.Close();
}
}
}
public void Subscribe(Action <T> onchange, CacheNotifyAction cacheNotifyAction)
{
if (ClientConfig == null)
{
MemoryCacheNotify.Subscribe(onchange, cacheNotifyAction);
return;
}
var channelName = GetChannelName(cacheNotifyAction);
Cts[channelName] = new CancellationTokenSource();
Actions[channelName] = onchange;
void action()
{
var conf = new ConsumerConfig(ClientConfig)
{
GroupId = Guid.NewGuid().ToString()
};
using var c = new ConsumerBuilder <AscCacheItem, T>(conf)
.SetErrorHandler((_, e) => Log.Error(e))
.SetKeyDeserializer(KeyDeserializer)
.SetValueDeserializer(ValueDeserializer)
.Build();
c.Assign(new TopicPartition(channelName, new Partition()));
try
{
while (true)
{
try
{
var cr = c.Consume(Cts[channelName].Token);
if (cr != null && cr.Value != null && !(new Guid(cr.Key.Id.ToByteArray())).Equals(Key) && Actions.TryGetValue(channelName, out var act))
{
try
{
act(cr.Value);
}
catch (Exception e)
{
Log.Error("Kafka onmessage", e);
}
}
}
catch (ConsumeException e)
{
Log.Error(e);
}
}
}
catch (OperationCanceledException)
{
c.Close();
}
}
var task = new Task(action, TaskCreationOptions.LongRunning);
task.Start();
}
static void Main(string[] args)
{
var config = new ConsumerConfig
{
GroupId = "re",
BootstrapServers = "192.168.3.10:9092",
AutoOffsetReset = AutoOffsetReset.Earliest
};
using (var consumer = new ConsumerBuilder <Ignore, string>(config).Build())
{
consumer.Subscribe("reserva");
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) => {
e.Cancel = true;
cts.Cancel();
};
try
{
while (true)
{
try
{
var cr = consumer.Consume(cts.Token);
Console.WriteLine(cr.Value);
OrdenCompraModel ordenCompra = JsonConvert.DeserializeObject <OrdenCompraModel>(cr.Value);
ordenCompra.sEstado = "reservado";
ProductoDAO productoDAO = new ProductoDAO();
productoDAO.actualizarStock(ordenCompra.lDetalleCompra);
OrdenCompraDAO ordenCompraDAO = new OrdenCompraDAO();
ordenCompraDAO.registrarOrdenCompra(ordenCompra);
var config2 = new ProducerConfig {
BootstrapServers = "192.168.3.10:9092"
};
Action <DeliveryReport <Null, string> > handler = r =>
Console.WriteLine(!r.Error.IsError
? $"Delivered message to {r.TopicPartitionOffset}"
: $"Delivery Error: {r.Error.Reason}");
using (var producer = new ProducerBuilder <Null, string>(config2).Build())
{
producer.ProduceAsync("factura", new Message <Null, string> {
Value = JsonConvert.SerializeObject(ordenCompra)
});
producer.Flush(TimeSpan.FromSeconds(10));
}
}
catch (ConsumeException e)
{
Console.WriteLine($"Error occured: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
consumer.Close();
}
}
}
static async Task Main(string[] args)
{
if (args.Length != 3)
{
Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName");
return;
}
string bootstrapServers = args[0];
string schemaRegistryUrl = args[1];
string topicName = args[2];
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
// Note: you can specify more than one schema registry url using the
// schema.registry.url property for redundancy (comma separated list).
// The property name is not plural to follow the convention set by
// the Java implementation.
Url = schemaRegistryUrl,
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = "protobuf-example-consumer-group"
};
CancellationTokenSource cts = new CancellationTokenSource();
var consumeTask = Task.Run(() =>
{
using (var consumer =
new ConsumerBuilder <string, User>(consumerConfig)
.SetValueDeserializer(new ProtobufDeserializer <User>().AsSyncOverAsync())
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.Build())
{
consumer.Subscribe(topicName);
try
{
while (true)
{
try
{
var consumeResult = consumer.Consume(cts.Token);
Console.WriteLine($"user name: {consumeResult.Message.Key}, favorite color: {consumeResult.Message.Value.FavoriteColor}");
}
catch (ConsumeException e)
{
Console.WriteLine($"Consume error: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
consumer.Close();
}
}
});
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, User>(producerConfig)
.SetValueSerializer(new ProtobufSerializer <User>(schemaRegistry))
.Build())
{
Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit.");
int i = 0;
string text;
while ((text = Console.ReadLine()) != "q")
{
User user = new User {
Name = text, FavoriteColor = "green", FavoriteNumber = i++
};
await producer
.ProduceAsync(topicName, new Message <string, User> {
Key = text, Value = user
})
.ContinueWith(task => task.IsFaulted
? $"error producing message: {task.Exception.Message}"
: $"produced to: {task.Result.TopicPartitionOffset}");
}
}
cts.Cancel();
}
public void Consumer_Poll_Error(string bootstrapServers)
{
LogToFile("start Consumer_Poll_Error");
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
TopicPartitionOffset firstProduced = null;
using (var producer = new ProducerBuilder <byte[], byte[]>(producerConfig).Build())
{
var keyData = Encoding.UTF8.GetBytes("key");
firstProduced = producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> {
Key = keyData
}).Result.TopicPartitionOffset;
var valData = Encoding.UTF8.GetBytes("val");
producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> {
Value = valData
});
Assert.True(producer.Flush(TimeSpan.FromSeconds(10)) == 0);
}
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000,
EnablePartitionEof = true
};
// test key deserialization error behavior
using (var consumer =
new ConsumerBuilder <Null, string>(consumerConfig)
.SetRebalanceHandler((c, e) =>
{
if (e.IsAssignment)
{
Assert.Single(e.Partitions);
Assert.Equal(firstProduced.TopicPartition, e.Partitions[0]);
c.Assign(e.Partitions.Select(p => new TopicPartitionOffset(p, firstProduced.Offset)));
}
else
{
c.Unassign();
}
})
.Build())
{
consumer.Subscribe(singlePartitionTopic);
int msgCnt = 0;
int errCnt = 0;
while (true)
{
var s = consumer.Subscription;
try
{
var record = consumer.Consume(TimeSpan.FromSeconds(10));
if (record == null)
{
continue;
}
if (record.IsPartitionEOF)
{
break;
}
msgCnt += 1;
}
catch (ConsumeException e)
{
errCnt += 1;
Assert.Equal(ErrorCode.Local_KeyDeserialization, e.Error.Code);
Assert.Equal(firstProduced.Offset.Value, e.ConsumerRecord.Offset.Value);
}
}
Assert.Equal(1, msgCnt);
Assert.Equal(1, errCnt);
consumer.Close();
}
// test value deserialization error behavior.
using (var consumer =
new ConsumerBuilder <string, Null>(consumerConfig)
.SetRebalanceHandler((c, e) =>
{
if (e.IsAssignment)
{
Assert.Single(e.Partitions);
Assert.Equal(firstProduced.TopicPartition, e.Partitions[0]);
c.Assign(e.Partitions.Select(p => new TopicPartitionOffset(p, firstProduced.Offset)));
}
else
{
c.Unassign();
}
})
.Build())
{
consumer.Subscribe(singlePartitionTopic);
int msgCnt = 0;
int errCnt = 0;
while (true)
{
try
{
var record = consumer.Consume(TimeSpan.FromSeconds(10));
if (record == null)
{
continue;
}
if (record.IsPartitionEOF)
{
break;
}
msgCnt += 1;
}
catch (ConsumeException e)
{
errCnt += 1;
Assert.Equal(ErrorCode.Local_ValueDeserialization, e.Error.Code);
Assert.Equal(firstProduced.Offset.Value + 1, e.ConsumerRecord.Offset.Value);
}
}
Assert.Equal(1, msgCnt);
Assert.Equal(1, errCnt);
consumer.Close();
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Consumer_Poll_Error");
}
private static void Consume()
{
var conf = new ConsumerConfig
{
GroupId = "test-consumer-group",
BootstrapServers = "omnibus-01.srvs.cloudkafka.com:9094,omnibus-02.srvs.cloudkafka.com:9094,omnibus-03.srvs.cloudkafka.com:9094",
SaslUsername = "******",
SaslPassword = "******",
SecurityProtocol = SecurityProtocol.SaslSsl,
SaslMechanism = SaslMechanism.ScramSha256,
AutoOffsetReset = AutoOffsetReset.Latest,
EnableAutoCommit = true
};
var counter = 0;
using (var c = new ConsumerBuilder <Null, User>(conf)
.SetValueDeserializer(new CustomDeserializer <User>())
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.Build())
{
c.Subscribe("ynm8ml8b-HelloWorld");
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true; // prevent the process from terminating.
cts.Cancel();
};
try
{
while (true)
{
var cr = new ConsumeResult <Null, User>();
try
{
cr = c.Consume(cts.Token);
counter++;
Console.WriteLine($"Counter:{counter}");
if (cr.Message != null)
{
for (int i = 0; i < cr.Message.Headers.Count; i++)
{
var key = cr.Message.Headers[i].Key;
var value = Encoding.UTF8.GetString(cr.Message.Headers[i].GetValueBytes());
Console.WriteLine($"Chave:{key} || Valor:{value}");
}
Console.WriteLine($"Consumed message id '{JsonConvert.SerializeObject(cr.Message.Value)}' at: '{cr.TopicPartitionOffset}'.");
}
else
{
cr.Message = new Message <Null, User>();
}
}
catch (ConsumeException e)
{
Console.WriteLine($"Error occured: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
// Ensure the consumer leaves the group cleanly and final offsets are committed.
c.Close();
}
}
}
static async Task Main(string[] args)
{
if (args.Length != 3)
{
Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName");
return;
}
string bootstrapServers = args[0];
string schemaRegistryUrl = args[1];
string topicName = args[2];
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
// Note: you can specify more than one schema registry url using the
// schema.registry.url property for redundancy (comma separated list).
// The property name is not plural to follow the convention set by
// the Java implementation.
Url = schemaRegistryUrl
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = "json-example-consumer-group"
};
// Note: Specifying json serializer configuration is optional.
var jsonSerializerConfig = new JsonSerializerConfig
{
BufferBytes = 100
};
CancellationTokenSource cts = new CancellationTokenSource();
var consumeTask = Task.Run(() =>
{
using (var consumer =
new ConsumerBuilder <Null, Person>(consumerConfig)
.SetValueDeserializer(new JsonDeserializer <Person>().AsSyncOverAsync())
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.Build())
{
consumer.Subscribe(topicName);
try
{
while (true)
{
try
{
var cr = consumer.Consume(cts.Token);
Console.WriteLine($"Name: {cr.Message.Value.FirstName} {cr.Message.Value.LastName}, age: {cr.Message.Value.Age}");
}
catch (ConsumeException e)
{
Console.WriteLine($"Consume error: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
consumer.Close();
}
}
});
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <Null, Person>(producerConfig)
.SetValueSerializer(new JsonSerializer <Person>(schemaRegistry, jsonSerializerConfig))
.Build())
{
Console.WriteLine($"{producer.Name} producing on {topicName}. Enter first names, q to exit.");
int i = 0;
string text;
while ((text = Console.ReadLine()) != "q")
{
Person person = new Person {
FirstName = text, LastName = "lastname", Age = i++ % 150
};
await producer
.ProduceAsync(topicName, new Message <Null, Person> {
Value = person
})
.ContinueWith(task => task.IsFaulted
? $"error producing message: {task.Exception.Message}"
: $"produced to: {task.Result.TopicPartitionOffset}");
}
}
cts.Cancel();
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
{
// Note: a subject name strategy was not configured, so the default "Topic" was used.
var schema = await schemaRegistry.GetLatestSchemaAsync(SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName));
Console.WriteLine("\nThe JSON schema corresponding to the written data:");
Console.WriteLine(schema.SchemaString);
}
}
static async Task Main(string[] args)
{
if (args.Length != 3)
{
Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName");
return;
}
string bootstrapServers = args[0];
string schemaRegistryUrl = args[1];
string topicName = args[2];
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
// Note: you can specify more than one schema registry url using the
// schema.registry.url property for redundancy (comma separated list).
// The property name is not plural to follow the convention set by
// the Java implementation.
SchemaRegistryUrl = schemaRegistryUrl,
// optional schema registry client properties:
SchemaRegistryRequestTimeoutMs = 5000,
SchemaRegistryMaxCachedSchemas = 10
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = "avro-specific-example-group"
};
var avroSerializerConfig = new AvroSerializerConfig
{
// optional Avro serializer properties:
BufferBytes = 100,
AutoRegisterSchemas = true
};
// Note: The User class in this project was generated using the Confluent fork of the avrogen.exe tool
// (avaliable from: https://github.com/confluentinc/avro/tree/confluent-fork) which includes modifications
// that prevent namespace clashes with user namespaces that include the identifier 'Avro'. AvroSerializer
// and AvroDeserializer are also compatible with classes generated by the official avrogen.exe tool
// (available from: https://github.com/apache/avro), with the above limitation.
CancellationTokenSource cts = new CancellationTokenSource();
var consumeTask = Task.Run(() =>
{
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <string, User>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry))
.SetValueDeserializer(new AvroDeserializer <User>(schemaRegistry))
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.Build())
{
consumer.Subscribe(topicName);
while (!cts.Token.IsCancellationRequested)
{
try
{
var consumeResult = consumer.Consume(cts.Token);
Console.WriteLine($"user key name: {consumeResult.Message.Key}, user value favorite color: {consumeResult.Value.favorite_color}");
}
catch (ConsumeException e)
{
Console.WriteLine("Consume error: " + e.Error.Reason);
}
}
consumer.Close();
}
}, cts.Token);
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, User>(producerConfig)
.SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
.SetValueSerializer(new AvroSerializer <User>(schemaRegistry))
.Build())
{
Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit.");
int i = 0;
string text;
while ((text = Console.ReadLine()) != "q")
{
User user = new User {
name = text, favorite_color = "green", favorite_number = i++
};
await producer
.ProduceAsync(topicName, new Message <string, User> {
Key = text, Value = user
})
.ContinueWith(task => task.IsFaulted
? $"error producing message: {task.Exception.Message}"
: $"produced to: {task.Result.TopicPartitionOffset}");
}
}
cts.Cancel();
}
private static void ProduceConsume(string bootstrapServers, string schemaRegistryServers, SubjectNameStrategy nameStrategy)
{
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = true
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = schemaRegistryServers,
};
var avroSerializerConfig = new AvroSerializerConfig
{
SubjectNameStrategy = nameStrategy
};
var adminClientConfig = new AdminClientConfig
{
BootstrapServers = bootstrapServers
};
string topic = Guid.NewGuid().ToString();
using (var adminClient = new AdminClientBuilder(adminClientConfig).Build())
{
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topic, NumPartitions = 1, ReplicationFactor = 1
}
}).Wait();
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, ProduceConsumeUser>(producerConfig)
.SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
// Test ValueSubjectNameStrategy here,
// and KeySubjectNameStrategy in ProduceConsumeGeneric.
.SetValueSerializer(new AvroSerializer <ProduceConsumeUser>(schemaRegistry, avroSerializerConfig))
.Build())
{
for (int i = 0; i < 100; ++i)
{
var user = new ProduceConsumeUser
{
name = i.ToString(),
favorite_number = i,
favorite_color = "blue"
};
producer
.ProduceAsync(topic, new Message <string, ProduceConsumeUser> {
Key = user.name, Value = user
})
.Wait();
}
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <string, ProduceConsumeUser>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(new AvroDeserializer <ProduceConsumeUser>(schemaRegistry).AsSyncOverAsync())
.SetErrorHandler((_, e) => Assert.True(false, e.Reason))
.Build())
{
consumer.Subscribe(topic);
int i = 0;
while (true)
{
var record = consumer.Consume(TimeSpan.FromMilliseconds(100));
if (record == null)
{
continue;
}
if (record.IsPartitionEOF)
{
break;
}
Assert.Equal(i.ToString(), record.Message.Key);
Assert.Equal(i.ToString(), record.Message.Value.name);
Assert.Equal(i, record.Message.Value.favorite_number);
Assert.Equal("blue", record.Message.Value.favorite_color);
i += 1;
}
Assert.Equal(100, i);
consumer.Close();
}
// Check that what's in schema registry is what's expected.
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
{
var subjects = schemaRegistry.GetAllSubjectsAsync().Result;
if (nameStrategy == SubjectNameStrategy.TopicRecord)
{
Assert.Equal(2, (int)subjects.Where(s => s.Contains(topic)).Count());
Assert.Single(subjects.Where(s => s == $"{topic}-key"));
Assert.Single(subjects.Where(s => s == $"{topic}-{((Avro.RecordSchema)ProduceConsumeUser._SCHEMA).Fullname}"));
}
if (nameStrategy == SubjectNameStrategy.Topic)
{
Assert.Equal(2, (int)subjects.Where(s => s.Contains(topic)).Count());
Assert.Single(subjects.Where(s => s == $"{topic}-key"));
Assert.Single(subjects.Where(s => s == $"{topic}-value"));
}
if (nameStrategy == SubjectNameStrategy.Record)
{
Assert.Single(subjects.Where(s => s.Contains(topic))); // the string key.
Assert.Single(subjects.Where(s => s == $"{topic}-key"));
Assert.Single(subjects.Where(s => s == $"{((Avro.RecordSchema)ProduceConsumeUser._SCHEMA).Fullname}"));
}
}
}
/// <summary>
/// Main method for console app.
/// </summary>
/// <param name="args">No arguments used.</param>
public static void Main(string[] args)
{
Console.WriteLine("Starting .net consumer.");
// Configure the group id, location of the bootstrap server, default deserializers,
// Confluent interceptors
var consumerConfig = new ConsumerConfig
{
BootstrapServers = "kafka:9092",
GroupId = "csharp-consumer-prev",
AutoOffsetReset = AutoOffsetReset.Earliest,
PluginLibraryPaths = "monitoring-interceptor",
};
using (var consumer = new ConsumerBuilder<string, string>(consumerConfig)
.SetPartitionsAssignedHandler((c, partitions) =>
{
// Calculate the time 5 minutes ago
var timestamp = new Confluent.Kafka.Timestamp(DateTime.Now.AddMinutes(-5));
// Build a list of TopicPartitionTimestamp
var timestamps = partitions.Select(tp => new TopicPartitionTimestamp(tp, timestamp));
// TODO: Request the offsets for the start timestamp
var offsets = c.OffsetsForTimes(???
foreach (var offset in offsets)
{
// TODO: Print the new offset for each partition
Console.WriteLine(???
}
// Return the new partition offsets
return offsets;
})
.Build())
{
// Subscribe to our topic
consumer.Subscribe(KafkaTopic);
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true; // prevent the process from terminating.
cts.Cancel();
};
try
{
int recordCount = 0;
while (true)
{
try
{
// Poll for available records
var cr = consumer.Consume(cts.Token);
Console.WriteLine($"{cr.Key},{cr.Value}");
recordCount++;
// Exit processing after 100 records
if (recordCount >= 100)
{
break;
}
}
catch (ConsumeException e)
{
Console.WriteLine($"Error occured: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
// Clean up when the application exits
Console.WriteLine("Closing consumer.");
consumer.Close();
}
}
}
static async Task Main(string[] args)
{
if (args.Length != 3)
{
Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName");
return;
}
string bootstrapServers = args[0];
string schemaRegistryUrl = args[1];
string topicName = args[2];
string groupName = "avro-generic-example-group";
// var s = (RecordSchema)RecordSchema.Parse(File.ReadAllText("my-schema.json"));
var s = (RecordSchema)RecordSchema.Parse(
@"{
""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"",
""type"": ""record"",
""name"": ""User"",
""fields"": [
{""name"": ""name"", ""type"": ""string""},
{""name"": ""favorite_number"", ""type"": [""int"", ""null""]},
{""name"": ""favorite_color"", ""type"": [""string"", ""null""]}
]
}"
);
CancellationTokenSource cts = new CancellationTokenSource();
var consumeTask = Task.Run(() =>
{
using (var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
SchemaRegistryUrl = schemaRegistryUrl
}))
using (var consumer =
new ConsumerBuilder <string, GenericRecord>(new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = groupName
})
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry))
.SetValueDeserializer(new AvroDeserializer <GenericRecord>(schemaRegistry))
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.Build())
{
consumer.Subscribe(topicName);
while (!cts.Token.IsCancellationRequested)
{
try
{
var consumeResult = consumer.Consume(cts.Token);
Console.WriteLine($"Key: {consumeResult.Message.Key}\nValue: {consumeResult.Value}");
}
catch (ConsumeException e)
{
Console.WriteLine("Consume error: " + e.Error.Reason);
}
}
consumer.Close();
}
}, cts.Token);
using (var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
SchemaRegistryUrl = schemaRegistryUrl
}))
using (var producer =
new ProducerBuilder <string, GenericRecord>(new ProducerConfig {
BootstrapServers = bootstrapServers
})
.SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
.SetValueSerializer(new AvroSerializer <GenericRecord>(schemaRegistry))
.Build())
{
Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit.");
int i = 0;
string text;
while ((text = Console.ReadLine()) != "q")
{
var record = new GenericRecord(s);
record.Add("name", text);
record.Add("favorite_number", i++);
record.Add("favorite_color", "blue");
await producer
.ProduceAsync(topicName, new Message <string, GenericRecord> {
Key = text, Value = record
})
.ContinueWith(task => task.IsFaulted
? $"error producing message: {task.Exception.Message}"
: $"produced to: {task.Result.TopicPartitionOffset}");
}
}
cts.Cancel();
}
public void Consumer_PartitionEOF(string bootstrapServers)
{
LogToFile("start Consumer_PartitionEOF");
int N = 2;
var firstProduced = Util.ProduceNullStringMessages(bootstrapServers, singlePartitionTopic, 100, N);
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
EnablePartitionEof = false
};
// no eof, non generic consumer case.
using (var consumer =
new ConsumerBuilder <byte[], byte[]>(consumerConfig)
.SetPartitionsAssignedHandler((c, partitions) =>
{
Assert.Single(partitions);
Assert.Equal(firstProduced.TopicPartition, partitions[0]);
return(partitions.Select(p => new TopicPartitionOffset(p, firstProduced.Offset)));
})
.Build())
{
consumer.Subscribe(singlePartitionTopic);
var cr1 = consumer.Consume();
Assert.NotNull(cr1.Message);
Assert.False(cr1.IsPartitionEOF);
var cr2 = consumer.Consume();
Assert.NotNull(cr2.Message);
Assert.False(cr2.IsPartitionEOF);
var cr3 = consumer.Consume(TimeSpan.FromSeconds(1));
Assert.Null(cr3);
consumer.Close();
}
// no eof, generic consumer case.
using (var consumer =
new ConsumerBuilder <Null, string>(consumerConfig)
.SetPartitionsAssignedHandler((c, partitions) =>
{
Assert.Single(partitions);
Assert.Equal(firstProduced.TopicPartition, partitions[0]);
return(partitions.Select(p => new TopicPartitionOffset(p, firstProduced.Offset)));
})
.Build())
{
consumer.Subscribe(singlePartitionTopic);
var cr1 = consumer.Consume();
Assert.NotNull(cr1.Message);
Assert.False(cr1.IsPartitionEOF);
var cr2 = consumer.Consume();
Assert.NotNull(cr2.Message);
Assert.False(cr2.IsPartitionEOF);
var cr3 = consumer.Consume(TimeSpan.FromSeconds(1));
Assert.Null(cr3);
consumer.Close();
}
consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
EnablePartitionEof = true
};
// eof, non-generic consumer case.
using (var consumer =
new ConsumerBuilder <byte[], byte[]>(consumerConfig)
.SetPartitionsAssignedHandler((c, partitions) =>
{
Assert.Single(partitions);
Assert.Equal(firstProduced.TopicPartition, partitions[0]);
return(partitions.Select(p => new TopicPartitionOffset(p, firstProduced.Offset)));
})
.Build())
{
consumer.Subscribe(singlePartitionTopic);
var cr1 = consumer.Consume();
Assert.NotNull(cr1.Message);
Assert.False(cr1.IsPartitionEOF);
var cr2 = consumer.Consume();
Assert.NotNull(cr2.Message);
Assert.False(cr2.IsPartitionEOF);
var cr3 = consumer.Consume();
Assert.Null(cr3.Message);
Assert.True(cr3.IsPartitionEOF);
var cr4 = consumer.Consume(TimeSpan.FromSeconds(1));
Assert.Null(cr4);
consumer.Close();
}
// eof, generic consumer case.
using (var consumer =
new ConsumerBuilder <Null, string>(consumerConfig)
.SetPartitionsAssignedHandler((c, partitions) =>
{
Assert.Single(partitions);
Assert.Equal(firstProduced.TopicPartition, partitions[0]);
return(partitions.Select(p => new TopicPartitionOffset(p, firstProduced.Offset)));
})
.Build())
{
consumer.Subscribe(singlePartitionTopic);
var cr1 = consumer.Consume();
Assert.NotNull(cr1.Message);
Assert.False(cr1.IsPartitionEOF);
var cr2 = consumer.Consume();
Assert.NotNull(cr2.Message);
Assert.False(cr2.IsPartitionEOF);
var cr3 = consumer.Consume();
Assert.Null(cr3.Message);
Assert.True(cr3.IsPartitionEOF);
var cr4 = consumer.Consume(TimeSpan.FromSeconds(1));
Assert.Null(cr4);
consumer.Close();
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Consumer_PartitionEOF");
}
/// <summary>
/// In this example
/// - consumer group functionality (i.e. .Subscribe + offset commits) is not used.
/// - the consumer is manually assigned to a partition and always starts consumption
/// from a specific offset (0).
/// </summary>
public static void Run_ManualAssign(string brokerList, List <string> topics, CancellationToken cancellationToken)
{
var config = new ConsumerConfig
{
// the group.id property must be specified when creating a consumer, even
// if you do not intend to use any consumer group functionality.
GroupId = new Guid().ToString(),
BootstrapServers = brokerList,
// partition offsets can be committed to a group even by consumers not
// subscribed to the group. in this example, auto commit is disabled
// to prevent this from occurring.
EnableAutoCommit = true
};
using (var consumer =
new ConsumerBuilder <string, string>(config)
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.Build())
{
List <TopicMessages> messages = new List <TopicMessages>();
consumer.Assign(topics.Select(topic => new TopicPartitionOffset(topic, 0, Offset.Beginning)).ToList());
try
{
while (true)
{
try
{
var consumeResult = consumer.Consume(cancellationToken);
// Note: End of partition notification has not been enabled, so
// it is guaranteed that the ConsumeResult instance corresponds
// to a Message, and not a PartitionEOF event.
Console.WriteLine($"Received message at {consumeResult.TopicPartitionOffset}: ${consumeResult.Message.Value}");
messages.Add(new TopicMessages()
{
Topic = consumeResult.Topic,
Offset = consumeResult.Offset.Value,
Key = consumeResult.Message.Key,
Value = consumeResult.Message.Value,
});
//Console.WriteLine($"Received message at topic: {consumeResult.Topic}, partition: {consumeResult.Partition}, message key: {consumeResult.Message.Key}, offset: {consumeResult.Offset}, topic partition offset: {consumeResult.TopicPartitionOffset}");
}
catch (ConsumeException e)
{
Console.WriteLine($"Consume error: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
Console.WriteLine("Closing consumer.");
consumer.Close();
}
//Only save the results at the end
//We can make that it is saved when we have a certain number of elements
using (var db = new TestDbContext())
{
db.Messages.AddRange(
messages
);
var count = db.SaveChanges();
Console.WriteLine("{0} records saved to database", count);
}
}
}