static async Task Main()
{
// Setting up the configuration for the Kafka client and Schema registry, saved in a local file
GlobalParameters.clientConfig = await ConfluentHelper.LoadKafkaConfiguration(@"C:\Github\confluent-configuration.txt", null);
GlobalParameters.schemaRegistryConfig = await ConfluentHelper.LoadSchemaRegistryConfiguration(@"C:\Github\schemaregistry-configuration.txt");
// Clear topic, if existing (reset environment)
await ConfluentHelper.DeleteTopic(GlobalParameters.topics[0], GlobalParameters.clientConfig);
// Create topic, if not existing yet
await ConfluentHelper.CreateTopicIfNotExists(GlobalParameters.topics[0], 1, 3, GlobalParameters.clientConfig);
await ConfluentHelper.CreateTopicIfNotExists(GlobalParameters.topics[1], 1, 3, GlobalParameters.clientConfig);
// Start a file watcher to monitor input directory for mapping Json files
// Event handles on file detection trigger the publishing actions
WatchForFiles(AppDomain.CurrentDomain.BaseDirectory + @"\examples-publication");
for (int i = 0; i < 100; i++)
{
var bla = new LocalSystemPerformance();
await PublishEventSystemPerformance(bla, GlobalParameters.schemaRegistryConfig, GlobalParameters.clientConfig, GlobalParameters.topics);
}
// Start waiting until Escape is pressed
Console.WriteLine();
Console.WriteLine("Press ESC to quit.");
do
{
while (!Console.KeyAvailable)
{
// Wait for anything to happen in the designated directory for the file watcher
}
} while (Console.ReadKey(true).Key != ConsoleKey.Escape);
// END OF APPLICATION
}
static async Task Main()
{
// Setting up the configuration for the Kafka client and Schema registry, saved in a local file
GlobalParameters.clientConfig = await ConfluentHelper.LoadKafkaConfiguration(@"C:\Github\confluent-configuration.txt", null);
Console.WriteLine("Loaded client configuration, commencing consuming.");
List <DataObjectMapping> localMappingList = new List <DataObjectMapping>();
// Consumer group
var consumerConfig = new ConsumerConfig(GlobalParameters.clientConfig)
{
GroupId = "data-object-mapping-consumer",
AutoOffsetReset = AutoOffsetReset.Earliest,
EnableAutoCommit = true,
EnablePartitionEof = true
};
// Cancellation input key (token) for quitting the process in async mode
CancellationTokenSource cts = new CancellationTokenSource();
Console.WriteLine("Press CTRL-C any time to quit polling for new events.");
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true; // prevent the process from terminating.
cts.Cancel();
};
using (var consumer = new ConsumerBuilder <string, DataObjectMapping>(consumerConfig)
//.SetValueDeserializer(new JsonDeserializer<DataObjectMapping>().AsSyncOverAsync())
.SetValueDeserializer(new JsonDeserializer <DataObjectMapping>().AsSyncOverAsync())
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.SetStatisticsHandler((_, json) => Console.WriteLine($"Statistics: {json}"))
.SetPartitionsAssignedHandler((c, partitions) =>
{
Console.WriteLine($"Assigned partitions: [{string.Join(", ", partitions)}]");
// possibly manually specify start offsets or override the partition assignment provided by
// the consumer group by returning a list of topic/partition/offsets to assign to, e.g.:
//
//return partitions.Select(tp => new TopicPartitionOffset(tp, externalOffsets[tp]));
})
.SetPartitionsRevokedHandler((c, partitions) =>
{
Console.WriteLine($"Revoking assignment: [{string.Join(", ", partitions)}]");
})
.Build())
{
consumer.Subscribe(GlobalParameters.topics);
// Manual assignment (not automatic subscribe)
//var partitionList = topics.Select(topic => new TopicPartitionOffset(topic, 0, Offset.Beginning)).ToList();
//consumer.Assign(partitionList);
try
{
while (true)
{
try
{
var consumeResult = consumer.Consume(cts.Token);
// End of partition notification has not been enabled, meaning it is guaranteed that the ConsumeResult instance corresponds to a Message, and not a PartitionEOF event.
if (consumeResult.IsPartitionEOF)
{
Console.WriteLine(
$"Reached end of topic {consumeResult.Topic}, partition {consumeResult.Partition}, offset {consumeResult.Offset}.");
continue;
}
Console.WriteLine($"Received message at {consumeResult.TopicPartitionOffset}: ${consumeResult.Message.Value.sourceDataObject.name}-{consumeResult.Message.Value.targetDataObject.name}");
localMappingList.Add(consumeResult.Message.Value);
}
catch (ConsumeException e)
{
Console.WriteLine($"Consume error: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
Console.WriteLine("Closing consumer.");
consumer.Close();
}
}
// Display received in-memory events back to the user
Console.WriteLine("The following events were received.");
foreach (var indvidiualMapping in localMappingList)
{
Console.WriteLine(indvidiualMapping);
}
Console.WriteLine("Press any key to stop the application.");
Console.ReadKey();
}
