public static void CreateTopicsIfRequired(string bootstrapServers, string tableSpecification, int numPartitions, bool recreate)
{
var tableSpec = new TableSpecification(tableSpecification);
var config = new AdminClientConfig
{
BootstrapServers = bootstrapServers
};
using (var ac = new Howlett.Kafka.Extensions.AdminClient(config))
{
foreach (var cs in tableSpec.ColumnSpecifications.Where(a => a.Unique))
{
if (recreate)
{
Console.WriteLine($"deleting topics for column {cs.Name}");
ac.DeleteTopicMaybeAsync(tableSpec.ChangeLogTopicName(cs.Name)).GetAwaiter().GetResult();
ac.DeleteTopicMaybeAsync(tableSpec.CommandTopicName(cs.Name)).GetAwaiter().GetResult();
Thread.Sleep(1000);
}
ac.CreateTopicMaybeAsync(
tableSpec.ChangeLogTopicName(cs.Name), numPartitions, 1,
new Dictionary <string, string>
{
{ "cleanup.policy", "compact" }
}
).GetAwaiter().GetResult();
ac.CreateTopicMaybeAsync(
tableSpec.CommandTopicName(cs.Name), numPartitions, 1, null
).GetAwaiter().GetResult();
}
}
}
private void InitClients(string bootstrapServers)
{
var pConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers,
EnableIdempotence = true,
MessageTimeoutMs = 900000, // 15 minutes. TODO: does this need to be infinite to guarantee gapless?
LingerMs = 5,
};
cmdProducer = new ProducerBuilder <Null, string>(pConfig).Build();
clProducer = new DependentProducerBuilder <string, string>(cmdProducer.Handle).Build();
var cConfig1 = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
// EnableAutoCommit = true,
// EnableAutoOffsetStore = false,
AutoOffsetReset = AutoOffsetReset.Earliest,
GroupId = tableSpecification.ChangeLogTopicName(this.columnName) + "_cg",
EnablePartitionEof = true,
FetchWaitMaxMs = 10
};
changeLogConsumer = new ConsumerBuilder <string, string>(cConfig1).Build();
var cConfig2 = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
// EnableAutoCommit = true,
// EnableAutoOffsetStore = false,
AutoOffsetReset = AutoOffsetReset.Earliest,
GroupId = tableSpecification.CommandTopicName(this.columnName) + "_cg",
FetchWaitMaxMs = 10
};
commandConsumer = new ConsumerBuilder <Null, string>(cConfig2)
.SetPartitionsAssignedHandler((c, ps) => new [] {
new TopicPartitionOffset(
this.tableSpecification.CommandTopicName(this.columnName),
this.partition,
Offset.Unset)
})
.Build();
}