public static void CreateTopicsIfRequired(string bootstrapServers, string tableSpecification, int numPartitions, bool recreate)
{
var tableSpec = new TableSpecification(tableSpecification);
var config = new AdminClientConfig
{
BootstrapServers = bootstrapServers
};
using (var ac = new Howlett.Kafka.Extensions.AdminClient(config))
{
foreach (var cs in tableSpec.ColumnSpecifications.Where(a => a.Unique))
{
if (recreate)
{
Console.WriteLine($"deleting topics for column {cs.Name}");
ac.DeleteTopicMaybeAsync(tableSpec.ChangeLogTopicName(cs.Name)).GetAwaiter().GetResult();
ac.DeleteTopicMaybeAsync(tableSpec.CommandTopicName(cs.Name)).GetAwaiter().GetResult();
Thread.Sleep(1000);
}
ac.CreateTopicMaybeAsync(
tableSpec.ChangeLogTopicName(cs.Name), numPartitions, 1,
new Dictionary <string, string>
{
{ "cleanup.policy", "compact" }
}
).GetAwaiter().GetResult();
ac.CreateTopicMaybeAsync(
tableSpec.CommandTopicName(cs.Name), numPartitions, 1, null
).GetAwaiter().GetResult();
}
}
}
public ColumnPartition(
TableSpecification tableSpecification,
string bootstrapServers,
string columnName,
int partition,
int numPartitions,
bool recreate,
bool logCommands,
CancellationToken ct)
{
this.logCommands = logCommands;
this.tableSpecification = tableSpecification;
this.otherUniqueColumns = tableSpecification.ColumnSpecifications.Where(s => s.Unique && s.Name != columnName).ToList();
this.partition = partition;
this.numPartitions = numPartitions;
this.columnName = columnName;
InitClients(bootstrapServers);
StartCommandConsumer(ct);
StartChangeLogConsumer(ct);
}
public TablePartition(
string bootstrapServers,
string tableSpec,
int partition,
int numPartitions,
bool recreate,
bool logCommands,
CancellationToken ct)
{
this.partition = partition;
this.tableSpecification = new TableSpecification(tableSpec);
columns = new Dictionary <string, ColumnPartition>();
foreach (var c in tableSpecification.ColumnSpecifications)
{
if (c.Unique)
{
var column = new ColumnPartition(tableSpecification, bootstrapServers, c.Name, partition, numPartitions, recreate, logCommands, ct);
columns.Add(c.Name, column);
}
}
}
