public static void CreateTopicsIfRequired(string bootstrapServers, string tableSpecification, int numPartitions, bool recreate) { var tableSpec = new TableSpecification(tableSpecification); var config = new AdminClientConfig { BootstrapServers = bootstrapServers }; using (var ac = new Howlett.Kafka.Extensions.AdminClient(config)) { foreach (var cs in tableSpec.ColumnSpecifications.Where(a => a.Unique)) { if (recreate) { Console.WriteLine($"deleting topics for column {cs.Name}"); ac.DeleteTopicMaybeAsync(tableSpec.ChangeLogTopicName(cs.Name)).GetAwaiter().GetResult(); ac.DeleteTopicMaybeAsync(tableSpec.CommandTopicName(cs.Name)).GetAwaiter().GetResult(); Thread.Sleep(1000); } ac.CreateTopicMaybeAsync( tableSpec.ChangeLogTopicName(cs.Name), numPartitions, 1, new Dictionary <string, string> { { "cleanup.policy", "compact" } } ).GetAwaiter().GetResult(); ac.CreateTopicMaybeAsync( tableSpec.CommandTopicName(cs.Name), numPartitions, 1, null ).GetAwaiter().GetResult(); } } }
public ColumnPartition( TableSpecification tableSpecification, string bootstrapServers, string columnName, int partition, int numPartitions, bool recreate, bool logCommands, CancellationToken ct) { this.logCommands = logCommands; this.tableSpecification = tableSpecification; this.otherUniqueColumns = tableSpecification.ColumnSpecifications.Where(s => s.Unique && s.Name != columnName).ToList(); this.partition = partition; this.numPartitions = numPartitions; this.columnName = columnName; InitClients(bootstrapServers); StartCommandConsumer(ct); StartChangeLogConsumer(ct); }
public TablePartition( string bootstrapServers, string tableSpec, int partition, int numPartitions, bool recreate, bool logCommands, CancellationToken ct) { this.partition = partition; this.tableSpecification = new TableSpecification(tableSpec); columns = new Dictionary <string, ColumnPartition>(); foreach (var c in tableSpecification.ColumnSpecifications) { if (c.Unique) { var column = new ColumnPartition(tableSpecification, bootstrapServers, c.Name, partition, numPartitions, recreate, logCommands, ct); columns.Add(c.Name, column); } } }