internal async Task CreateInternalTopicsAsync(
ITopicManager topicManager,
InternalTopologyBuilder builder)
{
var clusterMetadata = topicManager.AdminClient.GetMetadata(timeout);
brokerConfigResource.Name = clusterMetadata.Brokers[0].BrokerId.ToString();
var resultsConf = await topicManager.AdminClient.DescribeConfigsAsync(new List <ConfigResource> {
brokerConfigResource
});
var internalTopicsGroups = builder.MakeInternalTopicGroups();
foreach (var entry in internalTopicsGroups)
{
ComputeRepartitionTopicConfig(entry.Value, internalTopicsGroups, clusterMetadata);
ComputeChangelogTopicConfig(entry.Value, clusterMetadata, resultsConf);
var internalTopics = entry.Value.ChangelogTopics.Union(entry.Value.RepartitionTopics).ToDictionary();
await topicManager.ApplyAsync(entry.Key, internalTopics);
// refresh metadata
clusterMetadata = topicManager.AdminClient.GetMetadata(timeout);
}
}
internal TaskManager(InternalTopologyBuilder builder, TaskCreator taskCreator, IAdminClient adminClient,
IChangelogReader changelogReader)
{
this.builder = builder;
this.taskCreator = taskCreator;
this.adminClient = adminClient;
this.changelogReader = changelogReader;
}
public TaskCreator(InternalTopologyBuilder builder, IStreamConfig configuration, string threadId, IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer)
: base()
{
this.builder = builder;
this.configuration = configuration;
this.threadId = threadId;
this.kafkaSupplier = kafkaSupplier;
this.producer = producer;
}
public TaskCreator(InternalTopologyBuilder builder, IStreamConfig configuration, string threadId,
IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer, StoreChangelogReader storeChangelogReader,
StreamMetricsRegistry streamMetricsRegistry)
{
this.builder = builder;
this.configuration = configuration;
this.threadId = threadId;
this.kafkaSupplier = kafkaSupplier;
this.producer = producer;
this.storeChangelogReader = storeChangelogReader;
this.streamMetricsRegistry = streamMetricsRegistry;
createTaskSensor = ThreadMetrics.CreateTaskSensor(threadId, streamMetricsRegistry);
}
// Use for testing (TaskSynchronousTopologyDriver & ClusterInMemoryTopologyDriver) to create source topics before repartition & changelog topcis
internal InternalTopicManagerUtils CreateSourceTopics(InternalTopologyBuilder builder, IKafkaSupplier supplier)
{
var adminConfig = new AdminClientConfig();
adminConfig.ClientId = "internal-admin-create-soure-topic";
var sourceTopics = builder.BuildTopology().GetSourceTopics().ToList();
var globalTopo = builder.BuildGlobalStateTopology();
if (globalTopo != null)
{
sourceTopics.AddRange(globalTopo.StoresToTopics.Values);
}
supplier
.GetAdmin(adminConfig)
.CreateTopicsAsync(sourceTopics.Select(s => new TopicSpecification()
{
Name = s,
NumPartitions = -1
})).GetAwaiter().GetResult();
return(this);
}
public TaskManager(InternalTopologyBuilder builder, TaskCreator taskCreator, IAdminClient adminClient, IConsumer <byte[], byte[]> consumer)
: this(builder, taskCreator, adminClient)
{
Consumer = consumer;
}
public TaskManager(InternalTopologyBuilder builder, TaskCreator taskCreator, IAdminClient adminClient)
{
this.builder = builder;
this.taskCreator = taskCreator;
this.adminClient = adminClient;
}
public AbstractTaskCreator(InternalTopologyBuilder builder, IStreamConfig config)
{
this.builder = builder;
this.config = config;
this.log = Logger.GetLogger(this.GetType());
}
internal TaskManager(InternalTopologyBuilder builder, TaskCreator taskCreator, IAdminClient adminClient, IConsumer <byte[], byte[]> consumer, IChangelogReader changelogReader)
: this(builder, taskCreator, adminClient, changelogReader)
{
Consumer = consumer;
}
