internal async Task CreateInternalTopicsAsync( ITopicManager topicManager, InternalTopologyBuilder builder) { var clusterMetadata = topicManager.AdminClient.GetMetadata(timeout); brokerConfigResource.Name = clusterMetadata.Brokers[0].BrokerId.ToString(); var resultsConf = await topicManager.AdminClient.DescribeConfigsAsync(new List <ConfigResource> { brokerConfigResource }); var internalTopicsGroups = builder.MakeInternalTopicGroups(); foreach (var entry in internalTopicsGroups) { ComputeRepartitionTopicConfig(entry.Value, internalTopicsGroups, clusterMetadata); ComputeChangelogTopicConfig(entry.Value, clusterMetadata, resultsConf); var internalTopics = entry.Value.ChangelogTopics.Union(entry.Value.RepartitionTopics).ToDictionary(); await topicManager.ApplyAsync(entry.Key, internalTopics); // refresh metadata clusterMetadata = topicManager.AdminClient.GetMetadata(timeout); } }
internal TaskManager(InternalTopologyBuilder builder, TaskCreator taskCreator, IAdminClient adminClient, IChangelogReader changelogReader) { this.builder = builder; this.taskCreator = taskCreator; this.adminClient = adminClient; this.changelogReader = changelogReader; }
public TaskCreator(InternalTopologyBuilder builder, IStreamConfig configuration, string threadId, IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer) : base() { this.builder = builder; this.configuration = configuration; this.threadId = threadId; this.kafkaSupplier = kafkaSupplier; this.producer = producer; }
public TaskCreator(InternalTopologyBuilder builder, IStreamConfig configuration, string threadId, IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer, StoreChangelogReader storeChangelogReader, StreamMetricsRegistry streamMetricsRegistry) { this.builder = builder; this.configuration = configuration; this.threadId = threadId; this.kafkaSupplier = kafkaSupplier; this.producer = producer; this.storeChangelogReader = storeChangelogReader; this.streamMetricsRegistry = streamMetricsRegistry; createTaskSensor = ThreadMetrics.CreateTaskSensor(threadId, streamMetricsRegistry); }
// Use for testing (TaskSynchronousTopologyDriver & ClusterInMemoryTopologyDriver) to create source topics before repartition & changelog topcis internal InternalTopicManagerUtils CreateSourceTopics(InternalTopologyBuilder builder, IKafkaSupplier supplier) { var adminConfig = new AdminClientConfig(); adminConfig.ClientId = "internal-admin-create-soure-topic"; var sourceTopics = builder.BuildTopology().GetSourceTopics().ToList(); var globalTopo = builder.BuildGlobalStateTopology(); if (globalTopo != null) { sourceTopics.AddRange(globalTopo.StoresToTopics.Values); } supplier .GetAdmin(adminConfig) .CreateTopicsAsync(sourceTopics.Select(s => new TopicSpecification() { Name = s, NumPartitions = -1 })).GetAwaiter().GetResult(); return(this); }
public TaskManager(InternalTopologyBuilder builder, TaskCreator taskCreator, IAdminClient adminClient, IConsumer <byte[], byte[]> consumer) : this(builder, taskCreator, adminClient) { Consumer = consumer; }
public TaskManager(InternalTopologyBuilder builder, TaskCreator taskCreator, IAdminClient adminClient) { this.builder = builder; this.taskCreator = taskCreator; this.adminClient = adminClient; }
public AbstractTaskCreator(InternalTopologyBuilder builder, IStreamConfig config) { this.builder = builder; this.config = config; this.log = Logger.GetLogger(this.GetType()); }
internal TaskManager(InternalTopologyBuilder builder, TaskCreator taskCreator, IAdminClient adminClient, IConsumer <byte[], byte[]> consumer, IChangelogReader changelogReader) : this(builder, taskCreator, adminClient, changelogReader) { Consumer = consumer; }