public TaskSynchronousTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, IKafkaSupplier supplier, CancellationToken token) { this.configuration = configuration; this.configuration.ClientId = clientId; this.topicConfiguration = topicConfiguration; this.token = token; builder = topologyBuilder; this.supplier = supplier ?? new SyncKafkaSupplier(); producer = this.supplier.GetProducer(configuration.ToProducerConfig()) as SyncProducer; foreach (var sourceTopic in builder.GetSourceTopics().Union(builder.GetGlobalTopics())) { var part = new TopicPartition(sourceTopic, 0); var taskId = builder.GetTaskIdFromPartition(part); if (partitionsByTaskId.ContainsKey(taskId)) { partitionsByTaskId[taskId].Add(part); } else { partitionsByTaskId.Add(taskId, new List <TopicPartition> { part }); } } }
public TaskSynchronousTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, CancellationToken token) { this.configuration = configuration; this.configuration.ClientId = clientId; this.topicConfiguration = topicConfiguration; this.token = token; builder = topologyBuilder; supplier = new SyncKafkaSupplier(); producer = supplier.GetProducer(configuration.ToProducerConfig()) as SyncProducer; }
internal static IThread Create(string threadId, string clientId, InternalTopologyBuilder builder, StreamMetricsRegistry streamMetricsRegistry, IStreamConfig configuration, IKafkaSupplier kafkaSupplier, IAdminClient adminClient, int threadInd) { string logPrefix = $"stream-thread[{threadId}] "; var log = Logger.GetLogger(typeof(StreamThread)); var customerID = $"{clientId}-StreamThread-{threadInd}"; IProducer <byte[], byte[]> producer = null; // TODO : remove this limitations depends version of Kafka Cluster // Due to limitations outlined in KIP-447 (which KIP-447 overcomes), it is // currently necessary to use a separate producer per input partition. The // producerState dictionary is used to keep track of these, and the current // consumed offset. // https://cwiki.apache.org/confluence/display/KAFKA/KIP-447%3A+Producer+scalability+for+exactly+once+semantics // IF Guarantee is AT_LEAST_ONCE, producer is the same of all StreamTasks in this thread, // ELSE one producer by StreamTask. if (configuration.Guarantee == ProcessingGuarantee.AT_LEAST_ONCE) { log.LogInformation("{LogPrefix}Creating shared producer client", logPrefix); producer = kafkaSupplier.GetProducer(configuration.ToProducerConfig(GetThreadProducerClientId(threadId)).Wrap(threadId)); } var restoreConfig = configuration.ToConsumerConfig(GetRestoreConsumerClientId(customerID)); restoreConfig.GroupId = $"{configuration.ApplicationId}-restore-group"; var restoreConsumer = kafkaSupplier.GetRestoreConsumer(restoreConfig); var storeChangelogReader = new StoreChangelogReader( configuration, restoreConsumer, threadId, streamMetricsRegistry); var taskCreator = new TaskCreator(builder, configuration, threadId, kafkaSupplier, producer, storeChangelogReader, streamMetricsRegistry); var manager = new TaskManager(builder, taskCreator, adminClient, storeChangelogReader); var listener = new StreamsRebalanceListener(manager); log.LogInformation("{LogPrefix}Creating consumer client", logPrefix); var consumer = kafkaSupplier.GetConsumer(configuration.ToConsumerConfig(GetConsumerClientId(customerID)).Wrap(threadId), listener); manager.Consumer = consumer; var thread = new StreamThread(threadId, customerID, manager, consumer, builder, storeChangelogReader, streamMetricsRegistry, configuration); listener.Thread = thread; return(thread); }
public void RestoreFromRecovery() { storeChangelogReader.Restore(); stateMgr.Checkpoint(); stateMgr.Close(); store.Init(context, store); stateMgr.InitializeOffsetsFromCheckpoint(); var producer = supplier.GetProducer(config.ToProducerConfig()); producer.Produce(changelogTopic, CreateMessage(changelogTopic, "key3", "value3")); producer.Produce(changelogTopic, CreateMessage(changelogTopic, "key4", "value4")); restoreConsumer.Resume(new TopicPartition(changelogTopic, 0).ToSingle()); storeChangelogReader.Restore(); stateMgr.Checkpoint(); stateMgr.Close(); File.Delete(Path.Combine(config.StateDir, config.ApplicationId, "0-0", ".checkpoint")); store.Init(context, store); stateMgr.InitializeOffsetsFromCheckpoint(); restoreConsumer.Resume(new TopicPartition(changelogTopic, 0).ToSingle()); storeChangelogReader.Restore(); Assert.AreEqual(4, store.All().ToList().Count); var metadata = storeChangelogReader.GetMetadata(new TopicPartition(changelogTopic, 0)); Assert.IsNotNull(metadata); Assert.AreEqual(ChangelogState.COMPLETED, metadata.ChangelogState); Assert.AreEqual(3, metadata.RestoreEndOffset); Assert.AreEqual(2, metadata.TotalRestored); }
public TaskSynchronousTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, IKafkaSupplier supplier, CancellationToken token) { this.configuration = configuration; this.configuration.ClientId = clientId; this.topicConfiguration = topicConfiguration; metricsRegistry = new StreamMetricsRegistry(clientId, MetricsRecordingLevel.DEBUG); this.token = token; builder = topologyBuilder; this.supplier = supplier ?? new SyncKafkaSupplier(); this.supplier.MetricsRegistry = metricsRegistry; producer = this.supplier.GetProducer(configuration.ToProducerConfig()) as SyncProducer; foreach (var sourceTopic in builder .GetSourceTopics()) { var part = new TopicPartition(sourceTopic, 0); var taskId = builder.GetTaskIdFromPartition(part); if (partitionsByTaskId.ContainsKey(taskId)) { partitionsByTaskId[taskId].Add(part); } else { partitionsByTaskId.Add(taskId, new List <TopicPartition> { part }); } } ProcessorTopology globalTaskTopology = topologyBuilder.BuildGlobalStateTopology(); hasGlobalTopology = globalTaskTopology != null; if (hasGlobalTopology) { var globalConsumer = this.supplier.GetGlobalConsumer(configuration.ToGlobalConsumerConfig($"{clientId}-global-consumer")); var adminClient = this.supplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin")); var stateManager = new GlobalStateManager(globalConsumer, globalTaskTopology, adminClient, configuration); globalProcessorContext = new GlobalProcessorContext(configuration, stateManager, metricsRegistry); stateManager.SetGlobalProcessorContext(globalProcessorContext); globalTask = new GlobalStateUpdateTask(stateManager, globalTaskTopology, globalProcessorContext); globalTask.Initialize(); } }
public void Init() { stateDir = Path.Combine(".", Guid.NewGuid().ToString()); config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-storechangelog-app"; config.StateDir = stateDir; config.PollMs = 100; supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); restoreConsumer = supplier.GetRestoreConsumer(config.ToConsumerConfig()); var topicPart = new TopicPartition("topic", 0); changelogTopic = "store-changelog-topic"; var changelogsTopics = new Dictionary <string, string>(); changelogsTopics.Add("store", changelogTopic); var id = new TaskId { Id = 0, Partition = 0 }; store = new RocksDbKeyValueStore("store"); storeChangelogReader = new StoreChangelogReader(config, restoreConsumer, "thread-0", new StreamMetricsRegistry()); stateMgr = new ProcessorStateManager( id, topicPart.ToSingle(), changelogsTopics, storeChangelogReader, new OffsetCheckpointFile(Path.Combine(config.StateDir, config.ApplicationId, $"{id.Id}-{id.Partition}")) ); Mock <AbstractTask> moq = new Mock <AbstractTask>(); moq.Setup(t => t.Id).Returns(new TaskId { Id = 0, Partition = 0 }); context = new ProcessorContext(moq.Object, config, stateMgr, new StreamMetricsRegistry()); store.Init(context, store); producer.Produce(changelogTopic, CreateMessage(changelogTopic, "key1", "value1")); producer.Produce(changelogTopic, CreateMessage(changelogTopic, "key2", "value2")); }
internal static IThread Create(string threadId, string clientId, InternalTopologyBuilder builder, IStreamConfig configuration, IKafkaSupplier kafkaSupplier, IAdminClient adminClient, int threadInd) { string logPrefix = $"stream-thread[{threadId}] "; var log = Logger.GetLogger(typeof(StreamThread)); var customerID = $"{clientId}-StreamThread-{threadInd}"; IProducer <byte[], byte[]> producer = null; // Due to limitations outlined in KIP-447 (which KIP-447 overcomes), it is // currently necessary to use a separate producer per input partition. The // producerState dictionary is used to keep track of these, and the current // consumed offset. // https://cwiki.apache.org/confluence/display/KAFKA/KIP-447%3A+Producer+scalability+for+exactly+once+semantics // IF Guarantee is AT_LEAST_ONCE, producer is the same of all StreamTasks in this thread, // ELSE one producer by StreamTask. if (configuration.Guarantee == ProcessingGuarantee.AT_LEAST_ONCE) { log.Info($"{logPrefix}Creating shared producer client"); producer = kafkaSupplier.GetProducer(configuration.ToProducerConfig(GetThreadProducerClientId(threadId))); } var taskCreator = new TaskCreator(builder, configuration, threadId, kafkaSupplier, producer); var manager = new TaskManager(builder, taskCreator, adminClient); var listener = new StreamsRebalanceListener(manager); log.Info($"{logPrefix}Creating consumer client"); var consumer = kafkaSupplier.GetConsumer(configuration.ToConsumerConfig(customerID), listener); manager.Consumer = consumer; var thread = new StreamThread(threadId, customerID, manager, consumer, builder, TimeSpan.FromMilliseconds(configuration.PollMs), configuration.CommitIntervalMs); listener.Thread = thread; return(thread); }