public TestMultiInputTopic <K, V> CreateMultiInputTopic <K, V>(string[] topics, ISerDes <K> keySerdes = null, ISerDes <V> valueSerdes = null) { Dictionary <string, IPipeInput> pipes = new Dictionary <string, IPipeInput>(); foreach (var topic in topics) { var builder = CreateBuilder(topic); var pipeInput = builder.Input(topic, configuration); var topicsLink = new List <string>(); this.builder.GetLinkTopics(topic, topicsLink); var consumer = supplier.GetConsumer(topicConfiguration.ToConsumerConfig("consumer-repartition-forwarder"), null); foreach (var topicLink in topicsLink) { pipeInput.Flushed += () => ForwardRepartitionTopic(consumer, topicLink); } pipes.Add(topic, pipeInput); } return(new TestMultiInputTopic <K, V>(pipes, configuration, keySerdes, valueSerdes)); }
internal static IThread Create(string threadId, string clientId, InternalTopologyBuilder builder, StreamMetricsRegistry streamMetricsRegistry, IStreamConfig configuration, IKafkaSupplier kafkaSupplier, IAdminClient adminClient, int threadInd) { string logPrefix = $"stream-thread[{threadId}] "; var log = Logger.GetLogger(typeof(StreamThread)); var customerID = $"{clientId}-StreamThread-{threadInd}"; IProducer <byte[], byte[]> producer = null; // TODO : remove this limitations depends version of Kafka Cluster // Due to limitations outlined in KIP-447 (which KIP-447 overcomes), it is // currently necessary to use a separate producer per input partition. The // producerState dictionary is used to keep track of these, and the current // consumed offset. // https://cwiki.apache.org/confluence/display/KAFKA/KIP-447%3A+Producer+scalability+for+exactly+once+semantics // IF Guarantee is AT_LEAST_ONCE, producer is the same of all StreamTasks in this thread, // ELSE one producer by StreamTask. if (configuration.Guarantee == ProcessingGuarantee.AT_LEAST_ONCE) { log.LogInformation("{LogPrefix}Creating shared producer client", logPrefix); producer = kafkaSupplier.GetProducer(configuration.ToProducerConfig(GetThreadProducerClientId(threadId)).Wrap(threadId)); } var restoreConfig = configuration.ToConsumerConfig(GetRestoreConsumerClientId(customerID)); restoreConfig.GroupId = $"{configuration.ApplicationId}-restore-group"; var restoreConsumer = kafkaSupplier.GetRestoreConsumer(restoreConfig); var storeChangelogReader = new StoreChangelogReader( configuration, restoreConsumer, threadId, streamMetricsRegistry); var taskCreator = new TaskCreator(builder, configuration, threadId, kafkaSupplier, producer, storeChangelogReader, streamMetricsRegistry); var manager = new TaskManager(builder, taskCreator, adminClient, storeChangelogReader); var listener = new StreamsRebalanceListener(manager); log.LogInformation("{LogPrefix}Creating consumer client", logPrefix); var consumer = kafkaSupplier.GetConsumer(configuration.ToConsumerConfig(GetConsumerClientId(customerID)).Wrap(threadId), listener); manager.Consumer = consumer; var thread = new StreamThread(threadId, customerID, manager, consumer, builder, storeChangelogReader, streamMetricsRegistry, configuration); listener.Thread = thread; return(thread); }
internal StreamTask GetTask(string topicName) { StreamTask task = null; var id = builder.GetTaskIdFromPartition(new Confluent.Kafka.TopicPartition(topicName, 0)); if (tasks.ContainsKey(id)) { task = tasks[id]; } else { if (!builder.GetGlobalTopics().Contains(topicName)) { task = new StreamTask("thread-0", id, partitionsByTaskId[id], builder.BuildTopology(id), supplier.GetConsumer(configuration.ToConsumerConfig(), null), configuration, supplier, producer, new MockChangelogRegister(), metricsRegistry); task.InitializeStateStores(); task.InitializeTopology(); task.RestorationIfNeeded(); task.CompleteRestoration(); tasks.Add(id, task); } } return(task); }
internal StreamTask GetTask(string topicName) { StreamTask task; var id = builder.GetTaskIdFromPartition(new Confluent.Kafka.TopicPartition(topicName, 0)); if (tasks.ContainsKey(id)) { task = tasks[id]; } else { task = new StreamTask("thread-0", id, partitionsByTaskId[id], builder.BuildTopology(id), supplier.GetConsumer(configuration.ToConsumerConfig(), null), configuration, supplier, producer); task.InitializeStateStores(); task.InitializeTopology(); tasks.Add(id, task); } return(task); }
internal StreamTask GetTask(string topicName) { StreamTask task; if (tasks.ContainsKey(topicName)) { task = tasks[topicName]; } else { task = new StreamTask("thread-0", new TaskId { Id = id++, Partition = 0, Topic = topicName }, new Confluent.Kafka.TopicPartition(topicName, 0), builder.BuildTopology(topicName), supplier.GetConsumer(configuration.ToConsumerConfig(), null), configuration, supplier, producer); task.InitializeStateStores(); task.InitializeTopology(); tasks.Add(topicName, task); } return(task); }
public SyncPipeOutput(string topic, TimeSpan consumeTimeout, IStreamConfig configuration, SyncProducer producer, CancellationToken token) { this.token = token; topicName = topic; timeout = consumeTimeout; consumer = new SyncConsumer(configuration.ToConsumerConfig($"pipe-output-{configuration.ApplicationId}-{topicName}"), producer); consumer.Subscribe(topicName); }
public void Init() { stateDir = Path.Combine(".", Guid.NewGuid().ToString()); config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-storechangelog-app"; config.StateDir = stateDir; config.PollMs = 100; supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); restoreConsumer = supplier.GetRestoreConsumer(config.ToConsumerConfig()); var topicPart = new TopicPartition("topic", 0); changelogTopic = "store-changelog-topic"; var changelogsTopics = new Dictionary <string, string>(); changelogsTopics.Add("store", changelogTopic); var id = new TaskId { Id = 0, Partition = 0 }; store = new RocksDbKeyValueStore("store"); storeChangelogReader = new StoreChangelogReader(config, restoreConsumer, "thread-0", new StreamMetricsRegistry()); stateMgr = new ProcessorStateManager( id, topicPart.ToSingle(), changelogsTopics, storeChangelogReader, new OffsetCheckpointFile(Path.Combine(config.StateDir, config.ApplicationId, $"{id.Id}-{id.Partition}")) ); Mock <AbstractTask> moq = new Mock <AbstractTask>(); moq.Setup(t => t.Id).Returns(new TaskId { Id = 0, Partition = 0 }); context = new ProcessorContext(moq.Object, config, stateMgr, new StreamMetricsRegistry()); store.Init(context, store); producer.Produce(changelogTopic, CreateMessage(changelogTopic, "key1", "value1")); producer.Produce(changelogTopic, CreateMessage(changelogTopic, "key2", "value2")); }
internal static IThread Create(string threadId, string clientId, InternalTopologyBuilder builder, IStreamConfig configuration, IKafkaSupplier kafkaSupplier, IAdminClient adminClient, int threadInd) { string logPrefix = $"stream-thread[{threadId}] "; var log = Logger.GetLogger(typeof(StreamThread)); var customerID = $"{clientId}-StreamThread-{threadInd}"; IProducer <byte[], byte[]> producer = null; // Due to limitations outlined in KIP-447 (which KIP-447 overcomes), it is // currently necessary to use a separate producer per input partition. The // producerState dictionary is used to keep track of these, and the current // consumed offset. // https://cwiki.apache.org/confluence/display/KAFKA/KIP-447%3A+Producer+scalability+for+exactly+once+semantics // IF Guarantee is AT_LEAST_ONCE, producer is the same of all StreamTasks in this thread, // ELSE one producer by StreamTask. if (configuration.Guarantee == ProcessingGuarantee.AT_LEAST_ONCE) { log.Info($"{logPrefix}Creating shared producer client"); producer = kafkaSupplier.GetProducer(configuration.ToProducerConfig(GetThreadProducerClientId(threadId))); } var taskCreator = new TaskCreator(builder, configuration, threadId, kafkaSupplier, producer); var manager = new TaskManager(builder, taskCreator, adminClient); var listener = new StreamsRebalanceListener(manager); log.Info($"{logPrefix}Creating consumer client"); var consumer = kafkaSupplier.GetConsumer(configuration.ToConsumerConfig(customerID), listener); manager.Consumer = consumer; var thread = new StreamThread(threadId, customerID, manager, consumer, builder, TimeSpan.FromMilliseconds(configuration.PollMs), configuration.CommitIntervalMs); listener.Thread = thread; return(thread); }