internal StreamTask GetTask(string topicName) { StreamTask task = null; var id = builder.GetTaskIdFromPartition(new Confluent.Kafka.TopicPartition(topicName, 0)); if (tasks.ContainsKey(id)) { task = tasks[id]; } else { if (!builder.GetGlobalTopics().Contains(topicName)) { task = new StreamTask("thread-0", id, partitionsByTaskId[id], builder.BuildTopology(id), supplier.GetConsumer(configuration.ToConsumerConfig(), null), configuration, supplier, producer, new MockChangelogRegister(), metricsRegistry); task.InitializeStateStores(); task.InitializeTopology(); task.RestorationIfNeeded(); task.CompleteRestoration(); tasks.Add(id, task); } } return(task); }
internal StreamTask GetTask(string topicName) { StreamTask task; var id = builder.GetTaskIdFromPartition(new Confluent.Kafka.TopicPartition(topicName, 0)); if (tasks.ContainsKey(id)) { task = tasks[id]; } else { task = new StreamTask("thread-0", id, partitionsByTaskId[id], builder.BuildTopology(id), supplier.GetConsumer(configuration.ToConsumerConfig(), null), configuration, supplier, producer); task.InitializeStateStores(); task.InitializeTopology(); tasks.Add(id, task); } return(task); }
internal StreamTask GetTask(string topicName) { StreamTask task; if (tasks.ContainsKey(topicName)) { task = tasks[topicName]; } else { task = new StreamTask("thread-0", new TaskId { Id = id++, Partition = 0, Topic = topicName }, new Confluent.Kafka.TopicPartition(topicName, 0), builder.BuildTopology(topicName), supplier.GetConsumer(configuration.ToConsumerConfig(), null), configuration, supplier, producer); task.InitializeStateStores(); task.InitializeTopology(); tasks.Add(topicName, task); } return(task); }
public void WriteTopologyTest() { var builder = new InternalTopologyBuilder(); List <StreamGraphNode> nodes = new List <StreamGraphNode>(); RootNode root = new RootNode(); var source = new StreamSourceNode <string, string>( "topic", "source-01", new Stream.Internal.ConsumedInternal <string, string>("source-01", new StringSerDes(), new StringSerDes(), null)); root.AppendChild(source); nodes.Add(source); var filterParameters = new ProcessorParameters <string, string>( new KStreamFilter <string, string>((k, v) => true, false), "filter-02"); var filter = new ProcessorGraphNode <string, string>("filter-02", filterParameters); source.AppendChild(filter); nodes.Add(filter); var to = new StreamSinkNode <string, string>( new StaticTopicNameExtractor <string, string>("topic2"), "to-03", new Stream.Internal.Produced <string, string>( new StringSerDes(), new StringSerDes()) ); filter.AppendChild(to); nodes.Add(to); builder.BuildTopology(root, nodes); Assert.IsTrue(root.AllParentsWrittenToTopology); Assert.IsTrue(source.AllParentsWrittenToTopology); Assert.IsTrue(filter.AllParentsWrittenToTopology); Assert.IsTrue(to.AllParentsWrittenToTopology); var topology = builder.BuildTopology(); Assert.IsTrue(topology.SourceOperators.ContainsKey("topic")); Assert.IsTrue(topology.ProcessorOperators.ContainsKey("filter-02")); Assert.IsTrue(topology.SinkOperators.ContainsKey("topic2")); }
internal TopologyTestDriver(InternalTopologyBuilder builder, IStreamConfig config, Mode mode, IKafkaSupplier supplier) { topologyBuilder = builder; configuration = config; // ONLY 1 thread for test driver (use only for ASYNC_CLUSTER_IN_MEMORY) configuration.NumStreamThreads = 1; configuration.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; topicConfiguration = config.Clone(); topicConfiguration.ApplicationId = $"test-driver-{configuration.ApplicationId}"; var clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{configuration.ApplicationId.ToLower()}-{Guid.NewGuid()}" : configuration.ClientId; // sanity check topologyBuilder.BuildTopology(); topologyBuilder.RewriteTopology(configuration); switch (mode) { case Mode.SYNC_TASK: behavior = new TaskSynchronousTopologyDriver( clientId, topologyBuilder, configuration, topicConfiguration, supplier, tokenSource.Token); break; case Mode.ASYNC_CLUSTER_IN_MEMORY: behavior = new ClusterInMemoryTopologyDriver( clientId, topologyBuilder, configuration, topicConfiguration, supplier, tokenSource.Token); break; } behavior.StartDriver(); }
public ClusterInMemoryTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, TimeSpan startTimeout, CancellationToken token) { this.startTimeout = startTimeout; this.configuration = configuration; this.configuration.ClientId = clientId; this.topicConfiguration = topicConfiguration; this.token = token; kafkaSupplier = new MockKafkaSupplier(); pipeBuilder = new KafkaPipeBuilder(kafkaSupplier); // ONLY FOR CHECK IF TOLOGY IS CORRECT topologyBuilder.BuildTopology(); threadTopology = StreamThread.Create( $"{this.configuration.ApplicationId.ToLower()}-stream-thread-0", clientId, topologyBuilder, this.configuration, kafkaSupplier, kafkaSupplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin")), 0); }
internal void Build() { internalTopologyBuilder.BuildTopology(root, nodes); }