Ejemplo n.º 1
0
        internal StreamTask GetTask(string topicName)
        {
            StreamTask task = null;
            var        id   = builder.GetTaskIdFromPartition(new Confluent.Kafka.TopicPartition(topicName, 0));

            if (tasks.ContainsKey(id))
            {
                task = tasks[id];
            }
            else
            {
                if (!builder.GetGlobalTopics().Contains(topicName))
                {
                    task = new StreamTask("thread-0",
                                          id,
                                          partitionsByTaskId[id],
                                          builder.BuildTopology(id),
                                          supplier.GetConsumer(configuration.ToConsumerConfig(), null),
                                          configuration,
                                          supplier,
                                          producer,
                                          new MockChangelogRegister(),
                                          metricsRegistry);
                    task.InitializeStateStores();
                    task.InitializeTopology();
                    task.RestorationIfNeeded();
                    task.CompleteRestoration();
                    tasks.Add(id, task);
                }
            }

            return(task);
        }
Ejemplo n.º 2
0
        internal StreamTask GetTask(string topicName)
        {
            StreamTask task;
            var        id = builder.GetTaskIdFromPartition(new Confluent.Kafka.TopicPartition(topicName, 0));

            if (tasks.ContainsKey(id))
            {
                task = tasks[id];
            }
            else
            {
                task = new StreamTask("thread-0",
                                      id,
                                      partitionsByTaskId[id],
                                      builder.BuildTopology(id),
                                      supplier.GetConsumer(configuration.ToConsumerConfig(), null),
                                      configuration,
                                      supplier,
                                      producer);
                task.InitializeStateStores();
                task.InitializeTopology();
                tasks.Add(id, task);
            }
            return(task);
        }
        internal StreamTask GetTask(string topicName)
        {
            StreamTask task;

            if (tasks.ContainsKey(topicName))
            {
                task = tasks[topicName];
            }
            else
            {
                task = new StreamTask("thread-0",
                                      new TaskId {
                    Id = id++, Partition = 0, Topic = topicName
                },
                                      new Confluent.Kafka.TopicPartition(topicName, 0),
                                      builder.BuildTopology(topicName),
                                      supplier.GetConsumer(configuration.ToConsumerConfig(), null),
                                      configuration,
                                      supplier,
                                      producer);
                task.InitializeStateStores();
                task.InitializeTopology();
                tasks.Add(topicName, task);
            }
            return(task);
        }
        public void WriteTopologyTest()
        {
            var builder = new InternalTopologyBuilder();
            List <StreamGraphNode> nodes = new List <StreamGraphNode>();

            RootNode root   = new RootNode();
            var      source = new StreamSourceNode <string, string>(
                "topic",
                "source-01",
                new Stream.Internal.ConsumedInternal <string, string>("source-01", new StringSerDes(),
                                                                      new StringSerDes(), null));

            root.AppendChild(source);
            nodes.Add(source);

            var filterParameters =
                new ProcessorParameters <string, string>(
                    new KStreamFilter <string, string>((k, v) => true, false), "filter-02");
            var filter = new ProcessorGraphNode <string, string>("filter-02", filterParameters);

            source.AppendChild(filter);
            nodes.Add(filter);

            var to = new StreamSinkNode <string, string>(
                new StaticTopicNameExtractor <string, string>("topic2"), "to-03",
                new Stream.Internal.Produced <string, string>(
                    new StringSerDes(),
                    new StringSerDes())
                );

            filter.AppendChild(to);
            nodes.Add(to);

            builder.BuildTopology(root, nodes);

            Assert.IsTrue(root.AllParentsWrittenToTopology);
            Assert.IsTrue(source.AllParentsWrittenToTopology);
            Assert.IsTrue(filter.AllParentsWrittenToTopology);
            Assert.IsTrue(to.AllParentsWrittenToTopology);

            var topology = builder.BuildTopology();

            Assert.IsTrue(topology.SourceOperators.ContainsKey("topic"));
            Assert.IsTrue(topology.ProcessorOperators.ContainsKey("filter-02"));
            Assert.IsTrue(topology.SinkOperators.ContainsKey("topic2"));
        }
        internal TopologyTestDriver(InternalTopologyBuilder builder, IStreamConfig config, Mode mode, IKafkaSupplier supplier)
        {
            topologyBuilder = builder;
            configuration   = config;

            // ONLY 1 thread for test driver (use only for ASYNC_CLUSTER_IN_MEMORY)
            configuration.NumStreamThreads = 1;
            configuration.Guarantee        = ProcessingGuarantee.AT_LEAST_ONCE;

            topicConfiguration = config.Clone();
            topicConfiguration.ApplicationId = $"test-driver-{configuration.ApplicationId}";

            var clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{configuration.ApplicationId.ToLower()}-{Guid.NewGuid()}" : configuration.ClientId;

            // sanity check
            topologyBuilder.BuildTopology();

            topologyBuilder.RewriteTopology(configuration);

            switch (mode)
            {
            case Mode.SYNC_TASK:
                behavior = new TaskSynchronousTopologyDriver(
                    clientId,
                    topologyBuilder,
                    configuration,
                    topicConfiguration,
                    supplier,
                    tokenSource.Token);
                break;

            case Mode.ASYNC_CLUSTER_IN_MEMORY:
                behavior = new ClusterInMemoryTopologyDriver(
                    clientId,
                    topologyBuilder,
                    configuration,
                    topicConfiguration,
                    supplier,
                    tokenSource.Token);
                break;
            }

            behavior.StartDriver();
        }
Ejemplo n.º 6
0
        public ClusterInMemoryTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, TimeSpan startTimeout, CancellationToken token)
        {
            this.startTimeout           = startTimeout;
            this.configuration          = configuration;
            this.configuration.ClientId = clientId;
            this.topicConfiguration     = topicConfiguration;
            this.token = token;

            kafkaSupplier = new MockKafkaSupplier();
            pipeBuilder   = new KafkaPipeBuilder(kafkaSupplier);

            // ONLY FOR CHECK IF TOLOGY IS CORRECT
            topologyBuilder.BuildTopology();

            threadTopology = StreamThread.Create(
                $"{this.configuration.ApplicationId.ToLower()}-stream-thread-0",
                clientId,
                topologyBuilder,
                this.configuration,
                kafkaSupplier,
                kafkaSupplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin")),
                0);
        }
Ejemplo n.º 7
0
 internal void Build()
 {
     internalTopologyBuilder.BuildTopology(root, nodes);
 }