public GlobalStreamThread GetGlobalStreamThread()
        {
            var stateManager = new GlobalStateManager(topology, adminClient, configuration);
            var context      = new GlobalProcessorContext(configuration, stateManager);

            stateManager.SetGlobalProcessorContext(context);
            var globalStateUpdateTask = new GlobalStateUpdateTask(stateManager, topology, context);

            return(new GlobalStreamThread(threadClientId, globalConsumer, configuration, globalStateUpdateTask));
        }
Beispiel #2
0
        public TaskSynchronousTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder,
                                             IStreamConfig configuration, IStreamConfig topicConfiguration, IKafkaSupplier supplier,
                                             CancellationToken token)
        {
            this.configuration          = configuration;
            this.configuration.ClientId = clientId;
            this.topicConfiguration     = topicConfiguration;
            metricsRegistry             = new StreamMetricsRegistry(clientId, MetricsRecordingLevel.DEBUG);

            this.token    = token;
            builder       = topologyBuilder;
            this.supplier = supplier ?? new SyncKafkaSupplier();
            this.supplier.MetricsRegistry = metricsRegistry;
            producer = this.supplier.GetProducer(configuration.ToProducerConfig()) as SyncProducer;

            foreach (var sourceTopic in builder
                     .GetSourceTopics())
            {
                var part   = new TopicPartition(sourceTopic, 0);
                var taskId = builder.GetTaskIdFromPartition(part);
                if (partitionsByTaskId.ContainsKey(taskId))
                {
                    partitionsByTaskId[taskId].Add(part);
                }
                else
                {
                    partitionsByTaskId.Add(taskId, new List <TopicPartition> {
                        part
                    });
                }
            }

            ProcessorTopology globalTaskTopology = topologyBuilder.BuildGlobalStateTopology();

            hasGlobalTopology = globalTaskTopology != null;
            if (hasGlobalTopology)
            {
                var globalConsumer =
                    this.supplier.GetGlobalConsumer(configuration.ToGlobalConsumerConfig($"{clientId}-global-consumer"));
                var adminClient  = this.supplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin"));
                var stateManager =
                    new GlobalStateManager(globalConsumer, globalTaskTopology, adminClient, configuration);
                globalProcessorContext = new GlobalProcessorContext(configuration, stateManager, metricsRegistry);
                stateManager.SetGlobalProcessorContext(globalProcessorContext);
                globalTask = new GlobalStateUpdateTask(stateManager, globalTaskTopology, globalProcessorContext);

                globalTask.Initialize();
            }
        }
Beispiel #3
0
        public void Setup()
        {
            contextMock              = new Mock <GlobalProcessorContext>(null, null, new StreamMetricsRegistry());
            stateManagerMock         = new Mock <IGlobalStateManager>();
            processorMock            = new Mock <IProcessor>();
            sourceProcessorMock      = new Mock <ISourceProcessor>();
            otherSourceProcessorMock = new Mock <ISourceProcessor>();

            sourceProcessorMock.Setup(x => x.TopicName).Returns("topic1");
            otherSourceProcessorMock.Setup(x => x.TopicName).Returns("topic2");

            var sourceProcessors = new Dictionary <string, IProcessor>()
            {
                { "source1", sourceProcessorMock.Object }, { "source2", otherSourceProcessorMock.Object }
            };
            var processors = new Dictionary <string, IProcessor>()
            {
                { "processor", processorMock.Object }
            };

            var storesToTopics = new Dictionary <string, string>()
            {
                { "store1", "topic1" },
                { "store2", "topic2" },
            };

            stateManagerMock.Setup(x => x.Initialize()).Returns(new HashSet <string>()
            {
                "store1", "store2"
            });

            var topology = new ProcessorTopology(
                null,
                sourceProcessors,
                null,
                processors,
                null,
                null,
                storesToTopics,
                null);

            globalStateUpdateTask = new GlobalStateUpdateTask(stateManagerMock.Object, topology, contextMock.Object);
        }
Beispiel #4
0
 public SyncPipeBuilder(GlobalStateUpdateTask globalTask)
 {
     publisher = new GlobalTaskPublisher(globalTask);
 }
Beispiel #5
0
 public GlobalTaskPublisher(GlobalStateUpdateTask globalTask)
 {
     this.globalTask = globalTask;
 }