public TaskSynchronousTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, IKafkaSupplier supplier, CancellationToken token) { this.configuration = configuration; this.configuration.ClientId = clientId; this.topicConfiguration = topicConfiguration; metricsRegistry = new StreamMetricsRegistry(clientId, MetricsRecordingLevel.DEBUG); this.token = token; builder = topologyBuilder; this.supplier = supplier ?? new SyncKafkaSupplier(); this.supplier.MetricsRegistry = metricsRegistry; producer = this.supplier.GetProducer(configuration.ToProducerConfig()) as SyncProducer; foreach (var sourceTopic in builder .GetSourceTopics()) { var part = new TopicPartition(sourceTopic, 0); var taskId = builder.GetTaskIdFromPartition(part); if (partitionsByTaskId.ContainsKey(taskId)) { partitionsByTaskId[taskId].Add(part); } else { partitionsByTaskId.Add(taskId, new List <TopicPartition> { part }); } } ProcessorTopology globalTaskTopology = topologyBuilder.BuildGlobalStateTopology(); hasGlobalTopology = globalTaskTopology != null; if (hasGlobalTopology) { var globalConsumer = this.supplier.GetGlobalConsumer(configuration.ToGlobalConsumerConfig($"{clientId}-global-consumer")); var adminClient = this.supplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin")); var stateManager = new GlobalStateManager(globalConsumer, globalTaskTopology, adminClient, configuration); globalProcessorContext = new GlobalProcessorContext(configuration, stateManager, metricsRegistry); stateManager.SetGlobalProcessorContext(globalProcessorContext); globalTask = new GlobalStateUpdateTask(stateManager, globalTaskTopology, globalProcessorContext); globalTask.Initialize(); } }
/// <summary> /// Create a <see cref="KafkaStream"/> instance with your own <see cref="IKafkaSupplier" /> /// Please DO NOT FORGET to call Close to avoid resources leak ! /// </summary> /// <param name="topology">the topology specifying the computational logic</param> /// <param name="configuration">configuration about this stream</param> /// <param name="kafkaSupplier">the Kafka clients supplier which provides underlying producer and consumer clients for the new <see cref="KafkaStream"/> instance</param> public KafkaStream(Topology topology, IStreamConfig configuration, IKafkaSupplier kafkaSupplier) { this.topology = topology; this.kafkaSupplier = kafkaSupplier; // check if ApplicationId & BootstrapServers has been set if (string.IsNullOrEmpty(configuration.ApplicationId) || string.IsNullOrEmpty(configuration.BootstrapServers)) { throw new StreamConfigException($"Stream configuration is not correct. Please set ApplicationId and BootstrapServers as minimal."); } var processID = Guid.NewGuid(); clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId; logPrefix = $"stream-application[{configuration.ApplicationId}] "; logger.Info($"{logPrefix} Start creation of the stream application with this configuration: {configuration}"); // re-write the physical topology according to the config topology.Builder.RewriteTopology(configuration); // sanity check var processorTopology = topology.Builder.BuildTopology(); int numStreamThreads = topology.Builder.HasNoNonGlobalTopology ? 0 : configuration.NumStreamThreads; threads = new IThread[numStreamThreads]; var threadState = new Dictionary <long, Processors.ThreadState>(); ProcessorTopology globalTaskTopology = topology.Builder.BuildGlobalStateTopology(); bool hasGlobalTopology = globalTaskTopology != null; if (numStreamThreads == 0 && !hasGlobalTopology) { throw new TopologyException("Topology has no stream threads and no global threads, " + "must subscribe to at least one source topic or global table."); } GlobalThreadState globalThreadState = null; if (hasGlobalTopology) { string globalThreadId = $"{clientId}-GlobalStreamThread"; GlobalStreamThreadFactory globalStreamThreadFactory = new GlobalStreamThreadFactory(globalTaskTopology, globalThreadId, kafkaSupplier.GetGlobalConsumer(configuration.ToGlobalConsumerConfig(globalThreadId)), configuration, kafkaSupplier.GetAdmin(configuration.ToAdminConfig(clientId))); globalStreamThread = globalStreamThreadFactory.GetGlobalStreamThread(); globalThreadState = globalStreamThread.State; } List <StreamThreadStateStoreProvider> stateStoreProviders = new List <StreamThreadStateStoreProvider>(); for (int i = 0; i < numStreamThreads; ++i) { var threadId = $"{configuration.ApplicationId.ToLower()}-stream-thread-{i}"; var adminClient = this.kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId(clientId))); threads[i] = StreamThread.Create( threadId, clientId, this.topology.Builder, configuration, this.kafkaSupplier, adminClient, i); threadState.Add(threads[i].Id, threads[i].State); stateStoreProviders.Add(new StreamThreadStateStoreProvider(threads[i], this.topology.Builder)); } var manager = new StreamStateManager(this, threadState, globalThreadState); if (hasGlobalTopology) { globalStreamThread.StateChanged += manager.OnGlobalThreadStateChange; } foreach (var t in threads) { t.StateChanged += manager.OnChange; } var globalStateStoreProvider = new GlobalStateStoreProvider(topology.Builder.GlobalStateStores); queryableStoreProvider = new QueryableStoreProvider(stateStoreProviders, globalStateStoreProvider); StreamState = State.CREATED; }