public void SetUp() { globalConsumerMock = new Mock <IConsumer <byte[], byte[]> >(); globalStateMaintainerMock = new Mock <IGlobalStateMaintainer>(); streamConfigMock = new Mock <IStreamConfig>(); streamConfigMock.Setup(x => x.PollMs).Returns(1); streamConfigMock.Setup(x => x.CommitIntervalMs).Returns(1); globalStateMaintainerMock.Setup(x => x.Initialize()).Returns(new Dictionary <TopicPartition, long>()); cancellationTokenSource = new CancellationTokenSource(); globalStreamThread = new GlobalStreamThread("global", globalConsumerMock.Object, streamConfigMock.Object, globalStateMaintainerMock.Object); }
/// <summary> /// Create a <see cref="KafkaStream"/> instance with your own <see cref="IKafkaSupplier" /> /// Please DO NOT FORGET to call Close to avoid resources leak ! /// </summary> /// <param name="topology">the topology specifying the computational logic</param> /// <param name="configuration">configuration about this stream</param> /// <param name="kafkaSupplier">the Kafka clients supplier which provides underlying producer and consumer clients for the new <see cref="KafkaStream"/> instance</param> public KafkaStream(Topology topology, IStreamConfig configuration, IKafkaSupplier kafkaSupplier) { this.topology = topology; this.kafkaSupplier = kafkaSupplier; // check if ApplicationId & BootstrapServers has been set if (string.IsNullOrEmpty(configuration.ApplicationId) || string.IsNullOrEmpty(configuration.BootstrapServers)) { throw new StreamConfigException($"Stream configuration is not correct. Please set ApplicationId and BootstrapServers as minimal."); } var processID = Guid.NewGuid(); clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId; logPrefix = $"stream-application[{configuration.ApplicationId}] "; logger.Info($"{logPrefix} Start creation of the stream application with this configuration: {configuration}"); // re-write the physical topology according to the config topology.Builder.RewriteTopology(configuration); // sanity check var processorTopology = topology.Builder.BuildTopology(); int numStreamThreads = topology.Builder.HasNoNonGlobalTopology ? 0 : configuration.NumStreamThreads; threads = new IThread[numStreamThreads]; var threadState = new Dictionary <long, Processors.ThreadState>(); ProcessorTopology globalTaskTopology = topology.Builder.BuildGlobalStateTopology(); bool hasGlobalTopology = globalTaskTopology != null; if (numStreamThreads == 0 && !hasGlobalTopology) { throw new TopologyException("Topology has no stream threads and no global threads, " + "must subscribe to at least one source topic or global table."); } GlobalThreadState globalThreadState = null; if (hasGlobalTopology) { string globalThreadId = $"{clientId}-GlobalStreamThread"; GlobalStreamThreadFactory globalStreamThreadFactory = new GlobalStreamThreadFactory(globalTaskTopology, globalThreadId, kafkaSupplier.GetGlobalConsumer(configuration.ToGlobalConsumerConfig(globalThreadId)), configuration, kafkaSupplier.GetAdmin(configuration.ToAdminConfig(clientId))); globalStreamThread = globalStreamThreadFactory.GetGlobalStreamThread(); globalThreadState = globalStreamThread.State; } List <StreamThreadStateStoreProvider> stateStoreProviders = new List <StreamThreadStateStoreProvider>(); for (int i = 0; i < numStreamThreads; ++i) { var threadId = $"{configuration.ApplicationId.ToLower()}-stream-thread-{i}"; var adminClient = this.kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId(clientId))); threads[i] = StreamThread.Create( threadId, clientId, this.topology.Builder, configuration, this.kafkaSupplier, adminClient, i); threadState.Add(threads[i].Id, threads[i].State); stateStoreProviders.Add(new StreamThreadStateStoreProvider(threads[i], this.topology.Builder)); } var manager = new StreamStateManager(this, threadState, globalThreadState); if (hasGlobalTopology) { globalStreamThread.StateChanged += manager.OnGlobalThreadStateChange; } foreach (var t in threads) { t.StateChanged += manager.OnChange; } var globalStateStoreProvider = new GlobalStateStoreProvider(topology.Builder.GlobalStateStores); queryableStoreProvider = new QueryableStoreProvider(stateStoreProviders, globalStateStoreProvider); StreamState = State.CREATED; }