public override void WriteToTopology(InternalTopologyBuilder builder) { if (ProcessorParameters != null) { builder.AddInternalTopic(RepartitionTopic, NumberOfPartition); builder.AddProcessor(ProcessorParameters.ProcessorName, ProcessorParameters.Processor, ParentNodeNames()); builder.AddSinkOperator( new StaticTopicNameExtractor <K, V>(RepartitionTopic), SinkName, Produced <K, V> .Create(KeySerdes, ValueSerdes).WithPartitioner(StreamPartitioner), ProcessorParameters.ProcessorName); builder.AddSourceOperator( RepartitionTopic, SourceName, new ConsumedInternal <K, V>(SourceName, KeySerdes, ValueSerdes, new FailOnInvalidTimestamp())); } else { builder.AddInternalTopic(RepartitionTopic, NumberOfPartition); builder.AddSinkOperator( new StaticTopicNameExtractor <K, V>(RepartitionTopic), SinkName, Produced <K, V> .Create(KeySerdes, ValueSerdes).WithPartitioner(StreamPartitioner), ParentNodeNames()); builder.AddSourceOperator( RepartitionTopic, SourceName, new ConsumedInternal <K, V>(SourceName, KeySerdes, ValueSerdes, new FailOnInvalidTimestamp())); } }
public TaskSynchronousTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, IKafkaSupplier supplier, CancellationToken token) { this.configuration = configuration; this.configuration.ClientId = clientId; this.topicConfiguration = topicConfiguration; this.token = token; builder = topologyBuilder; this.supplier = supplier ?? new SyncKafkaSupplier(); producer = this.supplier.GetProducer(configuration.ToProducerConfig()) as SyncProducer; foreach (var sourceTopic in builder.GetSourceTopics().Union(builder.GetGlobalTopics())) { var part = new TopicPartition(sourceTopic, 0); var taskId = builder.GetTaskIdFromPartition(part); if (partitionsByTaskId.ContainsKey(taskId)) { partitionsByTaskId[taskId].Add(part); } else { partitionsByTaskId.Add(taskId, new List <TopicPartition> { part }); } } }
private StreamThread(string threadId, string clientId, TaskManager manager, IConsumer <byte[], byte[]> consumer, InternalTopologyBuilder builder, IChangelogReader storeChangelogReader, StreamMetricsRegistry streamMetricsRegistry, TimeSpan timeSpan, long commitInterval) { this.manager = manager; this.consumer = consumer; this.builder = builder; consumeTimeout = timeSpan; this.threadId = threadId; this.clientId = clientId; logPrefix = $"stream-thread[{threadId}] "; commitTimeMs = commitInterval; changelogReader = storeChangelogReader; this.streamMetricsRegistry = streamMetricsRegistry; thread = new Thread(Run); thread.Name = this.threadId; Name = this.threadId; State = ThreadState.CREATED; commitSensor = ThreadMetrics.CommitSensor(threadId, streamMetricsRegistry); pollSensor = ThreadMetrics.PollSensor(threadId, streamMetricsRegistry); pollRecordsSensor = ThreadMetrics.PollRecordsSensor(threadId, streamMetricsRegistry); pollRatioSensor = ThreadMetrics.PollRatioSensor(threadId, streamMetricsRegistry); processLatencySensor = ThreadMetrics.ProcessLatencySensor(threadId, streamMetricsRegistry); processRecordsSensor = ThreadMetrics.ProcessRecordsSensor(threadId, streamMetricsRegistry); processRateSensor = ThreadMetrics.ProcessRateSensor(threadId, streamMetricsRegistry); processRatioSensor = ThreadMetrics.ProcessRatioSensor(threadId, streamMetricsRegistry); commitRatioSensor = ThreadMetrics.CommitRatioSensor(threadId, streamMetricsRegistry); }
private StreamThread(string threadId, string clientId, TaskManager manager, IConsumer <byte[], byte[]> consumer, InternalTopologyBuilder builder, IChangelogReader storeChangelogReader, StreamMetricsRegistry streamMetricsRegistry, IStreamConfig configuration) : this(threadId, clientId, manager, consumer, builder, storeChangelogReader, streamMetricsRegistry, TimeSpan.FromMilliseconds(configuration.PollMs), configuration.CommitIntervalMs) { streamConfig = configuration; }
private TopologyTestDriver(InternalTopologyBuilder builder, IStreamConfig config) { this.topologyBuilder = builder; this.configuration = config; // ONLY 1 thread for test driver this.configuration.NumStreamThreads = 1; this.configuration.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; this.topicConfiguration = config.Clone(); this.topicConfiguration.ApplicationId = $"test-driver-{this.configuration.ApplicationId}"; var processID = Guid.NewGuid(); var clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{this.configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId; this.configuration.ClientId = clientId; kafkaSupplier = new MockKafkaSupplier(); pipeBuilder = new PipeBuilder(kafkaSupplier); this.processorTopology = this.topologyBuilder.BuildTopology(); this.threadTopology = StreamThread.Create( $"{this.configuration.ApplicationId.ToLower()}-stream-thread-0", clientId, builder, config, kafkaSupplier, kafkaSupplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin")), 0); RunDriver(); }
public override void WriteToTopology(InternalTopologyBuilder builder) { // TODO: we assume source KTables can only be timestamped-key-value stores for now. // should be expanded for other types of stores as well. StoreBuilder <State.TimestampedKeyValueStore <K, V> > storeBuilder = new TimestampedKeyValueStoreMaterializer <K, V>(materialized as Materialized <K, V, IKeyValueStore <Bytes, byte[]> >).Materialize(); if (isGlobalKTable) { builder.AddGlobalStore(topicName, storeBuilder, sourceName, consumed, processorParameters); } else { builder.AddSourceOperator(this.topicName, sourceName, consumed); builder.AddProcessor(processorParameters.ProcessorName, processorParameters.Processor, sourceName); //// only add state store if the source KTable should be materialized KTableSource <K, V> ktableSource = (KTableSource <K, V>)processorParameters.Processor; if (ktableSource.QueryableName != null) { builder.AddStateStore(storeBuilder, this.streamGraphNode); // TODO : //if (shouldReuseSourceTopicForChangelog) //{ // storeBuilder.withLoggingDisabled(); // topologyBuilder.connectSourceStoreAndTopic(storeBuilder.name(), topicName); //} } } }
public TaskSynchronousTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, CancellationToken token) { this.configuration = configuration; this.configuration.ClientId = clientId; this.topicConfiguration = topicConfiguration; this.token = token; builder = topologyBuilder; supplier = new SyncKafkaSupplier(); producer = supplier.GetProducer(configuration.ToProducerConfig()) as SyncProducer; }
public override void WriteToTopology(InternalTopologyBuilder builder) { // Stream - Table join (Global or KTable) builder.AddProcessor(processorParameters.ProcessorName, processorParameters.Processor, ParentNodeNames()); // Steam - KTable join only if (!string.IsNullOrEmpty(otherJoinSideNodeName)) { builder.ConnectProcessorAndStateStore(processorParameters.ProcessorName, storeNames); } }
public override void WriteToTopology(InternalTopologyBuilder builder) { string leftProcessorName = JoinLeftParams.ProcessorName; string rightProcessorName = JoinRightParams.ProcessorName; string windowedLeftProcessorName = windowedLeftParams.ProcessorName; string windowedRightProcessorName = windowedRightParams.ProcessorName; builder.AddProcessor(leftProcessorName, JoinLeftParams.Processor, windowedLeftProcessorName); builder.AddProcessor(rightProcessorName, JoinRightParams.Processor, windowedRightProcessorName); builder.AddProcessor(JoinMergeParams.ProcessorName, JoinMergeParams.Processor, leftProcessorName, rightProcessorName); builder.AddStateStore(windowedLeftStoreBuilder, windowedLeftProcessorName, rightProcessorName); builder.AddStateStore(windowedRightStoreBuilder, windowedRightProcessorName, leftProcessorName); }
public override void WriteToTopology(InternalTopologyBuilder builder) { builder.AddProcessor(ProcessorParameters.ProcessorName, ProcessorParameters.Processor, ParentNodeNames()); if (storeNames != null && storeNames.Length > 0) { builder.ConnectProcessorAndStateStore(ProcessorParameters.ProcessorName, storeNames); } if (storeBuilder != null) { builder.AddStateStore(storeBuilder, ProcessorParameters.ProcessorName); } }
public override void WriteToTopology(InternalTopologyBuilder builder) { builder.AddProcessor(JoinLeftParams.ProcessorName, JoinLeftParams.Processor, LeftJoinSideName); builder.AddProcessor(JoinRightParams.ProcessorName, JoinRightParams.Processor, RightJoinSideName); builder.AddProcessor(JoinMergeParams.ProcessorName, JoinMergeProcessorSupplier, JoinLeftParams.ProcessorName, JoinRightParams.ProcessorName); builder.ConnectProcessorAndStateStore(JoinLeftParams.ProcessorName, leftStoreNames); builder.ConnectProcessorAndStateStore(JoinRightParams.ProcessorName, rightStoreNames); if (storeBuilder != null) { builder.AddStateStore(storeBuilder, JoinMergeParams.ProcessorName); } }
internal static IThread Create(string threadId, string clientId, InternalTopologyBuilder builder, StreamMetricsRegistry streamMetricsRegistry, IStreamConfig configuration, IKafkaSupplier kafkaSupplier, IAdminClient adminClient, int threadInd) { string logPrefix = $"stream-thread[{threadId}] "; var log = Logger.GetLogger(typeof(StreamThread)); var customerID = $"{clientId}-StreamThread-{threadInd}"; IProducer <byte[], byte[]> producer = null; // TODO : remove this limitations depends version of Kafka Cluster // Due to limitations outlined in KIP-447 (which KIP-447 overcomes), it is // currently necessary to use a separate producer per input partition. The // producerState dictionary is used to keep track of these, and the current // consumed offset. // https://cwiki.apache.org/confluence/display/KAFKA/KIP-447%3A+Producer+scalability+for+exactly+once+semantics // IF Guarantee is AT_LEAST_ONCE, producer is the same of all StreamTasks in this thread, // ELSE one producer by StreamTask. if (configuration.Guarantee == ProcessingGuarantee.AT_LEAST_ONCE) { log.LogInformation("{LogPrefix}Creating shared producer client", logPrefix); producer = kafkaSupplier.GetProducer(configuration.ToProducerConfig(GetThreadProducerClientId(threadId)).Wrap(threadId)); } var restoreConfig = configuration.ToConsumerConfig(GetRestoreConsumerClientId(customerID)); restoreConfig.GroupId = $"{configuration.ApplicationId}-restore-group"; var restoreConsumer = kafkaSupplier.GetRestoreConsumer(restoreConfig); var storeChangelogReader = new StoreChangelogReader( configuration, restoreConsumer, threadId, streamMetricsRegistry); var taskCreator = new TaskCreator(builder, configuration, threadId, kafkaSupplier, producer, storeChangelogReader, streamMetricsRegistry); var manager = new TaskManager(builder, taskCreator, adminClient, storeChangelogReader); var listener = new StreamsRebalanceListener(manager); log.LogInformation("{LogPrefix}Creating consumer client", logPrefix); var consumer = kafkaSupplier.GetConsumer(configuration.ToConsumerConfig(GetConsumerClientId(customerID)).Wrap(threadId), listener); manager.Consumer = consumer; var thread = new StreamThread(threadId, customerID, manager, consumer, builder, storeChangelogReader, streamMetricsRegistry, configuration); listener.Thread = thread; return(thread); }
public TaskSynchronousTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, IKafkaSupplier supplier, CancellationToken token) { this.configuration = configuration; this.configuration.ClientId = clientId; this.topicConfiguration = topicConfiguration; metricsRegistry = new StreamMetricsRegistry(clientId, MetricsRecordingLevel.DEBUG); this.token = token; builder = topologyBuilder; this.supplier = supplier ?? new SyncKafkaSupplier(); this.supplier.MetricsRegistry = metricsRegistry; producer = this.supplier.GetProducer(configuration.ToProducerConfig()) as SyncProducer; foreach (var sourceTopic in builder .GetSourceTopics()) { var part = new TopicPartition(sourceTopic, 0); var taskId = builder.GetTaskIdFromPartition(part); if (partitionsByTaskId.ContainsKey(taskId)) { partitionsByTaskId[taskId].Add(part); } else { partitionsByTaskId.Add(taskId, new List <TopicPartition> { part }); } } ProcessorTopology globalTaskTopology = topologyBuilder.BuildGlobalStateTopology(); hasGlobalTopology = globalTaskTopology != null; if (hasGlobalTopology) { var globalConsumer = this.supplier.GetGlobalConsumer(configuration.ToGlobalConsumerConfig($"{clientId}-global-consumer")); var adminClient = this.supplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin")); var stateManager = new GlobalStateManager(globalConsumer, globalTaskTopology, adminClient, configuration); globalProcessorContext = new GlobalProcessorContext(configuration, stateManager, metricsRegistry); stateManager.SetGlobalProcessorContext(globalProcessorContext); globalTask = new GlobalStateUpdateTask(stateManager, globalTaskTopology, globalProcessorContext); globalTask.Initialize(); } }
private StreamThread(string threadId, TaskManager manager, IConsumer <byte[], byte[]> consumer, InternalTopologyBuilder builder, TimeSpan timeSpan, long commitInterval) { this.manager = manager; this.consumer = consumer; this.builder = builder; consumeTimeout = timeSpan; logPrefix = $"stream-thread[{threadId}] "; commitTimeMs = commitInterval; thread = new Thread(Run); thread.Name = threadId; Name = threadId; State = ThreadState.CREATED; }
public void WriteTopologyTest() { var builder = new InternalTopologyBuilder(); List <StreamGraphNode> nodes = new List <StreamGraphNode>(); RootNode root = new RootNode(); var source = new StreamSourceNode <string, string>( "topic", "source-01", new Stream.Internal.ConsumedInternal <string, string>("source-01", new StringSerDes(), new StringSerDes(), null)); root.AppendChild(source); nodes.Add(source); var filterParameters = new ProcessorParameters <string, string>( new KStreamFilter <string, string>((k, v) => true, false), "filter-02"); var filter = new ProcessorGraphNode <string, string>("filter-02", filterParameters); source.AppendChild(filter); nodes.Add(filter); var to = new StreamSinkNode <string, string>( new StaticTopicNameExtractor <string, string>("topic2"), "to-03", new Stream.Internal.Produced <string, string>( new StringSerDes(), new StringSerDes()) ); filter.AppendChild(to); nodes.Add(to); builder.BuildTopology(root, nodes); Assert.IsTrue(root.AllParentsWrittenToTopology); Assert.IsTrue(source.AllParentsWrittenToTopology); Assert.IsTrue(filter.AllParentsWrittenToTopology); Assert.IsTrue(to.AllParentsWrittenToTopology); var topology = builder.BuildTopology(); Assert.IsTrue(topology.SourceOperators.ContainsKey("topic")); Assert.IsTrue(topology.ProcessorOperators.ContainsKey("filter-02")); Assert.IsTrue(topology.SinkOperators.ContainsKey("topic2")); }
internal TopologyTestDriver(InternalTopologyBuilder builder, IStreamConfig config, Mode mode, IKafkaSupplier supplier) { topologyBuilder = builder; configuration = config; // ONLY 1 thread for test driver (use only for ASYNC_CLUSTER_IN_MEMORY) configuration.NumStreamThreads = 1; configuration.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; topicConfiguration = config.Clone(); topicConfiguration.ApplicationId = $"test-driver-{configuration.ApplicationId}"; var clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{configuration.ApplicationId.ToLower()}-{Guid.NewGuid()}" : configuration.ClientId; // sanity check topologyBuilder.BuildTopology(); topologyBuilder.RewriteTopology(configuration); switch (mode) { case Mode.SYNC_TASK: behavior = new TaskSynchronousTopologyDriver( clientId, topologyBuilder, configuration, topicConfiguration, supplier, tokenSource.Token); break; case Mode.ASYNC_CLUSTER_IN_MEMORY: behavior = new ClusterInMemoryTopologyDriver( clientId, topologyBuilder, configuration, topicConfiguration, supplier, tokenSource.Token); break; } behavior.StartDriver(); }
public ClusterInMemoryTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, TimeSpan startTimeout, CancellationToken token) { this.startTimeout = startTimeout; this.configuration = configuration; this.configuration.ClientId = clientId; this.topicConfiguration = topicConfiguration; this.token = token; kafkaSupplier = new MockKafkaSupplier(); pipeBuilder = new KafkaPipeBuilder(kafkaSupplier); // ONLY FOR CHECK IF TOLOGY IS CORRECT topologyBuilder.BuildTopology(); threadTopology = StreamThread.Create( $"{this.configuration.ApplicationId.ToLower()}-stream-thread-0", clientId, topologyBuilder, this.configuration, kafkaSupplier, kafkaSupplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin")), 0); }
internal static IThread Create(string threadId, string clientId, InternalTopologyBuilder builder, IStreamConfig configuration, IKafkaSupplier kafkaSupplier, IAdminClient adminClient, int threadInd) { string logPrefix = $"stream-thread[{threadId}] "; var log = Logger.GetLogger(typeof(StreamThread)); var customerID = $"{clientId}-StreamThread-{threadInd}"; IProducer <byte[], byte[]> producer = null; // Due to limitations outlined in KIP-447 (which KIP-447 overcomes), it is // currently necessary to use a separate producer per input partition. The // producerState dictionary is used to keep track of these, and the current // consumed offset. // https://cwiki.apache.org/confluence/display/KAFKA/KIP-447%3A+Producer+scalability+for+exactly+once+semantics // IF Guarantee is AT_LEAST_ONCE, producer is the same of all StreamTasks in this thread, // ELSE one producer by StreamTask. if (configuration.Guarantee == ProcessingGuarantee.AT_LEAST_ONCE) { log.Info($"{logPrefix}Creating shared producer client"); producer = kafkaSupplier.GetProducer(configuration.ToProducerConfig(GetThreadProducerClientId(threadId))); } var taskCreator = new TaskCreator(builder, configuration, threadId, kafkaSupplier, producer); var manager = new TaskManager(builder, taskCreator, adminClient); var listener = new StreamsRebalanceListener(manager); log.Info($"{logPrefix}Creating consumer client"); var consumer = kafkaSupplier.GetConsumer(configuration.ToConsumerConfig(customerID), listener); manager.Consumer = consumer; var thread = new StreamThread(threadId, customerID, manager, consumer, builder, TimeSpan.FromMilliseconds(configuration.PollMs), configuration.CommitIntervalMs); listener.Thread = thread; return(thread); }
public ClusterInMemoryTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, CancellationToken token) : this(clientId, topologyBuilder, configuration, topicConfiguration, TimeSpan.FromSeconds(30), token) { }
private StreamThread(string threadId, TaskManager manager, IConsumer <byte[], byte[]> consumer, InternalTopologyBuilder builder, IStreamConfig configuration) : this(threadId, manager, consumer, builder, TimeSpan.FromMilliseconds(configuration.PollMs), configuration.CommitIntervalMs) { streamConfig = configuration; }
public TaskSynchronousTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, CancellationToken token) : this(clientId, topologyBuilder, configuration, topicConfiguration, null, token) { }
public override void WriteToTopology(InternalTopologyBuilder builder) { }
public override void WriteToTopology(InternalTopologyBuilder builder) { builder.AddSourceOperator(topicName, this.streamGraphNode, consumed); }
private TopologyTestDriver(InternalTopologyBuilder builder, IStreamConfig config, Mode mode) : this(builder, config, mode, null) { }
internal InternalStreamBuilder(InternalTopologyBuilder internalTopologyBuilder) { this.internalTopologyBuilder = internalTopologyBuilder; }
public ClusterInMemoryTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, TimeSpan startTimeout, CancellationToken token) : this(clientId, topologyBuilder, configuration, topicConfiguration, startTimeout, null, token) { }
/// <summary> /// Constructor without arguments /// </summary> public StreamBuilder() { internalTopologyBuilder = topology.Builder; internalStreamBuilder = new InternalStreamBuilder(internalTopologyBuilder); }
public StreamThreadStateStoreProvider(IThread streamThread, InternalTopologyBuilder internalTopologyBuilder) { this.streamThread = streamThread; this.internalTopologyBuilder = internalTopologyBuilder; }
public abstract void WriteToTopology(InternalTopologyBuilder builder);