public ProcessorStateManager( TaskId taskId, IEnumerable <TopicPartition> partition, IDictionary <string, string> changelogTopics, IChangelogRegister changelogReader, IOffsetCheckpointManager offsetCheckpointManager) { log = Logger.GetLogger(typeof(ProcessorStateManager)); logPrefix = $"stream-task[{taskId.Id}|{taskId.Partition}] "; this.taskId = taskId; Partition = partition; this.changelogTopics = changelogTopics ?? new Dictionary <string, string>(); changelogRegister = changelogReader; this.offsetCheckpointManager = offsetCheckpointManager; }
public StreamTask(string threadId, TaskId id, IEnumerable <TopicPartition> partitions, ProcessorTopology processorTopology, IConsumer <byte[], byte[]> consumer, IStreamConfig configuration, IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer, IChangelogRegister changelogRegister, StreamMetricsRegistry streamMetricsRegistry) : base(id, partitions, processorTopology, consumer, configuration, changelogRegister) { this.threadId = threadId; this.kafkaSupplier = kafkaSupplier; this.streamMetricsRegistry = streamMetricsRegistry; consumedOffsets = new Dictionary <TopicPartition, long>(); maxTaskIdleMs = configuration.MaxTaskIdleMs; maxBufferedSize = configuration.BufferedRecordsPerPartition; followMetadata = configuration.FollowMetadata; idleStartTime = -1; // eos enabled if (producer == null) { this.producer = CreateEOSProducer(); InitializeTransaction(); eosEnabled = true; } else { this.producer = producer; } var droppedRecordsSensor = TaskMetrics.DroppedRecordsSensor(this.threadId, Id, this.streamMetricsRegistry); collector = new RecordCollector(logPrefix, configuration, id, droppedRecordsSensor); collector.Init(ref this.producer); Context = new ProcessorContext(this, configuration, stateMgr, streamMetricsRegistry) .UseRecordCollector(collector); Context.FollowMetadata = followMetadata; var partitionsQueue = new Dictionary <TopicPartition, RecordQueue>(); foreach (var p in partitions) { var sourceProcessor = processorTopology.GetSourceProcessor(p.Topic); sourceProcessor.SetTaskId(id); var sourceTimestampExtractor = sourceProcessor.Extractor ?? configuration.DefaultTimestampExtractor; var queue = new RecordQueue( logPrefix, $"record-queue-{p.Topic}-{id.Id}-{id.Partition}", sourceTimestampExtractor, p, sourceProcessor, droppedRecordsSensor); partitionsQueue.Add(p, queue); processors.Add(sourceProcessor); } partitionGrouper = new PartitionGrouper(partitionsQueue); closeTaskSensor = ThreadMetrics.ClosedTaskSensor(this.threadId, streamMetricsRegistry); activeBufferedRecordSensor = TaskMetrics.ActiveBufferedRecordsSensor(this.threadId, Id, streamMetricsRegistry); processSensor = TaskMetrics.ProcessSensor(this.threadId, Id, streamMetricsRegistry); processLatencySensor = TaskMetrics.ProcessLatencySensor(this.threadId, Id, streamMetricsRegistry); enforcedProcessingSensor = TaskMetrics.EnforcedProcessingSensor(this.threadId, Id, streamMetricsRegistry); commitSensor = TaskMetrics.CommitSensor(this.threadId, Id, streamMetricsRegistry); activeRestorationSensor = TaskMetrics.ActiveRestorationSensor(this.threadId, Id, streamMetricsRegistry); restorationRecordsSendsor = TaskMetrics.RestorationRecordsSensor(this.threadId, Id, streamMetricsRegistry); }
protected AbstractTask(TaskId id, IEnumerable <TopicPartition> partition, ProcessorTopology topology, IConsumer <byte[], byte[]> consumer, IStreamConfig config, IChangelogRegister changelogRegister) { log = Logger.GetLogger(GetType()); logPrefix = $"stream-task[{id.Id}|{id.Partition}] "; Partition = partition; Id = id; Topology = topology; this.consumer = consumer; configuration = config; var offsetCheckpointMngt = config.OffsetCheckpointManager ?? new OffsetCheckpointFile(Path.Combine(config.StateDir, config.ApplicationId, $"{id.Id}-{id.Partition}")); offsetCheckpointMngt.Configure(config, id); stateMgr = new ProcessorStateManager( id, partition, topology.StoresToTopics, changelogRegister, offsetCheckpointMngt); }