public ProcessorStateManager(
TaskId taskId,
IEnumerable <TopicPartition> partition,
IDictionary <string, string> changelogTopics,
IChangelogRegister changelogReader,
IOffsetCheckpointManager offsetCheckpointManager)
{
log = Logger.GetLogger(typeof(ProcessorStateManager));
logPrefix = $"stream-task[{taskId.Id}|{taskId.Partition}] ";
this.taskId = taskId;
Partition = partition;
this.changelogTopics = changelogTopics ?? new Dictionary <string, string>();
changelogRegister = changelogReader;
this.offsetCheckpointManager = offsetCheckpointManager;
}
public StreamTask(string threadId, TaskId id, IEnumerable <TopicPartition> partitions,
ProcessorTopology processorTopology, IConsumer <byte[], byte[]> consumer, IStreamConfig configuration,
IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer, IChangelogRegister changelogRegister,
StreamMetricsRegistry streamMetricsRegistry)
: base(id, partitions, processorTopology, consumer, configuration, changelogRegister)
{
this.threadId = threadId;
this.kafkaSupplier = kafkaSupplier;
this.streamMetricsRegistry = streamMetricsRegistry;
consumedOffsets = new Dictionary <TopicPartition, long>();
maxTaskIdleMs = configuration.MaxTaskIdleMs;
maxBufferedSize = configuration.BufferedRecordsPerPartition;
followMetadata = configuration.FollowMetadata;
idleStartTime = -1;
// eos enabled
if (producer == null)
{
this.producer = CreateEOSProducer();
InitializeTransaction();
eosEnabled = true;
}
else
{
this.producer = producer;
}
var droppedRecordsSensor = TaskMetrics.DroppedRecordsSensor(this.threadId, Id, this.streamMetricsRegistry);
collector = new RecordCollector(logPrefix, configuration, id, droppedRecordsSensor);
collector.Init(ref this.producer);
Context = new ProcessorContext(this, configuration, stateMgr, streamMetricsRegistry)
.UseRecordCollector(collector);
Context.FollowMetadata = followMetadata;
var partitionsQueue = new Dictionary <TopicPartition, RecordQueue>();
foreach (var p in partitions)
{
var sourceProcessor = processorTopology.GetSourceProcessor(p.Topic);
sourceProcessor.SetTaskId(id);
var sourceTimestampExtractor = sourceProcessor.Extractor ?? configuration.DefaultTimestampExtractor;
var queue = new RecordQueue(
logPrefix,
$"record-queue-{p.Topic}-{id.Id}-{id.Partition}",
sourceTimestampExtractor,
p,
sourceProcessor,
droppedRecordsSensor);
partitionsQueue.Add(p, queue);
processors.Add(sourceProcessor);
}
partitionGrouper = new PartitionGrouper(partitionsQueue);
closeTaskSensor = ThreadMetrics.ClosedTaskSensor(this.threadId, streamMetricsRegistry);
activeBufferedRecordSensor = TaskMetrics.ActiveBufferedRecordsSensor(this.threadId, Id, streamMetricsRegistry);
processSensor = TaskMetrics.ProcessSensor(this.threadId, Id, streamMetricsRegistry);
processLatencySensor = TaskMetrics.ProcessLatencySensor(this.threadId, Id, streamMetricsRegistry);
enforcedProcessingSensor = TaskMetrics.EnforcedProcessingSensor(this.threadId, Id, streamMetricsRegistry);
commitSensor = TaskMetrics.CommitSensor(this.threadId, Id, streamMetricsRegistry);
activeRestorationSensor = TaskMetrics.ActiveRestorationSensor(this.threadId, Id, streamMetricsRegistry);
restorationRecordsSendsor = TaskMetrics.RestorationRecordsSensor(this.threadId, Id, streamMetricsRegistry);
}
protected AbstractTask(TaskId id, IEnumerable <TopicPartition> partition, ProcessorTopology topology, IConsumer <byte[], byte[]> consumer, IStreamConfig config, IChangelogRegister changelogRegister)
{
log = Logger.GetLogger(GetType());
logPrefix = $"stream-task[{id.Id}|{id.Partition}] ";
Partition = partition;
Id = id;
Topology = topology;
this.consumer = consumer;
configuration = config;
var offsetCheckpointMngt = config.OffsetCheckpointManager
?? new OffsetCheckpointFile(Path.Combine(config.StateDir, config.ApplicationId, $"{id.Id}-{id.Partition}"));
offsetCheckpointMngt.Configure(config, id);
stateMgr = new ProcessorStateManager(
id,
partition,
topology.StoresToTopics,
changelogRegister,
offsetCheckpointMngt);
}
