private void RestoreChangelog(ChangelogMetadata changelogMetadata)
{
var numRecords = changelogMetadata.BufferedLimit;
if (numRecords > 0)
{
var records = changelogMetadata.BufferedRecords.Take(numRecords);
changelogMetadata.StateManager.Restore(changelogMetadata.StoreMetadata, records);
if (numRecords >= changelogMetadata.BufferedRecords.Count)
{
changelogMetadata.BufferedRecords.Clear();
}
long currentOffset = changelogMetadata.StoreMetadata.Offset.Value;
changelogMetadata.CurrentOffset = currentOffset;
log.LogDebug($"Restored {numRecords} records from " +
$"changelog {changelogMetadata.StoreMetadata.Store.Name} " +
$"to store {changelogMetadata.StoreMetadata.ChangelogTopicPartition}, " +
$"end offset is {(changelogMetadata.RestoreEndOffset.HasValue ? changelogMetadata.RestoreEndOffset.Value : "unknown")}, " +
$"current offset is {currentOffset}");
changelogMetadata.BufferedLimit = 0;
changelogMetadata.TotalRestored += numRecords;
// TODO : call trigger batchRestored
var restorationRecordSensor = TaskMetrics.RestorationRecordsSensor(
threadId,
changelogMetadata.StateManager.taskId,
metricsRegistry);
restorationRecordSensor.Record(Math.Max(changelogMetadata.RestoreEndOffset.Value - currentOffset, 0));
}
else if (changelogMetadata.StoreMetadata.Offset.HasValue)
{
changelogMetadata.CurrentOffset = changelogMetadata.StoreMetadata.Offset.Value;
}
if (HasRestoredToEnd(changelogMetadata))
{
log.LogInformation($"Finished restoring changelog {changelogMetadata.StoreMetadata.Store.Name} " +
$"to store {changelogMetadata.StoreMetadata.ChangelogTopicPartition} " +
$"with a total number of {changelogMetadata.TotalRestored} records");
changelogMetadata.ChangelogState = ChangelogState.COMPLETED;
if (!restoreConsumer.Assignment.Contains(changelogMetadata.StoreMetadata.ChangelogTopicPartition))
{
throw new IllegalStateException($"The current assignment {string.Join(",", restoreConsumer.Assignment.Select(t => $"{t.Topic}-{t.Partition}"))} " +
$"does not contain the partition {changelogMetadata.StoreMetadata.ChangelogTopicPartition} for pausing.");
}
restoreConsumer.Pause(changelogMetadata.StoreMetadata.ChangelogTopicPartition.ToSingle());
log.LogDebug($"Paused partition {changelogMetadata.StoreMetadata.ChangelogTopicPartition} from the restore consumer");
// TODO : call trigger restoredEnd
}
}
public void Init(ProcessorContext context, IStateStore root)
{
this.context = context;
expiredRecordSensor = TaskMetrics.DroppedRecordsSensor(
Thread.CurrentThread.Name,
context.Id,
context.Metrics);
segments.OpenExisting(context, observedStreamTime);
context.Register(root, (k, v, t) => Put(k, v));
isOpen = true;
}
public virtual void Init(ProcessorContext context)
{
log.LogDebug("{LogPrefix}Initializing process context", logPrefix);
Context = context;
droppedRecordsSensor = TaskMetrics.DroppedRecordsSensor(
Thread.CurrentThread.Name,
Context.Id,
Context.Metrics);
foreach (var n in Next)
{
n.Init(context);
}
log.LogDebug("{LogPrefix}Process context initialized", logPrefix);
}
public void Init(ProcessorContext context, IStateStore root)
{
expiredRecordSensor = TaskMetrics.DroppedRecordsSensor(
Thread.CurrentThread.Name,
context.Id,
context.Metrics);
this.context = context;
if (root != null)
{
// register the store
context.Register(root,
(key, value, timestamp) => Put(key, value, timestamp));
}
IsOpen = true;
}
public StreamTask(string threadId, TaskId id, IEnumerable <TopicPartition> partitions,
ProcessorTopology processorTopology, IConsumer <byte[], byte[]> consumer, IStreamConfig configuration,
IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer, IChangelogRegister changelogRegister,
StreamMetricsRegistry streamMetricsRegistry)
: base(id, partitions, processorTopology, consumer, configuration, changelogRegister)
{
this.threadId = threadId;
this.kafkaSupplier = kafkaSupplier;
this.streamMetricsRegistry = streamMetricsRegistry;
consumedOffsets = new Dictionary <TopicPartition, long>();
maxTaskIdleMs = configuration.MaxTaskIdleMs;
maxBufferedSize = configuration.BufferedRecordsPerPartition;
followMetadata = configuration.FollowMetadata;
idleStartTime = -1;
// eos enabled
if (producer == null)
{
this.producer = CreateEOSProducer();
InitializeTransaction();
eosEnabled = true;
}
else
{
this.producer = producer;
}
var droppedRecordsSensor = TaskMetrics.DroppedRecordsSensor(this.threadId, Id, this.streamMetricsRegistry);
collector = new RecordCollector(logPrefix, configuration, id, droppedRecordsSensor);
collector.Init(ref this.producer);
Context = new ProcessorContext(this, configuration, stateMgr, streamMetricsRegistry)
.UseRecordCollector(collector);
Context.FollowMetadata = followMetadata;
var partitionsQueue = new Dictionary <TopicPartition, RecordQueue>();
foreach (var p in partitions)
{
var sourceProcessor = processorTopology.GetSourceProcessor(p.Topic);
sourceProcessor.SetTaskId(id);
var sourceTimestampExtractor = sourceProcessor.Extractor ?? configuration.DefaultTimestampExtractor;
var queue = new RecordQueue(
logPrefix,
$"record-queue-{p.Topic}-{id.Id}-{id.Partition}",
sourceTimestampExtractor,
p,
sourceProcessor,
droppedRecordsSensor);
partitionsQueue.Add(p, queue);
processors.Add(sourceProcessor);
}
partitionGrouper = new PartitionGrouper(partitionsQueue);
closeTaskSensor = ThreadMetrics.ClosedTaskSensor(this.threadId, streamMetricsRegistry);
activeBufferedRecordSensor = TaskMetrics.ActiveBufferedRecordsSensor(this.threadId, Id, streamMetricsRegistry);
processSensor = TaskMetrics.ProcessSensor(this.threadId, Id, streamMetricsRegistry);
processLatencySensor = TaskMetrics.ProcessLatencySensor(this.threadId, Id, streamMetricsRegistry);
enforcedProcessingSensor = TaskMetrics.EnforcedProcessingSensor(this.threadId, Id, streamMetricsRegistry);
commitSensor = TaskMetrics.CommitSensor(this.threadId, Id, streamMetricsRegistry);
activeRestorationSensor = TaskMetrics.ActiveRestorationSensor(this.threadId, Id, streamMetricsRegistry);
restorationRecordsSendsor = TaskMetrics.RestorationRecordsSensor(this.threadId, Id, streamMetricsRegistry);
}