private void RestoreChangelog(ChangelogMetadata changelogMetadata)
{
var numRecords = changelogMetadata.BufferedLimit;
if (numRecords > 0)
{
var records = changelogMetadata.BufferedRecords.Take(numRecords);
changelogMetadata.StateManager.Restore(changelogMetadata.StoreMetadata, records);
if (numRecords >= changelogMetadata.BufferedRecords.Count)
{
changelogMetadata.BufferedRecords.Clear();
}
long currentOffset = changelogMetadata.StoreMetadata.Offset.Value;
changelogMetadata.CurrentOffset = currentOffset;
log.LogDebug($"Restored {numRecords} records from " +
$"changelog {changelogMetadata.StoreMetadata.Store.Name} " +
$"to store {changelogMetadata.StoreMetadata.ChangelogTopicPartition}, " +
$"end offset is {(changelogMetadata.RestoreEndOffset.HasValue ? changelogMetadata.RestoreEndOffset.Value : "unknown")}, " +
$"current offset is {currentOffset}");
changelogMetadata.BufferedLimit = 0;
changelogMetadata.TotalRestored += numRecords;
// TODO : call trigger batchRestored
var restorationRecordSensor = TaskMetrics.RestorationRecordsSensor(
threadId,
changelogMetadata.StateManager.taskId,
metricsRegistry);
restorationRecordSensor.Record(Math.Max(changelogMetadata.RestoreEndOffset.Value - currentOffset, 0));
}
else if (changelogMetadata.StoreMetadata.Offset.HasValue)
{
changelogMetadata.CurrentOffset = changelogMetadata.StoreMetadata.Offset.Value;
}
if (HasRestoredToEnd(changelogMetadata))
{
log.LogInformation($"Finished restoring changelog {changelogMetadata.StoreMetadata.Store.Name} " +
$"to store {changelogMetadata.StoreMetadata.ChangelogTopicPartition} " +
$"with a total number of {changelogMetadata.TotalRestored} records");
changelogMetadata.ChangelogState = ChangelogState.COMPLETED;
if (!restoreConsumer.Assignment.Contains(changelogMetadata.StoreMetadata.ChangelogTopicPartition))
{
throw new IllegalStateException($"The current assignment {string.Join(",", restoreConsumer.Assignment.Select(t => $"{t.Topic}-{t.Partition}"))} " +
$"does not contain the partition {changelogMetadata.StoreMetadata.ChangelogTopicPartition} for pausing.");
}
restoreConsumer.Pause(changelogMetadata.StoreMetadata.ChangelogTopicPartition.ToSingle());
log.LogDebug($"Paused partition {changelogMetadata.StoreMetadata.ChangelogTopicPartition} from the restore consumer");
// TODO : call trigger restoredEnd
}
}
// internal for testing
internal bool HasRestoredToEnd(ChangelogMetadata changelogMetadata)
{
long?endOffset = changelogMetadata.RestoreEndOffset;
if (endOffset == null || endOffset == Offset.Unset || endOffset == 0)
{
return(true);
}
if (!changelogMetadata.BufferedRecords.Any())
{
var offset = restoreConsumer.Position(changelogMetadata.StoreMetadata.ChangelogTopicPartition);
return(offset != Offset.Unset && offset >= endOffset);
}
return(changelogMetadata.CurrentOffset >= endOffset);
}
public void TestHasRestoredEnd()
{
ChangelogMetadata metadata = new ChangelogMetadata
{
RestoreEndOffset = null
};
Assert.IsTrue(storeChangelogReader.HasRestoredToEnd(metadata));
metadata = new ChangelogMetadata
{
RestoreEndOffset = 0
};
Assert.IsTrue(storeChangelogReader.HasRestoredToEnd(metadata));
metadata = new ChangelogMetadata
{
RestoreEndOffset = Offset.Unset
};
Assert.IsTrue(storeChangelogReader.HasRestoredToEnd(metadata));
restoreConsumer.Commit(new List <TopicPartitionOffset>()
{
new TopicPartitionOffset(new TopicPartition(changelogTopic, 0), 10)
});
metadata = new ChangelogMetadata
{
RestoreEndOffset = 10,
CurrentOffset = 12,
BufferedRecords = new List <ConsumeResult <byte[], byte[]> >(),
StoreMetadata = new ProcessorStateManager.StateStoreMetadata()
{
ChangelogTopicPartition = new TopicPartition(changelogTopic, 0)
}
};
Assert.IsTrue(storeChangelogReader.HasRestoredToEnd(metadata));
}
public void Register(TopicPartition topicPartition, ProcessorStateManager processorStateManager)
{
var storeMetadata = processorStateManager.GetStoreMetadata(topicPartition);
if (storeMetadata == null)
{
throw new StreamsException($"Cannot find the corresponding state store metadata for changelog {topicPartition}");
}
var changelogMetadata = new ChangelogMetadata
{
StoreMetadata = storeMetadata,
StateManager = processorStateManager,
ChangelogState = ChangelogState.REGISTERED,
RestoreEndOffset = null,
BeginOffset = null,
CurrentOffset = null,
TotalRestored = 0,
BufferedLimit = 0,
BufferedRecords = new List <ConsumeResult <byte[], byte[]> >()
};
changelogs.Add(topicPartition, changelogMetadata);
}
