public async Task ProcessAsync(ProcessMode processMode, CancellationToken token)
{
var start = await _cursorService.GetValueAsync(_processor.CursorName);
var end = await _cursorService.GetMinimumAsync(_processor.DependencyCursorNames);
int commitCount;
do
{
var commits = await _enumerator.GetCommitsAsync(
start,
end,
_processor.BatchSize);
var entityCount = commits.Sum(x => x.Entities.Count);
commitCount = commits.Count;
if (commits.Any())
{
var min = commits.Min(x => x.CommitTimestamp);
var max = commits.Max(x => x.CommitTimestamp);
start = max;
_logger.LogInformation(
"Fetched {CommitCount} commits ({EntityCount} {EntityName}) between {Min:O} and {Max:O}.",
commitCount,
entityCount,
typeof(TEntity).Name,
min,
max);
}
else
{
_logger.LogInformation("No more commits were found within the bounds.");
}
switch (processMode)
{
case ProcessMode.Sequentially:
await ProcessSequentiallyAsync(commits, token);
break;
case ProcessMode.TaskQueue:
await ProcessTaskQueueAsync(commits, token);
break;
default:
throw new NotImplementedException();
}
if (commits.Any())
{
_logger.LogInformation("Cursor {CursorName} moving to {Start:O}.", _processor.CursorName, start);
await _cursorService.SetValueAsync(_processor.CursorName, start);
}
}while (commitCount > 0);
}
private async Task WorkAsync(Work work)
{
if (!work.Leaves.Any())
{
return;
}
await _processor.ProcessAsync(work.Page, work.Leaves);
var cursorService = new CursorService();
await cursorService.SetValueAsync(_processor.CursorName, work.Leaves.Last().CommitTimeStamp);
}
private async Task ConsumeAsync(
IReadOnlyList <V2Package> packages,
string cursorName,
Func <V2Package, DateTimeOffset> getTimestamp)
{
var oldCUrsor = await _cursorService.GetValueAsync(cursorName);
await _service.AddOrUpdatePackagesAsync(packages);
var newCursor = packages.Max(getTimestamp);
if (newCursor > oldCUrsor)
{
await _cursorService.SetValueAsync(cursorName, newCursor);
}
}
