private async Task ProduceAsync(
TaskQueue <IReadOnlyList <V2Package> > taskQueue,
string cursorName,
V2OrderByTimestamp orderBy,
Func <V2Package, DateTimeOffset> getTimestamp)
{
var start = await _cursorService.GetValueAsync(cursorName);
if (start > DateTimeOffset.MinValue.Add(FuzzFactor))
{
start = start.Subtract(FuzzFactor);
}
var complete = false;
do
{
var packages = await _v2Client.GetPackagesAsync(
_settings.V2BaseUrl,
orderBy,
start,
PageSize);
// If we have a full page, take only packages with a created timestamp less than the max.
if (packages.Count == PageSize)
{
var max = packages.Max(getTimestamp);
var packagesBeforeMax = packages
.Where(x => getTimestamp(x) < max)
.ToList();
if (packages.Any() &&
!packagesBeforeMax.Any())
{
throw new InvalidOperationException($"All of the packages in the page have the same {orderBy} timestamp.");
}
packages = packagesBeforeMax;
}
else
{
complete = true;
}
while (taskQueue.Count > 50)
{
await Task.Delay(TimeSpan.FromSeconds(1));
}
if (packages.Count > 0)
{
taskQueue.Enqueue(packages);
start = packages.Max(getTimestamp);
}
}while (!complete);
}
public async Task ProcessAsync(CancellationToken token)
{
var cursorService = new CursorService();
var start = await cursorService.GetValueAsync(_processor.CursorName);
DateTimeOffset end;
var dependencyCursorNames = _processor.DependencyCursorNames;
if (dependencyCursorNames.Any())
{
end = await cursorService.GetMinimumAsync(dependencyCursorNames);
}
else
{
end = DateTimeOffset.UtcNow;
}
_taskQueue.Start();
var failureTask = _taskQueue.FailureTask;
var produceThenCompleteTask = ProduceThenCompleteAsync(start, end, token);
var firstTask = await Task.WhenAny(failureTask, produceThenCompleteTask);
if (firstTask == failureTask)
{
await await failureTask;
}
else
{
await produceThenCompleteTask;
}
}
public async Task ProcessAsync(ProcessMode processMode, CancellationToken token)
{
var start = await _cursorService.GetValueAsync(_processor.CursorName);
var end = await _cursorService.GetMinimumAsync(_processor.DependencyCursorNames);
int commitCount;
do
{
var commits = await _enumerator.GetCommitsAsync(
start,
end,
_processor.BatchSize);
var entityCount = commits.Sum(x => x.Entities.Count);
commitCount = commits.Count;
if (commits.Any())
{
var min = commits.Min(x => x.CommitTimestamp);
var max = commits.Max(x => x.CommitTimestamp);
start = max;
_logger.LogInformation(
"Fetched {CommitCount} commits ({EntityCount} {EntityName}) between {Min:O} and {Max:O}.",
commitCount,
entityCount,
typeof(TEntity).Name,
min,
max);
}
else
{
_logger.LogInformation("No more commits were found within the bounds.");
}
switch (processMode)
{
case ProcessMode.Sequentially:
await ProcessSequentiallyAsync(commits, token);
break;
case ProcessMode.TaskQueue:
await ProcessTaskQueueAsync(commits, token);
break;
default:
throw new NotImplementedException();
}
if (commits.Any())
{
_logger.LogInformation("Cursor {CursorName} moving to {Start:O}.", _processor.CursorName, start);
await _cursorService.SetValueAsync(_processor.CursorName, start);
}
}while (commitCount > 0);
}
