private async Task ProduceAsync( TaskQueue <IReadOnlyList <V2Package> > taskQueue, string cursorName, V2OrderByTimestamp orderBy, Func <V2Package, DateTimeOffset> getTimestamp) { var start = await _cursorService.GetValueAsync(cursorName); if (start > DateTimeOffset.MinValue.Add(FuzzFactor)) { start = start.Subtract(FuzzFactor); } var complete = false; do { var packages = await _v2Client.GetPackagesAsync( _settings.V2BaseUrl, orderBy, start, PageSize); // If we have a full page, take only packages with a created timestamp less than the max. if (packages.Count == PageSize) { var max = packages.Max(getTimestamp); var packagesBeforeMax = packages .Where(x => getTimestamp(x) < max) .ToList(); if (packages.Any() && !packagesBeforeMax.Any()) { throw new InvalidOperationException($"All of the packages in the page have the same {orderBy} timestamp."); } packages = packagesBeforeMax; } else { complete = true; } while (taskQueue.Count > 50) { await Task.Delay(TimeSpan.FromSeconds(1)); } if (packages.Count > 0) { taskQueue.Enqueue(packages); start = packages.Max(getTimestamp); } }while (!complete); }
public async Task ProcessAsync(CancellationToken token) { var cursorService = new CursorService(); var start = await cursorService.GetValueAsync(_processor.CursorName); DateTimeOffset end; var dependencyCursorNames = _processor.DependencyCursorNames; if (dependencyCursorNames.Any()) { end = await cursorService.GetMinimumAsync(dependencyCursorNames); } else { end = DateTimeOffset.UtcNow; } _taskQueue.Start(); var failureTask = _taskQueue.FailureTask; var produceThenCompleteTask = ProduceThenCompleteAsync(start, end, token); var firstTask = await Task.WhenAny(failureTask, produceThenCompleteTask); if (firstTask == failureTask) { await await failureTask; } else { await produceThenCompleteTask; } }
public async Task ProcessAsync(ProcessMode processMode, CancellationToken token) { var start = await _cursorService.GetValueAsync(_processor.CursorName); var end = await _cursorService.GetMinimumAsync(_processor.DependencyCursorNames); int commitCount; do { var commits = await _enumerator.GetCommitsAsync( start, end, _processor.BatchSize); var entityCount = commits.Sum(x => x.Entities.Count); commitCount = commits.Count; if (commits.Any()) { var min = commits.Min(x => x.CommitTimestamp); var max = commits.Max(x => x.CommitTimestamp); start = max; _logger.LogInformation( "Fetched {CommitCount} commits ({EntityCount} {EntityName}) between {Min:O} and {Max:O}.", commitCount, entityCount, typeof(TEntity).Name, min, max); } else { _logger.LogInformation("No more commits were found within the bounds."); } switch (processMode) { case ProcessMode.Sequentially: await ProcessSequentiallyAsync(commits, token); break; case ProcessMode.TaskQueue: await ProcessTaskQueueAsync(commits, token); break; default: throw new NotImplementedException(); } if (commits.Any()) { _logger.LogInformation("Cursor {CursorName} moving to {Start:O}.", _processor.CursorName, start); await _cursorService.SetValueAsync(_processor.CursorName, start); } }while (commitCount > 0); }