private void DoRun(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
foreach (var grabberEntry in _grabberEntries)
{
var entry = grabberEntry.Value;
if (entry.RunningJobsCount < 0)
{
throw new Exception($"RunningJobsCount < 0 for {entry.Grabber.GetSourceType()}");
}
if (entry.RunningJobsCount == entry.JobsLimit)
{
continue;
}
var job = _advertService.GetJob(entry.Grabber.GetSourceType());
if (job == null)
{
_logger.LogInformation("Notified sitemap manager to get more jobs for " +
entry.Grabber.GetSourceType());
_sitemapManager.AddJobDemand(entry.Grabber.GetSourceType(), 10);
continue;
}
Task.Factory.StartNew(() => entry.Grabber.Process(job), cancellationToken).ToObservable().Subscribe(
result => HandleResult(result, entry),
error => HandleError(error, entry, job)
);
entry.RunningJobsCount++;
_logger.LogTrace($"Added new job {job.Id} for {entry.Grabber.GetSourceType()}");
}
Task.Delay(QueueDelay, cancellationToken).Wait(cancellationToken);
}
}