private async Task DoWork(object state)
{
TaskState taskState = (TaskState)state;
Stopwatch stopwatch = new Stopwatch();
while (active)
{
taskState.Status = State.Running;
if (!await CanProcess() && downloadersActive)
{
taskState.Status = State.Paused;
await Task.Delay(250);
continue;
}
if (!PageProcessingQueue.IsEmpty)
{
UnparsedPage page;
if (!PageProcessingQueue.TryDequeue(out page))
{
//another thread grabbed the page
continue;
}
stopwatch.Time(() =>
{
return(ProcessThreadPage(page.Thread, page.Html));
},
(long time) => TelemetryManager.Incriment(TelemetryType.processed_pages, taskState.Id, time)
);
}
if (!ProcessingQueue.IsEmpty)
{
UnparsedThread thread;
if (!ProcessingQueue.TryDequeue(out thread))
{
continue;
}
stopwatch.Time(() =>
{
ProcessThread(thread.Id, thread.Html);
},
(long time) => TelemetryManager.Incriment(TelemetryType.processed_threads, taskState.Id, time)
);
}
else if (!downloadersActive) //Queue is empty, no more downloads are being performed, set active to false after 1s delay
{
taskState.Status = State.Paused;
await Task.Delay(1000);
active = false;
break;
}
}
taskState.Status = State.Complete;
}