private async Task DoWork(object state)
{
TaskState taskState = (TaskState)state;
Stopwatch stopwatch = new Stopwatch();
while (active)
{
taskState.Status = State.Running;
if (!await CanProcess() && downloadersActive)
{
taskState.Status = State.Paused;
await Task.Delay(250);
continue;
}
if (!PageProcessingQueue.IsEmpty)
{
UnparsedPage page;
if (!PageProcessingQueue.TryDequeue(out page))
{
//another thread grabbed the page
continue;
}
stopwatch.Time(() =>
{
return(ProcessThreadPage(page.Thread, page.Html));
},
(long time) => TelemetryManager.Incriment(TelemetryType.processed_pages, taskState.Id, time)
);
}
if (!ProcessingQueue.IsEmpty)
{
UnparsedThread thread;
if (!ProcessingQueue.TryDequeue(out thread))
{
continue;
}
stopwatch.Time(() =>
{
ProcessThread(thread.Id, thread.Html);
},
(long time) => TelemetryManager.Incriment(TelemetryType.processed_threads, taskState.Id, time)
);
}
else if (!downloadersActive) //Queue is empty, no more downloads are being performed, set active to false after 1s delay
{
taskState.Status = State.Paused;
await Task.Delay(1000);
active = false;
break;
}
}
taskState.Status = State.Complete;
}
/**************************
* ===== Worker Loops =====
**************************/
private async Task DoDownloadWork(object state)
{
TaskState taskState = (TaskState)state;
Stopwatch stopwatch = new Stopwatch();
while (active && downloadersActive)
{
taskState.Status = State.Running;
if (!await CanDownload())
{
taskState.Status = State.Paused;
await Task.Delay(500);
continue;
}
if (!PageDownloadingQueue.IsEmpty)
{
RobloxThread thread;
if (!PageDownloadingQueue.TryDequeue(out thread))
{
//Another thread grabbed the item before we did, continue other work
continue;
}
try
{
await stopwatch.TimeAsync(
async() => await DownloadThreadPage(thread),
async (long time) => TelemetryManager.Incriment(TelemetryType.downloaded_pages, taskState.Id, time)
);
}
catch (Exception ex)
{
taskState.Status = State.Error;
active = false;
exception = ex;
break;
}
continue;
}
if (!ThreadQueue.IsEmpty)
{
int id;
if (!ThreadQueue.TryDequeue(out id))
{
//Nothing left in queue?
//TODO: Handle end conditions
continue;
}
try
{
await stopwatch.TimeAsync(
async() => await DownloadThread(id),
async (long time) => TelemetryManager.Incriment(TelemetryType.downloaded_threads, taskState.Id, time)
);
}
catch (Exception ex)
{
taskState.Status = State.Error;
active = false;
exception = ex;
break;
}
}
else
{
downloadersActive = false;
break;
}
}
taskState.Status = State.Complete;
}
