/**************************
* ===== Processing =====
**************************/
private void ProcessThread(int id, string html)
{
if (String.IsNullOrEmpty(html))
{
DatabaseQueue.Enqueue(new DbModels.Thread()
{
Id = id
});
TelemetryManager.IncrimentEmptyThreads();
return;
}
RobloxThread thread = new RobloxThread(id);
thread.AddPage(html);
if (thread.IsEmpty)
{
DatabaseQueue.Enqueue(new DbModels.Thread()
{
Id = id
});
TelemetryManager.IncrimentEmptyThreads();
return;
}
if (thread.PagesCount > 1 && thread.CurrentPage < thread.PagesCount)
{
PageDownloadingQueue.Enqueue(thread);
return;
}
DbModels.Thread dbThread = thread.ToDbThread();
DatabaseQueue.Enqueue(dbThread);
thread = null; //TODO: Evaluate necessity
return;
}
//Return value indicates if the thread is finished or not
private bool ProcessThreadPage(RobloxThread thread, string html)
{
if (String.IsNullOrEmpty(html))
{
thread.Errors += $"; Page {thread.CurrentPage} is error";
return(true);
}
thread.AddPage(html);
if (thread.CurrentPage < thread.PagesCount)
{
PageDownloadingQueue.Enqueue(thread);
return(false);
}
return(true);
}
/**************************
* ===== Worker Loops =====
**************************/
private async Task DoDownloadWork(object state)
{
TaskState taskState = (TaskState)state;
Stopwatch stopwatch = new Stopwatch();
while (active && downloadersActive)
{
taskState.Status = State.Running;
if (!await CanDownload())
{
taskState.Status = State.Paused;
await Task.Delay(500);
continue;
}
if (!PageDownloadingQueue.IsEmpty)
{
RobloxThread thread;
if (!PageDownloadingQueue.TryDequeue(out thread))
{
//Another thread grabbed the item before we did, continue other work
continue;
}
try
{
await stopwatch.TimeAsync(
async() => await DownloadThreadPage(thread),
async (long time) => TelemetryManager.Incriment(TelemetryType.downloaded_pages, taskState.Id, time)
);
}
catch (Exception ex)
{
taskState.Status = State.Error;
active = false;
exception = ex;
break;
}
continue;
}
if (!ThreadQueue.IsEmpty)
{
int id;
if (!ThreadQueue.TryDequeue(out id))
{
//Nothing left in queue?
//TODO: Handle end conditions
continue;
}
try
{
await stopwatch.TimeAsync(
async() => await DownloadThread(id),
async (long time) => TelemetryManager.Incriment(TelemetryType.downloaded_threads, taskState.Id, time)
);
}
catch (Exception ex)
{
taskState.Status = State.Error;
active = false;
exception = ex;
break;
}
}
else
{
downloadersActive = false;
break;
}
}
taskState.Status = State.Complete;
}
