/************************** * ===== Processing ===== **************************/ private void ProcessThread(int id, string html) { if (String.IsNullOrEmpty(html)) { DatabaseQueue.Enqueue(new DbModels.Thread() { Id = id }); TelemetryManager.IncrimentEmptyThreads(); return; } RobloxThread thread = new RobloxThread(id); thread.AddPage(html); if (thread.IsEmpty) { DatabaseQueue.Enqueue(new DbModels.Thread() { Id = id }); TelemetryManager.IncrimentEmptyThreads(); return; } if (thread.PagesCount > 1 && thread.CurrentPage < thread.PagesCount) { PageDownloadingQueue.Enqueue(thread); return; } DbModels.Thread dbThread = thread.ToDbThread(); DatabaseQueue.Enqueue(dbThread); thread = null; //TODO: Evaluate necessity return; }
//Return value indicates if the thread is finished or not private bool ProcessThreadPage(RobloxThread thread, string html) { if (String.IsNullOrEmpty(html)) { thread.Errors += $"; Page {thread.CurrentPage} is error"; return(true); } thread.AddPage(html); if (thread.CurrentPage < thread.PagesCount) { PageDownloadingQueue.Enqueue(thread); return(false); } return(true); }
/************************** * ===== Worker Loops ===== **************************/ private async Task DoDownloadWork(object state) { TaskState taskState = (TaskState)state; Stopwatch stopwatch = new Stopwatch(); while (active && downloadersActive) { taskState.Status = State.Running; if (!await CanDownload()) { taskState.Status = State.Paused; await Task.Delay(500); continue; } if (!PageDownloadingQueue.IsEmpty) { RobloxThread thread; if (!PageDownloadingQueue.TryDequeue(out thread)) { //Another thread grabbed the item before we did, continue other work continue; } try { await stopwatch.TimeAsync( async() => await DownloadThreadPage(thread), async (long time) => TelemetryManager.Incriment(TelemetryType.downloaded_pages, taskState.Id, time) ); } catch (Exception ex) { taskState.Status = State.Error; active = false; exception = ex; break; } continue; } if (!ThreadQueue.IsEmpty) { int id; if (!ThreadQueue.TryDequeue(out id)) { //Nothing left in queue? //TODO: Handle end conditions continue; } try { await stopwatch.TimeAsync( async() => await DownloadThread(id), async (long time) => TelemetryManager.Incriment(TelemetryType.downloaded_threads, taskState.Id, time) ); } catch (Exception ex) { taskState.Status = State.Error; active = false; exception = ex; break; } } else { downloadersActive = false; break; } } taskState.Status = State.Complete; }