// Run private async void btnStart_Click(object sender, EventArgs e) { //btnCancel.Visible = true; int batchIndex = 0; bool finishBatch = false; isRunning = true; bool isNewBatch = true; BatchProcessing batch = null; disableConfig(); await GetBatchCollectionAsync(); while (isRunning) { //while(paused) //{ // int pau = 0; // if (pau == 0) // tbLog.Text += "Paused"; // Task.Delay(100); // pau = 1; // btnStart.Text = "Start"; //} if (isNewBatch) { if (BatchCollection.Count > 0) { batch = BatchCollection.Take(); isNewBatch = false; } } if (batch.LastComplete > batchIndex) { finishBatch = true; while (finishBatch) { var dispose = batch.DataArray.Take(); batchIndex++; if (batchIndex == batch.LastComplete) { finishBatch = false; } await Task.Delay(5); dispose = null; } } WebBrowser browser = null; if (BrowserCollection.Count < 1) { browser = await GetBrowser(); } string policy = null; if (batch.DataArray.Count > 0) { policy = batch.DataArray.Take(); } if (browser != null && policy != null) { Crawler.LossRunCrawlerResult result; int i = 0; while (i < 10) { try { tbLog.Text += "Starting WebCrawler\n"; result = await StartCrawler(browser, policy, batch.Key); if (result.Result == Crawler.LossRunResult.Success) { tbLog.Text += string.Format("Policy {0} Result: Success", policy); tbLog.Text += "\n"; lblBatchRemain.Text = string.Format("Batch Remaining: {0}", batch.DataArray.Count.ToString()); PendingBrowsers.Add(browser); } if (result.Result == Crawler.LossRunResult.Duplicate) { tbLog.Text += string.Format("Policy {0} Result: Duplicate", policy); tbLog.Text += "\n"; lblBatchRemain.Text = string.Format("Batch Remaining: {0}", batch.DataArray.Count.ToString()); PendingBrowsers.Add(browser); } if (result.Result == Crawler.LossRunResult.Failure) { tbLog.Text += string.Format("Policy {0} Result: Failure", policy); tbLog.Text += "\n"; lblBatchRemain.Text = string.Format("Batch Remaining: {0}", batch.DataArray.Count.ToString()); PendingBrowsers.Add(browser); } i = 15; batchIndex++; } catch { batch.LastComplete = batchIndex; if (i < 3) { await Task.Delay(config.globalTimeout); } if (i >= 3 && i < 7) { await Task.Delay(config.globalTimeout * 2); } if (i >= 7 && i < 10) { await Task.Delay(config.globalTimeout * 60 * 60); } } finally { i++; } } var upReq = new BatchRequest(tbHostUrl.Text); var incBatch = new BatchData(); incBatch.Id = batch.Id; incBatch.Key = batch.Key; incBatch.Data = batch.Data; incBatch.LastComplete = batch.LastComplete; incBatch.isListed = batch.isListed; if (i >= 10 && i < 15) { incBatch.isAvail = true; await upReq.UpdateBatch(incBatch); } else { incBatch.isAvail = false; await upReq.UpdateBatch(incBatch); } } else { isRunning = false; } var req = new BatchRequest(tbHostUrl.Text); var stats = await req.GetStats(); lblGlobalBatRem.Text = "Batches Remaining: " + stats.BatchesRemaining.ToString(); lblGlobalPer.Text = "Percent Complete: " + (stats.BatchesCompletePercent * 100).ToString(); if (batch.DataArray.Count == 0) { // Post results to server // AddCode isNewBatch = true; } if (batch.DataArray.Count < 10) { isBatchReady = true; } if (isBatchReady && !isNewBatch) { isBatchReady = false; int j = 0; while (j < 3) { try { if (batch.DataArray.Count < 10) { await GetBatchCollectionAsync(); j = 4; } } catch { await Task.Delay(config.globalTimeout); Debug.WriteLine("Failed - Trying Again"); tbLog.Text += "Failed - Trying Again\n"; j++; } } if (j == 3) { isRunning = false; } } } tbLog.Text += "All Finished!"; }