// Policy List Control
private async Task GetBatchCollectionAsync()
{
var batchReq = new BatchRequest(config.hostUrl);
var batch = await batchReq.GetNextBatchAsync();
Debug.WriteLine("Requested New Batch");
// add something to keep track of these
string[] batchList = batch.Data.Replace("\r", "").Split(',');
var batchProc = new BatchProcessing();
batchProc.Id = batch.Id;
batchProc.Key = batch.Key;
batchProc.Data = batch.Data;
batchProc.LastComplete = batch.LastComplete;
batchProc.isListed = batch.isListed;
BlockingCollection <string> tmpBlock = new BlockingCollection <string>();
foreach (var b in batchList)
{
tmpBlock.Add(b);
}
batchProc.DataArray = tmpBlock;
BatchCollection.Add(batchProc);
tbLog.Text += string.Format("Policy Collection Added {0} items", batchList.Length.ToString());
tbLog.Text += "\n";
}
// Run
private async void btnStart_Click(object sender, EventArgs e)
{
//btnCancel.Visible = true;
int batchIndex = 0;
bool finishBatch = false;
isRunning = true;
bool isNewBatch = true;
BatchProcessing batch = null;
disableConfig();
await GetBatchCollectionAsync();
while (isRunning)
{
//while(paused)
//{
// int pau = 0;
// if (pau == 0)
// tbLog.Text += "Paused";
// Task.Delay(100);
// pau = 1;
// btnStart.Text = "Start";
//}
if (isNewBatch)
{
if (BatchCollection.Count > 0)
{
batch = BatchCollection.Take();
isNewBatch = false;
}
}
if (batch.LastComplete > batchIndex)
{
finishBatch = true;
while (finishBatch)
{
var dispose = batch.DataArray.Take();
batchIndex++;
if (batchIndex == batch.LastComplete)
{
finishBatch = false;
}
await Task.Delay(5);
dispose = null;
}
}
WebBrowser browser = null;
if (BrowserCollection.Count < 1)
{
browser = await GetBrowser();
}
string policy = null;
if (batch.DataArray.Count > 0)
{
policy = batch.DataArray.Take();
}
if (browser != null && policy != null)
{
Crawler.LossRunCrawlerResult result;
int i = 0;
while (i < 10)
{
try
{
tbLog.Text += "Starting WebCrawler\n";
result = await StartCrawler(browser, policy, batch.Key);
if (result.Result == Crawler.LossRunResult.Success)
{
tbLog.Text += string.Format("Policy {0} Result: Success", policy);
tbLog.Text += "\n";
lblBatchRemain.Text = string.Format("Batch Remaining: {0}", batch.DataArray.Count.ToString());
PendingBrowsers.Add(browser);
}
if (result.Result == Crawler.LossRunResult.Duplicate)
{
tbLog.Text += string.Format("Policy {0} Result: Duplicate", policy);
tbLog.Text += "\n";
lblBatchRemain.Text = string.Format("Batch Remaining: {0}", batch.DataArray.Count.ToString());
PendingBrowsers.Add(browser);
}
if (result.Result == Crawler.LossRunResult.Failure)
{
tbLog.Text += string.Format("Policy {0} Result: Failure", policy);
tbLog.Text += "\n";
lblBatchRemain.Text = string.Format("Batch Remaining: {0}", batch.DataArray.Count.ToString());
PendingBrowsers.Add(browser);
}
i = 15;
batchIndex++;
}
catch
{
batch.LastComplete = batchIndex;
if (i < 3)
{
await Task.Delay(config.globalTimeout);
}
if (i >= 3 && i < 7)
{
await Task.Delay(config.globalTimeout * 2);
}
if (i >= 7 && i < 10)
{
await Task.Delay(config.globalTimeout * 60 * 60);
}
}
finally
{
i++;
}
}
var upReq = new BatchRequest(tbHostUrl.Text);
var incBatch = new BatchData();
incBatch.Id = batch.Id;
incBatch.Key = batch.Key;
incBatch.Data = batch.Data;
incBatch.LastComplete = batch.LastComplete;
incBatch.isListed = batch.isListed;
if (i >= 10 && i < 15)
{
incBatch.isAvail = true;
await upReq.UpdateBatch(incBatch);
}
else
{
incBatch.isAvail = false;
await upReq.UpdateBatch(incBatch);
}
}
else
{
isRunning = false;
}
var req = new BatchRequest(tbHostUrl.Text);
var stats = await req.GetStats();
lblGlobalBatRem.Text = "Batches Remaining: " + stats.BatchesRemaining.ToString();
lblGlobalPer.Text = "Percent Complete: " + (stats.BatchesCompletePercent * 100).ToString();
if (batch.DataArray.Count == 0)
{
// Post results to server
// AddCode
isNewBatch = true;
}
if (batch.DataArray.Count < 10)
{
isBatchReady = true;
}
if (isBatchReady && !isNewBatch)
{
isBatchReady = false;
int j = 0;
while (j < 3)
{
try
{
if (batch.DataArray.Count < 10)
{
await GetBatchCollectionAsync();
j = 4;
}
}
catch
{
await Task.Delay(config.globalTimeout);
Debug.WriteLine("Failed - Trying Again");
tbLog.Text += "Failed - Trying Again\n";
j++;
}
}
if (j == 3)
{
isRunning = false;
}
}
}
tbLog.Text += "All Finished!";
}
