private void OnWebPageLoaded(object sender, ChildPage page)
{
lock (childPages)
{
childPages.Add(page);
waitForPages.Set();
}
}
private void LoadChildPages(MultiValueDictionary <string, ObjectId> links)
{
childPages = new List <ChildPage>();
waitForPages = new ManualResetEvent(false);
foreach (KeyValuePair <string, IReadOnlyCollection <ObjectId> > pair in links)
{
ChildPage page = new ChildPage(pair.Key, DateTime.Now, pair.Value);
page.WebPageLoaded += new EventHandler <ChildPage>(OnWebPageLoaded);
page.LoadError += new EventHandler(OnLoadError);
WebCrawler.Instance.EnqueueWork(page);
}
while (childPages.Count != links.Count)
{
waitForPages.Reset();
waitForPages.WaitOne();
}
}
public void AddChildPage(ChildPage page)
{
childPages.Add(page);
}
private void GetResponse(IAsyncResult webRequest)
{
HttpWebRequest request = (HttpWebRequest)webRequest.AsyncState;
HttpWebResponse response = null;
try
{
response = (HttpWebResponse)request.EndGetResponse(webRequest);
System.Diagnostics.Debug.Print("\tLoading: " + domain.AbsoluteUri);
}
catch (WebException webEx)
{
System.Diagnostics.Debug.Print("\t[" + domain.AbsoluteUri + "]");
System.Diagnostics.Debug.Print("\t" + webEx.ToString());
if (this is HtmlRecord)
{
jobStatus = JobStatus.ErrorRequesting;
}
else
{
ChildPage page = this as ChildPage;
page.InvokeLoadErrorEvent();
}
if (waitTime < WebCrawler.TimeoutPeriod)
{
HttpWebResponse resp = webEx.Response as HttpWebResponse;
if (resp != null)
{
HttpStatusCode statuscode = resp.StatusCode;
switch (statuscode)
{
case HttpStatusCode.Forbidden:
serverResponse = statuscode;
KillProcess();
break;
case HttpStatusCode.BadRequest:
serverResponse = statuscode;
KillProcess();
break;
default:
SetWaitTime(waitTime + 10000);
WebCrawler.Instance.EnqueueWork(this);
break;
}
}
else
{
SetWaitTime(waitTime + 10000);
WebCrawler.Instance.EnqueueWork(this);
}
}
else
{
if (this is ChildPage)
{
// Allow parent thread to continue, disregard this web page
ChildPage page = this as ChildPage;
page.InvokeLoadedEvent();
}
}
}
catch (Exception ex)
{
System.Diagnostics.Debug.WriteLine(ex.ToString());
}
finally
{
if (response != null)
{
HandleResponse(response);
}
}
}
