private async Task StartCrawlingPage(Uri page, CancellationToken ctsToken)
{
MainForm.UpdateSemaphoresStatus(semaphore.CurrentCount);
crawledPages.Add(page);
// Wait for semaphore
await this.semaphore.WaitAsync(cancellationToken);
try {
// Checking cancelation token (checking whether stop button has been pressed)
if (!cts.IsCancellationRequested)
{
PageFragment pf = new PageFragment {
Address = page.AbsoluteUri
};
// Download page
HttpClient httpClient = new HttpClient();
HttpResponseMessage response = await httpClient.GetAsync(page);
try
{
// Check whether page is internal or external
if (Uri.Compare(BaseUrl, page, UriComponents.Host,
UriFormat.SafeUnescaped, StringComparison.CurrentCulture) == 0)
{
// Get page source
string sourceHtml = await response.Content.ReadAsStringAsync();
HtmlDocument htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(sourceHtml);
// Check whether url depth is in acceptable range if not don't crawl further
if (page.Segments.Length - 1 <= Utils.CrawlDepthLimit)
{
// Crawl deeper through urls found on this page
// (it happens in separate threads simultanously)
CrawlFurther(htmlDocument, ref pf);
}
// Fulfill PageFragment with data
ManagePageFragment(ref pf, ref response, ref htmlDocument, page);
// Update data source
UpdateDataTable(pf);
}
else
{
// Fulfill PageFragment with data
ManagePageFragmentIfExternal(ref pf, ref response, page);
// Update data source
UpdateDataTable(pf);
}
}
catch (UriFormatException ex)
{
Debug.WriteLine(" Podstrona: " + page + " ma niepoprawnie sformatowany url. Message: " + ex.Message);
}
catch (WebException ex) when((ex.Response as HttpWebResponse)?.StatusCode == HttpStatusCode.NotFound)
{
pf.StatusCode = "404";
Debug.WriteLine(" strona " + page + " jest niedostepna -> 404 NotFound");
}
catch (WebException ex)
{
try
{
string status = (ex.Response as HttpWebResponse)?.StatusCode.ToString();
pf.StatusCode = status;
Debug.WriteLine(" strona " + page + " WebEx: " + status);
}
catch (NullReferenceException e)
{
pf.StatusCode = "Undefined";
Debug.WriteLine(" strona " + page + " WebEx: Undefined. Message: " + e);
}
}
catch (Exception ex)
{
Debug.WriteLine(" strona " + page + " spotkala niezdefiniowany (nieobsłużony indywidualnie) wyjątek: " + ex.Message);
}
UpdateCountersList(ref pf);
}
}
catch (OperationCanceledException)
{
Debug.WriteLine("Task anulowany");
}
catch (Exception)
{
Debug.WriteLine("Task sie nie powiodl");
}
this.semaphore.Release();
}