private void crawler_ProcessPageCrawlCompleted(object sender, PageCrawlCompletedArgs e) { CrawledPage crawledPage = e.CrawledPage; if (crawledPage.WebException != null || crawledPage.HttpWebResponse.StatusCode != HttpStatusCode.OK) _logger.Error(String.Format("Crawl of page failed {0} StatusCode: [{1}]", crawledPage.Uri.AbsoluteUri, crawledPage.StatusCode)); else { IDatabase database = _databaseConnectionPool.GetObject(); try { string command = "INSERT INTO [CrawledPage] ([PageGUID], [ProjectGUID], [URL], [StatusCode], [PageSize], [HttpResponse], [CrawlDepth], [ParentURI]) VALUES (@PageGUID, @ProjectGUID, @URL, @StatusCode, @PageSize, @HttpResponse, @CrawlDepth, @ParentURI);"; Dictionary<string, object> parameters = new Dictionary<string, object>(); parameters.Add("@PageGUID", crawledPage.PageGUID); parameters.Add("@ProjectGUID", _projectGUID); parameters.Add("@URL", crawledPage.Uri.AbsoluteUri); parameters.Add("@StatusCode", crawledPage.StatusCode); parameters.Add("@PageSize", crawledPage.PageSizeInBytes); parameters.Add("@HttpResponse", crawledPage.HttpWebResponse.Headers.ToString()); parameters.Add("@CrawlDepth", crawledPage.CrawlDepth); parameters.Add("@ParentURI", crawledPage.ParentUri.AbsoluteUri); database.ExecuteNonQueryWithParameters(command, parameters); if (string.IsNullOrEmpty(crawledPage.RawContent) == false) { string contentCommand = "INSERT INTO [PageContent] ([PageGUID], [ProjectGUID], [URL], [RawContent]) VALUES (@pageGUID, @projectGUID, @url, @rawContent);"; Dictionary<string, object> contentParams = new Dictionary<string, object>(); contentParams.Add("@pageGUID", crawledPage.PageGUID); contentParams.Add("@projectGUID", _projectGUID); contentParams.Add("@url", crawledPage.Uri.AbsoluteUri); contentParams.Add("@rawContent", crawledPage.RawContent); database.ExecuteNonQueryWithParameters(contentCommand, contentParams); } } catch (Exception exception) { Console.WriteLine(exception.ToString()); _logger.Error("An unhandled exception was thrown inserting crawled page details/content to DB"); _logger.Error(exception); } finally { _databaseConnectionPool.PutObject(database); } } Console.WriteLine("Crawl of page succeeded {0} StatusCode: [{1}]", crawledPage.Uri.AbsoluteUri, crawledPage.StatusCode); }
private void crawler_ProcessPageCrawlCompleted(object sender, PageCrawlCompletedArgs e) { CrawledPage crawledPage = e.CrawledPage; if (crawledPage.WebException != null || crawledPage.HttpWebResponse.StatusCode != HttpStatusCode.OK) Console.WriteLine("Crawl of page failed {0}", crawledPage.Uri.AbsoluteUri); else { Console.WriteLine("Crawl of page succeeded {0}", crawledPage.Uri.AbsoluteUri); IDatabase database = _databaseConnectionPool.GetObject(); _databaseConnectionPool.PutObject(database); } if (string.IsNullOrEmpty(crawledPage.RawContent)) Console.WriteLine("Page had no content {0}", crawledPage.Uri.AbsoluteUri); }