/* fetchPage() - takes a _SpiderWorkItemDataWrapper object that will be cast from an object * (because the work method of a C# ThreadPool work item has to take a single * object argument, and be static/void), and fetches the _SpiderPageCandidate * at the index specified by the _index field in the _SpiderWorkItemDataWrapper * @o - the object argument to be cast into a _SpiderWorkItemDataWrapper */ static void fetchPage(object o) { // unpack the _SpiderWorkItemDataWrapper object _SpiderWorkItemDataWrapper wi = (_SpiderWorkItemDataWrapper)o; // get our spider object and our candidate_page to process Spider spider_object = wi.getSpiderObject(); _SpiderPageCandidate candidate_page = wi.getCandidatePage(); List <string> pre_pages = new List <string>(); byte[] buf = new byte[8192]; StringBuilder sb = new StringBuilder(); HttpWebResponse resp = null; try { HttpWebRequest req = (HttpWebRequest)WebRequest.Create(candidate_page._candidate_getUrl()); //req.Timeout = 1000; // sleep for the niceness time of this spider object spider_object.acquireFetchLock(); resp = (HttpWebResponse)req.GetResponse(); } catch (Exception e) { candidate_page._candidate_setError(); spider_object.writeStatus("ERROR: " + e.Message); spider_object.writeStatus("\tpage - " + candidate_page._candidate_getUrl() + "\n\t\treferred to by:"); List <SpiderLink> curr_refs = candidate_page.getReferredByLinks(); for (int i = 0; i < curr_refs.Count; i++) { spider_object.writeStatus("\t\t\t" + curr_refs.ElementAt(i).getReferringUrl()); } } if (resp != null) { // record the final Url after any redirects from this link string normalized_final_url = spider_object.normalizeUrl(resp.ResponseUri.ToString(), ""); if (normalized_final_url.Count() < 1) { candidate_page._candidate_setError(); spider_object.writeStatus("fetchPage(): candidate page " + candidate_page._candidate_getUrl() + " redirected to an illegal page."); } candidate_page.setUrl(normalized_final_url); spider_object.writeStatus("thread id: " + Thread.CurrentThread.ManagedThreadId + ", fetchPage(): fetched " + candidate_page._candidate_getUrl() + "\n\tfetchPage(): normalized final url - " + candidate_page.getUrl()); if (!candidate_page._candidate_isError()) { // read in the content of the page Stream resp_stream = resp.GetResponseStream(); string temp_string = null; int count = 0; do { count = resp_stream.Read(buf, 0, buf.Length); if (count != 0) { temp_string = Encoding.ASCII.GetString(buf, 0, count); sb.Append(temp_string); } }while (count > 0); // add the source into the candidate page object candidate_page.setPageContent(sb.ToString()); // parse the page for links HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(sb.ToString()); var linksOnPage = from lnks in doc.DocumentNode.Descendants() where lnks.Name == "a" && lnks.Attributes["href"] != null && lnks.InnerText.Trim().Length > 0 select new { Url = lnks.Attributes["href"].Value, }; foreach (var link in linksOnPage) { pre_pages.Add(link.Url); } ; // parse out the distinct links on this page, removing any duplicates, and marking illegal links as such List <string> distinct_pre_pages = pre_pages.Distinct().ToList(); for (int m = 0; m < distinct_pre_pages.Count; m++) { string new_url = distinct_pre_pages.ElementAt(m); SpiderLink new_link = new SpiderLink(new_url, spider_object.normalizeUrl(new_url, candidate_page.getUrl()), candidate_page.getUrl()); if (new_link.getNormalizedUrl().Count() < 1) { new_link.setIllegalLink(); } candidate_page.addLinkingToLink(new_link); } // set this candidate page as processed candidate_page._candidate_setDone(); } } // mark this thread as done in _thread_status spider_object.removeThreadStatus(); }