public void ParseHTML(string link) { //HtmlWeb web = new HtmlWeb(); var web = new HtmlWeb() { PreRequest = request => { // Make any changes to the request object that will be used. request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip; return(true); } }; WebClient w = new WebClient(); WebException except = new WebException(); try { string s = w.DownloadString(link); var htmlDoc = web.Load(link); var node = htmlDoc.DocumentNode.Descendants("title").FirstOrDefault(); string title = "No Title"; if (node != null) { title = node.InnerText; } if (title.Equals("Error")) { sm.AddErrorToPerformance(link, title); } else { // Get Date if possible HtmlNode mdnode = htmlDoc.DocumentNode.SelectSingleNode("//meta[@name='pubdate']"); if (mdnode != null) { HtmlAttribute desc; desc = mdnode.Attributes["content"]; string date = desc.Value; string[] wordsInTitle = title.Split(' '); if (date.Length > 8) { DateTime d = this.StringToDateTime(date); // Go word by word and insert into the table for (int i = 0; i < wordsInTitle.Length; i++) { // To LowerCase string toAdd = wordsInTitle[i].ToLower(); sm.AddLinkToTableStorage(link, toAdd, title, d); } } else { // Go word by word and insert into the table for (int i = 0; i < wordsInTitle.Length; i++) { // ToLowerCase string toAdd = wordsInTitle[i].ToLower(); sm.AddLinkToTableStorage(link, toAdd, title); } } } sm.AddLinkToPerformance(link, title); // Extracting Links: List <string> linksOnPage = this.ExtractAllAHrefTags(htmlDoc); List <string> addToURLQueue = this.ExamineLinksOnPage(linksOnPage); this.AddToURLQueue(addToURLQueue); } } catch (WebException e) { except = e; sm.AddErrorToPerformance(e.ToString(), "Catched Error"); } }