public static void DownloadHTMLString() { try { DownloadedString = string.Empty; System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12; HttpWebRequest request = (HttpWebRequest)WebRequest.Create(SourceLink); request.KeepAlive = true; request.Headers.Set(HttpRequestHeader.CacheControl, "max-age=0"); request.Headers.Add("Upgrade-Insecure-Requests", @"1"); request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"; request.Headers.Add("Sec-Fetch-User", @"?1"); request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"; request.Headers.Add("Sec-Fetch-Site", @"same-origin"); request.Headers.Add("Sec-Fetch-Mode", @"navigate"); //request.Referer = "https://www.fastenal.com/product/abrasives/coated-and-non-woven-abrasives/600955?categoryId=600955&level=2&isExpanded=true"; //request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate, br"); request.Headers.Set(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.9"); request.Headers.Set(HttpRequestHeader.Cookie, @"JSESSIONID=5AfPn21Y8SuFclt30UFXjOX4.12a9fabb-3422-3f02-933e-c39f262061e2; NEW_SEARCH_EXPERIENCE=0.057376146; mt.v=2.1128676059.1575031992726; COOKIE_AGREEMENT=0; org.springframework.web.servlet.i18n.CookieLocaleResolver.LOCALE=en_US; _gcl_au=1.1.1259528889.1575032013; __mauuid=2ec53ab1-70ca-43ab-92b5-19945fc27165; __mauuid=2ec53ab1-70ca-43ab-92b5-19945fc27165; __mauuid=2ec53ab1-70ca-43ab-92b5-19945fc27165; _ga=GA1.3.718821677.1575032013; _gid=GA1.3.1437952048.1575032013; _hjid=fdad0035-a120-4927-8d72-689f4ffc18d0; _ga=GA1.2.718821677.1575032013; _gid=GA1.2.1437952048.1575032013"); HttpWebResponse response = (HttpWebResponse)request.GetResponse(); Stream responseStream = response.GetResponseStream(); StreamReader streamReader = new StreamReader(responseStream, Encoding.UTF8); DownloadedString = streamReader.ReadToEnd(); } catch (WebException ex) { HttpWebResponse resp = ex.Response as HttpWebResponse; if (resp != null && resp.StatusCode == HttpStatusCode.NotFound) { fastenal = new fastenal(); Console.WriteLine("Downdstring failed..." + ex.ToString()); fastenal.Category = ex.ToString(); UpdateProductData(); fastenal = null; } } }
public static void ProcessProductLink() { try { fastenal = new fastenal(); h1 = null; h1 = new HtmlDocument(); h1.LoadHtml(DownloadedString); try { fastenal.Title = h1.DocumentNode.SelectSingleNode("//div[@class='info--description ']").InnerText.ToString().Trim(); } catch { fastenal.Title = ""; } try { var tabledata = h1.DocumentNode.SelectSingleNode("//table[@class='table general-info__table margin--none']").InnerHtml.ToString(); h2 = null; h2 = new HtmlDocument(); h2.LoadHtml(tabledata); foreach (var data in h2.DocumentNode.SelectNodes("//tr")) { if (data.InnerText.Contains("Fastenal Part No")) { try { h3 = null; h3 = new HtmlDocument(); h3.LoadHtml(data.InnerHtml.ToString()); fastenal.FastePartNo = "#" + h3.DocumentNode.SelectNodes("//td")[1].InnerText.ToString().Trim(); } catch { fastenal.FastePartNo = ""; } } else if (data.InnerText.Contains("Manufacturer Part")) { try { h3 = null; h3 = new HtmlDocument(); h3.LoadHtml(data.InnerHtml.ToString()); fastenal.ManuPartNo = "#" + h3.DocumentNode.SelectNodes("//td")[1].InnerText.ToString().Trim(); } catch { fastenal.ManuPartNo = ""; } } else if (data.InnerText.Contains("UNSPSC")) { try { h3 = null; h3 = new HtmlDocument(); h3.LoadHtml(data.InnerHtml.ToString()); fastenal.UNSPSC = "#" + h3.DocumentNode.SelectNodes("//td")[1].InnerText.ToString().Trim(); } catch { fastenal.UNSPSC = ""; } } else if (data.InnerText.Contains("Manufacturer")) { try { h3 = null; h3 = new HtmlDocument(); h3.LoadHtml(data.InnerHtml.ToString()); fastenal.ManuName = h3.DocumentNode.SelectNodes("//td")[1].InnerText.ToString().Trim(); } catch { fastenal.ManuName = ""; } } } } catch { } try { fastenal.WPrice = h1.DocumentNode.SelectSingleNode("//div[@class='whole__sale--label text--highlight color--blue margin-bottom--5']").InnerText.ToString().Replace("\n", "").Replace("Wholesale:", "").Replace(" ", " ").Trim().Split('/').Aggregate((a, b) => a.Trim() + " " + b.Trim()); } catch { fastenal.WPrice = ""; } try { fastenal.Oprice = h1.DocumentNode.SelectSingleNode("//div[@class='color-highlight text--highlight margin-bottom--5']").InnerText.ToString().Replace("\n", "").Replace("Online Price:", "").Trim().Replace(" ", " ").Split('/').Aggregate((a, b) => a.Trim() + " " + b.Trim()); } catch { fastenal.Oprice = ""; } try { fastenal.UPrice = h1.DocumentNode.SelectSingleNode("//div[@class='color-highlight text--small margin-bottom--5']").InnerText.ToString().Replace("Unit Price:", "").Replace(" ", " ").Split('/')[0].Replace("\n", "").Trim(); } catch { fastenal.UPrice = ""; } //try //{ // fastenal.UOM = h1.DocumentNode.SelectSingleNode("//div[@class='color-highlight text--small margin-bottom--5']").InnerText.ToString().Replace("Unit Price:", "").Split('/')[1].Replace("\n", "").Trim(); //} //catch { fastenal.UOM = ""; } try { var spectable = h1.DocumentNode.SelectSingleNode("//table[@class='table product__attribute--info']").InnerHtml.ToString(); h2 = null; h2 = new HtmlDocument(); h2.LoadHtml(spectable); var trdata = h2.DocumentNode.SelectNodes("//tr"); foreach (var tr in trdata) { try { h3 = null; h3 = new HtmlDocument(); h3.LoadHtml(tr.InnerHtml); if (tr.InnerText.Contains("UOM")) { fastenal.UOM = h3.DocumentNode.SelectNodes("//td")[1].InnerText.ToString().Trim(); } else { fastenal.Spec += h3.DocumentNode.SelectNodes("//td").Select(s => s.InnerText.ToString().Trim()).Aggregate((a, b) => a + " : " + b).ToString() + " | "; } } catch { } } } catch { fastenal.Spec = ""; } try { var imgdiv = h1.DocumentNode.SelectSingleNode("//div[@id='primary-image']").InnerHtml.ToString(); h2 = null; h2 = new HtmlDocument(); h2.LoadHtml(imgdiv); fastenal.ImageLink = h2.DocumentNode.SelectNodes("//img").Select(s => "https:" + s.Attributes["src"].Value.ToString()).Aggregate((a, b) => a + " | " + b).ToString(); } catch { } try { var breadcrum = h1.DocumentNode.SelectSingleNode("//div[@class='breadcrumbs']").InnerHtml.ToString(); h2 = null; h2 = new HtmlDocument(); h2.LoadHtml(breadcrum); fastenal.Category = h2.DocumentNode.SelectNodes("//li").Select(s => s.InnerText.ToString().Trim()).Aggregate((a, b) => a + " | " + b).ToString(); } catch { fastenal.Category = ""; } } catch { } finally { h1 = null; h2 = null; h3 = null; DownloadedString = string.Empty; UpdateProductData(); fastenal = null; } }