public static void DownloadHTMLString()
{
try
{
DownloadedString = string.Empty;
System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(SourceLink);
request.KeepAlive = true;
request.Headers.Set(HttpRequestHeader.CacheControl, "max-age=0");
request.Headers.Add("Upgrade-Insecure-Requests", @"1");
request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36";
request.Headers.Add("Sec-Fetch-User", @"?1");
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3";
request.Headers.Add("Sec-Fetch-Site", @"same-origin");
request.Headers.Add("Sec-Fetch-Mode", @"navigate");
//request.Referer = "https://www.fastenal.com/product/abrasives/coated-and-non-woven-abrasives/600955?categoryId=600955&level=2&isExpanded=true";
//request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate, br");
request.Headers.Set(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.9");
request.Headers.Set(HttpRequestHeader.Cookie, @"JSESSIONID=5AfPn21Y8SuFclt30UFXjOX4.12a9fabb-3422-3f02-933e-c39f262061e2; NEW_SEARCH_EXPERIENCE=0.057376146; mt.v=2.1128676059.1575031992726; COOKIE_AGREEMENT=0; org.springframework.web.servlet.i18n.CookieLocaleResolver.LOCALE=en_US; _gcl_au=1.1.1259528889.1575032013; __mauuid=2ec53ab1-70ca-43ab-92b5-19945fc27165; __mauuid=2ec53ab1-70ca-43ab-92b5-19945fc27165; __mauuid=2ec53ab1-70ca-43ab-92b5-19945fc27165; _ga=GA1.3.718821677.1575032013; _gid=GA1.3.1437952048.1575032013; _hjid=fdad0035-a120-4927-8d72-689f4ffc18d0; _ga=GA1.2.718821677.1575032013; _gid=GA1.2.1437952048.1575032013");
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Stream responseStream = response.GetResponseStream();
StreamReader streamReader = new StreamReader(responseStream, Encoding.UTF8);
DownloadedString = streamReader.ReadToEnd();
}
catch (WebException ex)
{
HttpWebResponse resp = ex.Response as HttpWebResponse;
if (resp != null && resp.StatusCode == HttpStatusCode.NotFound)
{
fastenal = new fastenal();
Console.WriteLine("Downdstring failed..." + ex.ToString());
fastenal.Category = ex.ToString();
UpdateProductData();
fastenal = null;
}
}
}
public static void ProcessProductLink()
{
try
{
fastenal = new fastenal();
h1 = null;
h1 = new HtmlDocument();
h1.LoadHtml(DownloadedString);
try
{
fastenal.Title = h1.DocumentNode.SelectSingleNode("//div[@class='info--description ']").InnerText.ToString().Trim();
}
catch
{
fastenal.Title = "";
}
try
{
var tabledata = h1.DocumentNode.SelectSingleNode("//table[@class='table general-info__table margin--none']").InnerHtml.ToString();
h2 = null;
h2 = new HtmlDocument();
h2.LoadHtml(tabledata);
foreach (var data in h2.DocumentNode.SelectNodes("//tr"))
{
if (data.InnerText.Contains("Fastenal Part No"))
{
try
{
h3 = null;
h3 = new HtmlDocument();
h3.LoadHtml(data.InnerHtml.ToString());
fastenal.FastePartNo = "#" + h3.DocumentNode.SelectNodes("//td")[1].InnerText.ToString().Trim();
}
catch { fastenal.FastePartNo = ""; }
}
else if (data.InnerText.Contains("Manufacturer Part"))
{
try
{
h3 = null;
h3 = new HtmlDocument();
h3.LoadHtml(data.InnerHtml.ToString());
fastenal.ManuPartNo = "#" + h3.DocumentNode.SelectNodes("//td")[1].InnerText.ToString().Trim();
}
catch { fastenal.ManuPartNo = ""; }
}
else if (data.InnerText.Contains("UNSPSC"))
{
try
{
h3 = null;
h3 = new HtmlDocument();
h3.LoadHtml(data.InnerHtml.ToString());
fastenal.UNSPSC = "#" + h3.DocumentNode.SelectNodes("//td")[1].InnerText.ToString().Trim();
}
catch { fastenal.UNSPSC = ""; }
}
else if (data.InnerText.Contains("Manufacturer"))
{
try
{
h3 = null;
h3 = new HtmlDocument();
h3.LoadHtml(data.InnerHtml.ToString());
fastenal.ManuName = h3.DocumentNode.SelectNodes("//td")[1].InnerText.ToString().Trim();
}
catch { fastenal.ManuName = ""; }
}
}
}
catch { }
try
{
fastenal.WPrice = h1.DocumentNode.SelectSingleNode("//div[@class='whole__sale--label text--highlight color--blue margin-bottom--5']").InnerText.ToString().Replace("\n", "").Replace("Wholesale:", "").Replace(" ", " ").Trim().Split('/').Aggregate((a, b) => a.Trim() + " " + b.Trim());
}
catch { fastenal.WPrice = ""; }
try
{
fastenal.Oprice = h1.DocumentNode.SelectSingleNode("//div[@class='color-highlight text--highlight margin-bottom--5']").InnerText.ToString().Replace("\n", "").Replace("Online Price:", "").Trim().Replace(" ", " ").Split('/').Aggregate((a, b) => a.Trim() + " " + b.Trim());
}
catch { fastenal.Oprice = ""; }
try
{
fastenal.UPrice = h1.DocumentNode.SelectSingleNode("//div[@class='color-highlight text--small margin-bottom--5']").InnerText.ToString().Replace("Unit Price:", "").Replace(" ", " ").Split('/')[0].Replace("\n", "").Trim();
}
catch { fastenal.UPrice = ""; }
//try
//{
// fastenal.UOM = h1.DocumentNode.SelectSingleNode("//div[@class='color-highlight text--small margin-bottom--5']").InnerText.ToString().Replace("Unit Price:", "").Split('/')[1].Replace("\n", "").Trim();
//}
//catch { fastenal.UOM = ""; }
try
{
var spectable = h1.DocumentNode.SelectSingleNode("//table[@class='table product__attribute--info']").InnerHtml.ToString();
h2 = null;
h2 = new HtmlDocument();
h2.LoadHtml(spectable);
var trdata = h2.DocumentNode.SelectNodes("//tr");
foreach (var tr in trdata)
{
try
{
h3 = null;
h3 = new HtmlDocument();
h3.LoadHtml(tr.InnerHtml);
if (tr.InnerText.Contains("UOM"))
{
fastenal.UOM = h3.DocumentNode.SelectNodes("//td")[1].InnerText.ToString().Trim();
}
else
{
fastenal.Spec += h3.DocumentNode.SelectNodes("//td").Select(s => s.InnerText.ToString().Trim()).Aggregate((a, b) => a + " : " + b).ToString() + " | ";
}
}
catch { }
}
}
catch { fastenal.Spec = ""; }
try
{
var imgdiv = h1.DocumentNode.SelectSingleNode("//div[@id='primary-image']").InnerHtml.ToString();
h2 = null;
h2 = new HtmlDocument();
h2.LoadHtml(imgdiv);
fastenal.ImageLink = h2.DocumentNode.SelectNodes("//img").Select(s => "https:" + s.Attributes["src"].Value.ToString()).Aggregate((a, b) => a + " | " + b).ToString();
}
catch { }
try
{
var breadcrum = h1.DocumentNode.SelectSingleNode("//div[@class='breadcrumbs']").InnerHtml.ToString();
h2 = null;
h2 = new HtmlDocument();
h2.LoadHtml(breadcrum);
fastenal.Category = h2.DocumentNode.SelectNodes("//li").Select(s => s.InnerText.ToString().Trim()).Aggregate((a, b) => a + " | " + b).ToString();
}
catch { fastenal.Category = ""; }
}
catch { }
finally
{
h1 = null;
h2 = null;
h3 = null;
DownloadedString = string.Empty;
UpdateProductData();
fastenal = null;
}
}
