// This function gets a url path and an xquery and returns an array of the values in the xquery taken from the path
static HtmlNode[] GetTable(string path, string xquery)
{
using (WebClient client = new WebClient())
{
client.Headers["User-Agent"] =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36";
client.UseDefaultCredentials = true;
HtmlDocument htmlDoc = new HtmlDocument();
try
{
htmlDoc.LoadHtml(client.DownloadString(path));
htmlDoc.Save(root + "websiteDump.txt");
HtmlNodeCollection currentData = htmlDoc.DocumentNode.SelectNodes(xquery);
return(currentData.Cast <HtmlNode>().ToArray());
}
catch
{
Console.WriteLine("Didn't get to the server");
return(null);
}
}
}