public static void GetLinks_ps(string dateUrl, DateTime startDate, DateTime endDate)
{
HtmlWeb web = new HtmlWeb();
HtmlAgilityPack.HtmlDocument doc = web.Load(dateUrl);
string linx = "";
List<string> linkList = new List<string>();
var div = doc.DocumentNode.SelectSingleNode("//*[contains(@class,'stories')]");
HtmlNodeCollection storyLinks;
if (div != null)
{
var links = div.Descendants("a")
.Select(a => a.Attributes["href"].Value)
.ToList();
linkList = div.Descendants("a")
.Select(a => a.Attributes["href"].Value)
.ToList();
linx = string.Join(" , ", links.ToArray());
}
foreach (string l in linkList)
{
if (l.Substring(0, 6) != "/news/")
continue;
if (Convert.ToDateTime(l.Substring(6, 10)) < startDate ||
Convert.ToDateTime(l.Substring(6, 10)) > endDate)
continue;
if (!dicLinks.ContainsKey(l))
{
linkDets ld = new linkDets();
ld.source = "Bloomberg";
ld.date = l.Substring(6, 10);
ld.title = l.Substring(17, l.Length - 17);
ld.url = "http://www.bloomberg.com" + l;
dicLinks.Add(ld.url, ld);
}
}
}
public void GetLinks(string dateUrl)
{
try
{
HtmlWeb web = new HtmlWeb();
HtmlAgilityPack.HtmlDocument doc = web.Load(dateUrl);
string linx = "";
List<string> linkList = new List<string>();
var div = doc.DocumentNode.SelectSingleNode("//*[contains(@class,'stories')]"); //"//div[@class='stories']"
HtmlNodeCollection storyLinks;
if (div != null)
{
var links = div.Descendants("a")
.Select(a => a.Attributes["href"].Value) //.InnerText)
.ToList();
linkList = div.Descendants("a")
.Select(a => a.Attributes["href"].Value) //.InnerText)
.ToList();
linx = string.Join(" , ", links.ToArray());
}
//FILL DICT
foreach (string l in linkList)
{
if (l.Substring(0, 6) != "/news/")
continue;
if (Convert.ToDateTime(l.Substring(6, 10)) < dtpStartDate.Value ||
Convert.ToDateTime(l.Substring(6, 10)) > dtpEndDate.Value)
continue;
if (!dicLinks.ContainsKey(l))
{
linkDets ld = new linkDets();
ld.source = "Bloomberg";
ld.date = l.Substring(6, 10);
ld.title = l.Substring(17, l.Length - 17);
ld.url = "http://www.bloomberg.com" + l;
ld.done = false;
dicLinks.Add(l, ld);
}
}
}
catch (Exception e)
{
using (StreamWriter writer = new StreamWriter("errors-exceptions-Log.txt", true))
{
writer.WriteLine(DateTime.Now.ToString("MM/dd/yy | hh:mm:ss :: ") + "Historical.GetLinks() : ");
}
}
}