// Private methods
private bool ParseFINRAHtml(string html, DateTime date, out StockAdvDecl investGrade, out StockAdvDecl highYield)
{
try
{
WebBrowser browser = new WebBrowser();
browser.ScriptErrorsSuppressed = true;
browser.DocumentText = html;
browser.Document.OpenNew(true);
browser.Document.Write(html);
browser.Refresh();
var table = browser.Document.GetElementsByTagName("table").Cast<HtmlElement>().First();
var data = getTableData(table);
investGrade = new StockAdvDecl()
{
Date = date,
Advances = int.Parse(data[2][2]),
Declines = int.Parse(data[3][2]),
Unchanged = int.Parse(data[4][2]),
Volume = data[7][2] != null ? int.Parse(data[7][2]) : 0
};
highYield = new StockAdvDecl()
{
Date = date,
Advances = int.Parse(data[2][3]),
Declines = int.Parse(data[3][3]),
Unchanged = int.Parse(data[4][3]),
Volume = data[7][3] != null ? int.Parse(data[7][3]) : 0
};
return true;
}
catch (Exception)
{
investGrade = null;
highYield = null;
return false;
}
}
private bool DowloadFromFINRA(DateTime date, out StockAdvDecl investGrade, out StockAdvDecl highYield)
{
bool res = false;
investGrade = null;
highYield = null;
StockWeb.StockWebHelper wh = new StockWeb.StockWebHelper();
string url = "http://finra-markets.morningstar.com/transferPage.jsp?path=http%3A%2F%2Fmuni-internal.morningstar.com%2Fpublic%2FMarketBreadth%2FC&date=$MM%2F$DD%2F$YYYY";
url = url.Replace("$YYYY", date.Year.ToString());
url = url.Replace("$DD", date.Day.ToString());
url = url.Replace("$MM", date.Month.ToString());
//Console.WriteLine(url);
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
req.CookieContainer = new CookieContainer();
foreach (Cookie cookie in Cookies)
{
req.CookieContainer.Add(cookie);
}
req.ContentType = "application/x-www-form-urlencoded";
req.Method = "Get";
req.AllowAutoRedirect = true;
req.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
req.Headers.Add("Accept-Language", "en-US,en;q=0.8,fr;q=0.6");
req.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36";
req.Referer = url;
//DumpHeader("Inner Req", req.Headers);
//DumpCookies(req.CookieContainer);
bool success = false;
int tries = 3;
while (!success && tries > 0)
{
tries--;
try
{
using (HttpWebResponse response = (HttpWebResponse)req.GetResponse())
{
//DumpHeader("Inner Response", response.Headers);
// Get the stream containing content returned by the server.
using (Stream dataStream = response.GetResponseStream())
{
using (StreamReader reader = new StreamReader(dataStream))
{
string html = reader.ReadToEnd();
if (string.IsNullOrWhiteSpace(html)) return false;
return ParseFINRAHtml(html, date, out investGrade, out highYield);
}
}
}
}
catch (Exception)
{
}
}
return false;
}