public override bool DownloadDailyData(string rootFolder, StockSerie stockSerie)
{
string url = @"http://www.gurufocus.com/ownership/%TICKER";
url = url.Replace("%TICKER", stockSerie.ShortName);
ShortInterestSerie siSerie = StockDictionary.StockDictionarySingleton.ShortInterestDictionary[stockSerie.StockName];
siSerie.Initialise();
if (siSerie.Count > 0 && siSerie.Keys.Last() > (DateTime.Today.AddDays(15))) return false;
if (File.GetLastWriteTime(Settings.Default.RootFolder + SHORTINTEREST_FOLDER + stockSerie.ShortName + "_SI.html").Date != DateTime.Today)
{
StockWebHelper swh = new StockWebHelper();
if (swh.DownloadFile(Settings.Default.RootFolder + SHORTINTEREST_FOLDER, stockSerie.ShortName + "_SI.html", url))
{
string html = string.Empty;
using (
StreamReader rw =
new StreamReader(Settings.Default.RootFolder + SHORTINTEREST_FOLDER + stockSerie.ShortName +
"_SI.html"))
{
html = rw.ReadToEnd();
html = html.Remove(0, html.IndexOf("<th>Short Interest</th>"));
html = html.Remove(0, html.IndexOf("<tr><td>"));
html = html.Remove(html.IndexOf("</tbody>"));
html = html.Replace("</td></tr><tr><td>", Environment.NewLine);
html = html.Replace("</td><td>", ";");
html = html.Replace("<tr><td>", "");
html = html.Replace("</td></tr>", "");
}
using (Stream stringStream = new MemoryStream(Encoding.UTF8.GetBytes(html.Trim())))
{
using (StreamReader rw = new StreamReader(stringStream))
{
while (!rw.EndOfStream)
{
string[] fields = rw.ReadLine().Split(';');
ShortInterestValue siValue = new ShortInterestValue(DateTime.Parse(fields[0], usCulture),
float.Parse(fields[1], usCulture), 0f, 0f);
if (!siSerie.ContainsKey(siValue.Date))
{
siSerie.Add(siValue.Date, siValue);
}
}
}
}
siSerie.SaveToFile();
}
}
return false;
}
public override void InitDictionary(string rootFolder, StockDictionary stockDictionary, bool download)
{
SortedDictionary<string, string> cotIncludeList = new SortedDictionary<string, string>();
string[] fields;
string fileName = Settings.Default.RootFolder + @"\COT.cfg";
if (File.Exists(fileName))
{
using (StreamReader sr = new StreamReader(fileName))
{
while (!sr.EndOfStream)
{
fields = sr.ReadLine().Split(';');
if (!stockDictionary.CotDictionary.ContainsKey(fields[0]))
{
stockDictionary.CotDictionary.Add(fields[0], new CotSerie(fields[0]));
if (fields.Length > 1)
{
cotIncludeList.Add(fields[0], fields[1]);
if (stockDictionary.ContainsKey(fields[1]))
{
stockDictionary[fields[1]].CotSerie = stockDictionary.CotDictionary[fields[0]];
}
}
}
}
}
// Check if download Needed
CotSerie SP500CotSerie = stockDictionary["SP500"].CotSerie;
if (SP500CotSerie != null)
{
if (SP500CotSerie.Initialise())
{
DateTime lastDate = DateTime.Today.AddMonths(-2);
lastDate = SP500CotSerie.Keys.Last();
if ((DateTime.Today - lastDate) > new TimeSpan(10, 0, 0, 0, 0))
{
// Need to download new COT
StockWebHelper swh = new StockWebHelper();
bool upToDate = false;
NotifyProgress("Downloading commitment of traders data...");
if (swh.DownloadCOT(Settings.Default.RootFolder, ref upToDate))
{
NotifyProgress("Parsing commitment of traders data...");
ParseFullCotSeries(cotIncludeList, stockDictionary);
}
}
}
}
}
}
public bool DownloadDailyData2(string rootFolder, StockSerie stockSerie)
{
string url = @"http://www.nasdaq.com/symbol/%TICKER/short-interest";
url = url.Replace("%TICKER", stockSerie.ShortName);
ShortInterestSerie siSerie = StockDictionary.StockDictionarySingleton.ShortInterestDictionary[stockSerie.StockName];
siSerie.Initialise();
if (siSerie.Count > 0 && siSerie.Keys.Last() > (DateTime.Today.AddDays(15))) return false;
StockWebHelper swh = new StockWebHelper();
if (swh.DownloadFile(Settings.Default.RootFolder + SHORTINTEREST_FOLDER, stockSerie.ShortName + "_SI.html", url))
{
string html = string.Empty;
using (StreamReader rw = new StreamReader(Settings.Default.RootFolder + SHORTINTEREST_FOLDER + stockSerie.ShortName + "_SI.html"))
{
html = rw.ReadToEnd();
html = html.Remove(0, html.IndexOf("Settlement Date"));
html = html.Remove(0, html.IndexOf("<tr>"));
html = html.Remove(html.IndexOf("</tbody>"));
html = html.Replace("</tr>", "");
html = html.Replace("<tr>", "");
html = html.Replace("</td><td>", ";");
html = html.Replace("<td>", "");
html = html.Replace("</td>", "");
html = html.Replace("\t", "");
html = html.Replace("\r\n\r\n", "\r\n");
int index = html.IndexOf("<tr>");
}
using (Stream stringStream = new MemoryStream(Encoding.UTF8.GetBytes(html)))
{
using (StreamReader rw = new StreamReader(stringStream))
{
string line = rw.ReadLine();
while (!rw.EndOfStream)
{
string[] fields = rw.ReadLine().Split(';');
ShortInterestValue siValue = new ShortInterestValue(DateTime.Parse(fields[0], usCulture), float.Parse(fields[1], usCulture), float.Parse(fields[2], usCulture), float.Parse(fields[3], usCulture));
if (!siSerie.ContainsKey(siValue.Date))
{
siSerie.Add(siValue.Date, siValue);
}
}
}
}
siSerie.SaveToFile();
File.Delete(Settings.Default.RootFolder + SHORTINTEREST_FOLDER + stockSerie.ShortName + "_SI.html");
}
return false;
}
//public static void DownloadFinancial2(StockSerie stockSerie)
//{
// if (stockSerie.StockAnalysis.Financial != null && stockSerie.StockAnalysis.Financial.DownloadDate.AddDays(7) > DateTime.Now) return;
// string url = "http://www.abcbourse.com/analyses/chiffres.aspx?s=$ShortNamep".Replace("$ShortName", stockSerie.ShortName);
// url = "http://www.boursorama.com/bourse/profil/profil_finance.phtml?symbole=1rP$ShortName".Replace("$ShortName", stockSerie.ShortName);
// StockWebHelper swh = new StockWebHelper();
// string html = swh.DownloadHtml(url);
// WebBrowser browser = new WebBrowser();
// browser.ScriptErrorsSuppressed = true;
// browser.DocumentText = html;
// browser.Document.OpenNew(true);
// browser.Document.Write(html);
// browser.Refresh();
// HtmlDocument doc = browser.Document;
// HtmlElementCollection tables = doc.GetElementsByTagName("div");
// List<List<string>> data = new List<List<string>>();
// StockFinancial financial = new StockFinancial();
// HtmlElement tbl = tables.Cast<HtmlElement>().FirstOrDefault(t => t.InnerText.StartsWith("Marché"));
// if (tbl != null)
// {
// //ParseFinancialGeneral(stockSerie, financial, tbl);
// }
// bool found = false;
// int count = 0;
// foreach (HtmlElement table in tables)
// {
// if (found)
// {
// switch (count)
// {
// case 0:
// financial.IncomeStatement = getTableData(table);
// count++;
// break;
// case 1:
// financial.BalanceSheet = getTableData(table);
// count++;
// break;
// case 2:
// financial.Ratios = getTableData(table);
// count++;
// break;
// case 3:
// financial.Quaterly = getTableData(table);
// count++;
// break;
// }
// }
// else
// {
// found = table.InnerText.StartsWith("Compte de");
// }
// }
// if (found)
// tbl = tables.Cast<HtmlElement>().FirstOrDefault(t => t.InnerText.StartsWith("Compte"));
// if (tbl != null)
// {
// ParseFinancialDetails(stockSerie, financial, tbl);
// }
//}
public static void DownloadFinancialSummary(StockFinancial financial, string shortName)
{
string url = "http://www.boursorama.com/bourse/profil/resume_societe.phtml?symbole=1r$ShortName".Replace("$ShortName", shortName);
StockWebHelper swh = new StockWebHelper();
string html = swh.DownloadHtml(url);
WebBrowser browser = new WebBrowser();
browser.ScriptErrorsSuppressed = true;
browser.DocumentText = html;
browser.Document.OpenNew(true);
browser.Document.Write(html);
browser.Refresh();
HtmlDocument doc = browser.Document;
var divs = doc.GetElementsByTagName("div").Cast<HtmlElement>();
foreach (var div in divs)
{
if (div.InnerText != null && div.InnerText.StartsWith("Nombre de titres"))
{
var list = div.InnerText.Replace(Environment.NewLine, "|");
var split = list.Split('|');
var nbTitres = split[0].Split(':')[1].Replace(" ", "");
financial.ShareNumber = long.Parse(nbTitres);
financial.Coupon = split.First(l => l.StartsWith("Dern")).Split(':')[1].Trim();
financial.Sector = split.First(l => l.StartsWith("Secteur")).Split(':')[1].Trim();
financial.PEA = split.First(l => l.Contains("PEA")).Split(':')[1].Trim();
financial.SRD = split.First(l => l.Contains("SRD")).Split(':')[1].Trim();
financial.Indices = split.First(l => l.StartsWith("Indice")).Split(':')[1].Trim();
break;
}
}
foreach (var div in divs)
{
if (div.InnerText != null && div.InnerText.StartsWith("Prévisions des analystes"))
{
var tables = div.GetElementsByTagName("table").Cast<HtmlElement>();
var previsions = getTableData(tables.First());
var dividendLine = previsions.FirstOrDefault(l => l[0] == "Dividende");
if (dividendLine != null)
{
float dividend = 0;
float.TryParse(dividendLine[1], out dividend);
financial.Dividend = dividend;
}
break;
}
}
//foreach (var div in divs)
//{
// if (div.InnerText != null && div.InnerText.StartsWith("Activité"))
// {
// Console.WriteLine(div.InnerText);
// financial.Activity = div.InnerHtml;
// break;
// }
//}
}
public static void DownloadFinancial(StockSerie stockSerie)
{
if (stockSerie.Financial != null && stockSerie.Financial.DownloadDate.AddDays(7) > DateTime.Now) return;
StockFinancial financial = new StockFinancial();
try
{
string shortName = stockSerie.StockGroup == StockSerie.Groups.ALTERNEXT ? "EP" : "P";
shortName += stockSerie.ShortName;
DownloadFinancialSummary(financial, shortName);
string url = "http://www.boursorama.com/bourse/profil/profil_finance.phtml?symbole=1r$ShortName".Replace("$ShortName", shortName);
StockWebHelper swh = new StockWebHelper();
string html = swh.DownloadHtml(url);
WebBrowser browser = new WebBrowser();
browser.ScriptErrorsSuppressed = true;
browser.DocumentText = html;
browser.Document.OpenNew(true);
browser.Document.Write(html);
browser.Refresh();
HtmlDocument doc = browser.Document;
var divs = doc.GetElementsByTagName("div").Cast<HtmlElement>();
foreach (var div in divs)
{
if (div.InnerText != null && div.InnerText.StartsWith("Compte de"))
{
Console.WriteLine(div.InnerText);
var tables = div.GetElementsByTagName("table").Cast<HtmlElement>();
financial.IncomeStatement = getTableData(tables.First());
break;
}
}
foreach (var div in divs)
{
if (div.InnerText != null && div.InnerText.StartsWith("Bilan"))
{
Console.WriteLine(div.InnerText);
var tables = div.GetElementsByTagName("table").Cast<HtmlElement>();
financial.BalanceSheet = getTableData(tables.First());
break;
}
}
foreach (var div in divs)
{
if (div.InnerText != null && div.InnerText.StartsWith("Chiffres d'affaires"))
{
Console.WriteLine(div.InnerText);
var tables = div.GetElementsByTagName("table").Cast<HtmlElement>();
financial.Quaterly = getTableData(tables.First());
break;
}
}
financial.DownloadDate = DateTime.Now;
stockSerie.Financial = financial;
stockSerie.SaveFinancial();
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
//private static void ParseFinancialGeneral(StockSerie stockSerie, StockFinancial financial, HtmlElement tbl)
//{
// List<List<string>> data = getTableData(tbl);
// foreach (var row in data)
// {
// if (row.Count == 2)
// {
// switch (row[0].Trim())
// {
// case "Marché":
// financial.Market = row[1];
// break;
// case "Nombre de titres":
// financial.ShareNumber = long.Parse(row[1].Replace(" ", ""));
// break;
// case "Place de cotation":
// financial.MarketPlace = row[1];
// break;
// case "Secteur d'activité":
// financial.Sector = row[1];
// break;
// case "Eligible au SRD":
// financial.SRD = row[1];
// break;
// case "Eligible au PEA":
// financial.PEA = row[1];
// break;
// case "Indices":
// financial.Indices = row[1];
// break;
// case "Capitalisation (milliers d'euros)":
// //financial.MarketCap = int.Parse(row[1].Replace(" ", ""));
// break;
// case "Rendement":
// float yield = 0f;
// if (float.TryParse(row[1].Replace(",", ".").Replace("%", ""), out yield))
// {
// financial.Yield = yield / 100f;
// }
// break;
// case "Dividende (Date de versement)":
// financial.Dividend = row[1];
// break;
// case "Date Assemblée Générale":
// financial.MeetingDate = row[1];
// break;
// }
// }
// financial.DownloadDate = DateTime.Now;
// stockSerie.StockAnalysis.Financial = financial;
// }
//}
public static void DownloadAgenda(StockSerie stockSerie)
{
string url = "http://www.abcbourse.com/marches/events.aspx?s=$ShortNamep".Replace("$ShortName",
stockSerie.ShortName);
StockWebHelper swh = new StockWebHelper();
string html = swh.DownloadHtml(url);
WebBrowser browser = new WebBrowser();
browser.ScriptErrorsSuppressed = true;
browser.DocumentText = html;
browser.Document.OpenNew(true);
browser.Document.Write(html);
browser.Refresh();
HtmlDocument doc = browser.Document;
HtmlElementCollection tables = doc.GetElementsByTagName("table");
List<List<string>> data = new List<List<string>>();
foreach (HtmlElement tbl in tables)
{
if (tbl.InnerText.StartsWith("Date"))
{
data = getTableData(tbl).Skip(1).ToList();
break;
}
}
//
foreach (var row in data)
{
if (row[0].StartsWith("du")) row[0] = row[0].Substring(row[0].IndexOf("au ") + 3);
DateTime date = DateTime.Parse(row[0]);
string comment = row[1];
if (row[2] != null) comment += Environment.NewLine + row[2];
if (!stockSerie.StockAnalysis.Comments.ContainsKey(date))
{
stockSerie.StockAnalysis.Comments.Add(date, comment);
}
else
{
if (!stockSerie.StockAnalysis.Comments[date].Contains(comment))
{
stockSerie.StockAnalysis.Comments[date] = stockSerie.StockAnalysis.Comments[date] +
Environment.NewLine + comment;
}
}
}
}