public override bool DownloadDailyData(string rootFolder, StockSerie stockSerie)
        {
            string url = @"http://www.gurufocus.com/ownership/%TICKER";
             url = url.Replace("%TICKER", stockSerie.ShortName);

             ShortInterestSerie siSerie = StockDictionary.StockDictionarySingleton.ShortInterestDictionary[stockSerie.StockName];
             siSerie.Initialise();

             if (siSerie.Count > 0 && siSerie.Keys.Last() > (DateTime.Today.AddDays(15))) return false;

             if (File.GetLastWriteTime(Settings.Default.RootFolder + SHORTINTEREST_FOLDER + stockSerie.ShortName + "_SI.html").Date != DateTime.Today)
             {
            StockWebHelper swh = new StockWebHelper();
            if (swh.DownloadFile(Settings.Default.RootFolder + SHORTINTEREST_FOLDER, stockSerie.ShortName + "_SI.html", url))
            {
               string html = string.Empty;
               using (
                  StreamReader rw =
                     new StreamReader(Settings.Default.RootFolder + SHORTINTEREST_FOLDER + stockSerie.ShortName +
                                      "_SI.html"))
               {
                  html = rw.ReadToEnd();
                  html = html.Remove(0, html.IndexOf("<th>Short Interest</th>"));
                  html = html.Remove(0, html.IndexOf("<tr><td>"));
                  html = html.Remove(html.IndexOf("</tbody>"));

                  html = html.Replace("</td></tr><tr><td>", Environment.NewLine);

                  html = html.Replace("</td><td>", ";");

                  html = html.Replace("<tr><td>", "");
                  html = html.Replace("</td></tr>", "");
               }
               using (Stream stringStream = new MemoryStream(Encoding.UTF8.GetBytes(html.Trim())))
               {
                  using (StreamReader rw = new StreamReader(stringStream))
                  {
                     while (!rw.EndOfStream)
                     {
                        string[] fields = rw.ReadLine().Split(';');
                        ShortInterestValue siValue = new ShortInterestValue(DateTime.Parse(fields[0], usCulture),
                           float.Parse(fields[1], usCulture), 0f, 0f);

                        if (!siSerie.ContainsKey(siValue.Date))
                        {
                           siSerie.Add(siValue.Date, siValue);
                        }
                     }
                  }
               }

               siSerie.SaveToFile();
            }
             }
             return false;
        }
        public override void InitDictionary(string rootFolder, StockDictionary stockDictionary, bool download)
        {
            SortedDictionary<string, string> cotIncludeList = new SortedDictionary<string, string>();
             string[] fields;

             string fileName = Settings.Default.RootFolder + @"\COT.cfg";
             if (File.Exists(fileName))
             {
            using (StreamReader sr = new StreamReader(fileName))
            {
               while (!sr.EndOfStream)
               {
                  fields = sr.ReadLine().Split(';');
                  if (!stockDictionary.CotDictionary.ContainsKey(fields[0]))
                  {
                     stockDictionary.CotDictionary.Add(fields[0], new CotSerie(fields[0]));
                     if (fields.Length > 1)
                     {
                        cotIncludeList.Add(fields[0], fields[1]);
                        if (stockDictionary.ContainsKey(fields[1]))
                        {
                           stockDictionary[fields[1]].CotSerie = stockDictionary.CotDictionary[fields[0]];
                        }
                     }
                  }
               }
            }
            // Check if download Needed
            CotSerie SP500CotSerie = stockDictionary["SP500"].CotSerie;
            if (SP500CotSerie != null)
            {
               if (SP500CotSerie.Initialise())
               {
                   DateTime lastDate = DateTime.Today.AddMonths(-2);
                      lastDate = SP500CotSerie.Keys.Last();
                  if ((DateTime.Today - lastDate) > new TimeSpan(10, 0, 0, 0, 0))
                  {
                     // Need to download new COT
                     StockWebHelper swh = new StockWebHelper();
                     bool upToDate = false;

                     NotifyProgress("Downloading commitment of traders data...");

                     if (swh.DownloadCOT(Settings.Default.RootFolder, ref upToDate))
                     {
                        NotifyProgress("Parsing commitment of traders data...");
                        ParseFullCotSeries(cotIncludeList, stockDictionary);
                     }
                  }
               }
            }
             }
        }
        public bool DownloadDailyData2(string rootFolder, StockSerie stockSerie)
        {
            string url = @"http://www.nasdaq.com/symbol/%TICKER/short-interest";
             url = url.Replace("%TICKER", stockSerie.ShortName);

             ShortInterestSerie siSerie = StockDictionary.StockDictionarySingleton.ShortInterestDictionary[stockSerie.StockName];
             siSerie.Initialise();

             if (siSerie.Count > 0 && siSerie.Keys.Last() > (DateTime.Today.AddDays(15))) return false;

             StockWebHelper swh = new StockWebHelper();
             if (swh.DownloadFile(Settings.Default.RootFolder + SHORTINTEREST_FOLDER, stockSerie.ShortName + "_SI.html", url))
             {
            string html = string.Empty;
            using (StreamReader rw = new StreamReader(Settings.Default.RootFolder + SHORTINTEREST_FOLDER + stockSerie.ShortName + "_SI.html"))
            {
               html = rw.ReadToEnd();
               html = html.Remove(0, html.IndexOf("Settlement Date"));
               html = html.Remove(0, html.IndexOf("<tr>"));
               html = html.Remove(html.IndexOf("</tbody>"));

               html = html.Replace("</tr>", "");
               html = html.Replace("<tr>", "");

               html = html.Replace("</td><td>", ";");

               html = html.Replace("<td>", "");
               html = html.Replace("</td>", "");
               html = html.Replace("\t", "");
               html = html.Replace("\r\n\r\n", "\r\n");
               int index = html.IndexOf("<tr>");
            }
            using (Stream stringStream = new MemoryStream(Encoding.UTF8.GetBytes(html)))
            {
               using (StreamReader rw = new StreamReader(stringStream))
               {
                  string line = rw.ReadLine();
                  while (!rw.EndOfStream)
                  {
                     string[] fields = rw.ReadLine().Split(';');
                     ShortInterestValue siValue = new ShortInterestValue(DateTime.Parse(fields[0], usCulture), float.Parse(fields[1], usCulture), float.Parse(fields[2], usCulture), float.Parse(fields[3], usCulture));

                     if (!siSerie.ContainsKey(siValue.Date))
                     {
                        siSerie.Add(siValue.Date, siValue);
                     }
                  }
               }
            }

            siSerie.SaveToFile();

            File.Delete(Settings.Default.RootFolder + SHORTINTEREST_FOLDER + stockSerie.ShortName + "_SI.html");
             }
             return false;
        }
        //public static void DownloadFinancial2(StockSerie stockSerie)
        //{
        //    if (stockSerie.StockAnalysis.Financial != null && stockSerie.StockAnalysis.Financial.DownloadDate.AddDays(7) > DateTime.Now) return;
        //    string url = "http://www.abcbourse.com/analyses/chiffres.aspx?s=$ShortNamep".Replace("$ShortName", stockSerie.ShortName);
        //    url = "http://www.boursorama.com/bourse/profil/profil_finance.phtml?symbole=1rP$ShortName".Replace("$ShortName", stockSerie.ShortName);
        //    StockWebHelper swh = new StockWebHelper();
        //    string html = swh.DownloadHtml(url);
        //    WebBrowser browser = new WebBrowser();
        //    browser.ScriptErrorsSuppressed = true;
        //    browser.DocumentText = html;
        //    browser.Document.OpenNew(true);
        //    browser.Document.Write(html);
        //    browser.Refresh();
        //    HtmlDocument doc = browser.Document;
        //    HtmlElementCollection tables = doc.GetElementsByTagName("div");
        //    List<List<string>> data = new List<List<string>>();
        //    StockFinancial financial = new StockFinancial();
        //    HtmlElement tbl = tables.Cast<HtmlElement>().FirstOrDefault(t => t.InnerText.StartsWith("Marché"));
        //    if (tbl != null)
        //    {
        //        //ParseFinancialGeneral(stockSerie, financial, tbl);
        //    }
        //    bool found = false;
        //    int count = 0;
        //    foreach (HtmlElement table in tables)
        //    {
        //        if (found)
        //        {
        //            switch (count)
        //            {
        //                case 0:
        //                    financial.IncomeStatement = getTableData(table);
        //                    count++;
        //                    break;
        //                case 1:
        //                    financial.BalanceSheet = getTableData(table);
        //                    count++;
        //                    break;
        //                case 2:
        //                    financial.Ratios = getTableData(table);
        //                    count++;
        //                    break;
        //                case 3:
        //                    financial.Quaterly = getTableData(table);
        //                    count++;
        //                    break;
        //            }
        //        }
        //        else
        //        {
        //            found = table.InnerText.StartsWith("Compte de");
        //        }
        //    }
        //    if (found)
        //        tbl = tables.Cast<HtmlElement>().FirstOrDefault(t => t.InnerText.StartsWith("Compte"));
        //    if (tbl != null)
        //    {
        //        ParseFinancialDetails(stockSerie, financial, tbl);
        //    }
        //}
        public static void DownloadFinancialSummary(StockFinancial financial, string shortName)
        {
            string url = "http://www.boursorama.com/bourse/profil/resume_societe.phtml?symbole=1r$ShortName".Replace("$ShortName", shortName);
            StockWebHelper swh = new StockWebHelper();
            string html = swh.DownloadHtml(url);

            WebBrowser browser = new WebBrowser();
            browser.ScriptErrorsSuppressed = true;
            browser.DocumentText = html;
            browser.Document.OpenNew(true);
            browser.Document.Write(html);
            browser.Refresh();

            HtmlDocument doc = browser.Document;

            var divs = doc.GetElementsByTagName("div").Cast<HtmlElement>();
            foreach (var div in divs)
            {
                if (div.InnerText != null && div.InnerText.StartsWith("Nombre de titres"))
                {
                    var list = div.InnerText.Replace(Environment.NewLine, "|");
                    var split = list.Split('|');
                    var nbTitres = split[0].Split(':')[1].Replace(" ", "");
                    financial.ShareNumber = long.Parse(nbTitres);
                    financial.Coupon = split.First(l => l.StartsWith("Dern")).Split(':')[1].Trim();
                    financial.Sector = split.First(l => l.StartsWith("Secteur")).Split(':')[1].Trim();
                    financial.PEA = split.First(l => l.Contains("PEA")).Split(':')[1].Trim();
                    financial.SRD = split.First(l => l.Contains("SRD")).Split(':')[1].Trim();
                    financial.Indices = split.First(l => l.StartsWith("Indice")).Split(':')[1].Trim();
                    break;
                }
            }
            foreach (var div in divs)
            {
                if (div.InnerText != null && div.InnerText.StartsWith("Prévisions des analystes"))
                {

                    var tables = div.GetElementsByTagName("table").Cast<HtmlElement>();
                    var previsions = getTableData(tables.First());

                    var dividendLine = previsions.FirstOrDefault(l => l[0] == "Dividende");
                    if (dividendLine != null)
                    {
                        float dividend = 0;
                        float.TryParse(dividendLine[1], out dividend);
                        financial.Dividend = dividend;
                    }

                    break;
                }
            }
            //foreach (var div in divs)
            //{
            //    if (div.InnerText != null && div.InnerText.StartsWith("Activité"))
            //    {
            //        Console.WriteLine(div.InnerText);
            //        financial.Activity = div.InnerHtml;
            //        break;
            //    }
            //}
        }
        public static void DownloadFinancial(StockSerie stockSerie)
        {
            if (stockSerie.Financial != null && stockSerie.Financial.DownloadDate.AddDays(7) > DateTime.Now) return;

            StockFinancial financial = new StockFinancial();
            try
            {
                string shortName = stockSerie.StockGroup == StockSerie.Groups.ALTERNEXT ? "EP" : "P";
                shortName += stockSerie.ShortName;
                DownloadFinancialSummary(financial, shortName);

                string url = "http://www.boursorama.com/bourse/profil/profil_finance.phtml?symbole=1r$ShortName".Replace("$ShortName", shortName);
                StockWebHelper swh = new StockWebHelper();
                string html = swh.DownloadHtml(url);

                WebBrowser browser = new WebBrowser();
                browser.ScriptErrorsSuppressed = true;
                browser.DocumentText = html;
                browser.Document.OpenNew(true);
                browser.Document.Write(html);
                browser.Refresh();

                HtmlDocument doc = browser.Document;

                var divs = doc.GetElementsByTagName("div").Cast<HtmlElement>();
                foreach (var div in divs)
                {
                    if (div.InnerText != null && div.InnerText.StartsWith("Compte de"))
                    {
                        Console.WriteLine(div.InnerText);
                        var tables = div.GetElementsByTagName("table").Cast<HtmlElement>();
                        financial.IncomeStatement = getTableData(tables.First());
                        break;
                    }
                }
                foreach (var div in divs)
                {
                    if (div.InnerText != null && div.InnerText.StartsWith("Bilan"))
                    {
                        Console.WriteLine(div.InnerText);
                        var tables = div.GetElementsByTagName("table").Cast<HtmlElement>();
                        financial.BalanceSheet = getTableData(tables.First());
                        break;
                    }
                }
                foreach (var div in divs)
                {
                    if (div.InnerText != null && div.InnerText.StartsWith("Chiffres d'affaires"))
                    {
                        Console.WriteLine(div.InnerText);
                        var tables = div.GetElementsByTagName("table").Cast<HtmlElement>();
                        financial.Quaterly = getTableData(tables.First());
                        break;
                    }
                }

                financial.DownloadDate = DateTime.Now;
                stockSerie.Financial = financial;
                stockSerie.SaveFinancial();
            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message);
            }
        }
        //private static void ParseFinancialGeneral(StockSerie stockSerie, StockFinancial financial, HtmlElement tbl)
        //{
        //    List<List<string>> data = getTableData(tbl);
        //    foreach (var row in data)
        //    {
        //        if (row.Count == 2)
        //        {
        //            switch (row[0].Trim())
        //            {
        //                case "Marché":
        //                    financial.Market = row[1];
        //                    break;
        //                case "Nombre de titres":
        //                    financial.ShareNumber = long.Parse(row[1].Replace(" ", ""));
        //                    break;
        //                case "Place de cotation":
        //                    financial.MarketPlace = row[1];
        //                    break;
        //                case "Secteur d'activité":
        //                    financial.Sector = row[1];
        //                    break;
        //                case "Eligible au SRD":
        //                    financial.SRD = row[1];
        //                    break;
        //                case "Eligible au PEA":
        //                    financial.PEA = row[1];
        //                    break;
        //                case "Indices":
        //                    financial.Indices = row[1];
        //                    break;
        //                case "Capitalisation (milliers d'euros)":
        //                    //financial.MarketCap = int.Parse(row[1].Replace(" ", ""));
        //                    break;
        //                case "Rendement":
        //                    float yield = 0f;
        //                    if (float.TryParse(row[1].Replace(",", ".").Replace("%", ""), out yield))
        //                    {
        //                        financial.Yield = yield / 100f;
        //                    }
        //                    break;
        //                case "Dividende (Date de versement)":
        //                    financial.Dividend = row[1];
        //                    break;
        //                case "Date Assemblée Générale":
        //                    financial.MeetingDate = row[1];
        //                    break;
        //            }
        //        }
        //        financial.DownloadDate = DateTime.Now;
        //        stockSerie.StockAnalysis.Financial = financial;
        //    }
        //}
        public static void DownloadAgenda(StockSerie stockSerie)
        {
            string url = "http://www.abcbourse.com/marches/events.aspx?s=$ShortNamep".Replace("$ShortName",
               stockSerie.ShortName);

            StockWebHelper swh = new StockWebHelper();
            string html = swh.DownloadHtml(url);

            WebBrowser browser = new WebBrowser();
            browser.ScriptErrorsSuppressed = true;
            browser.DocumentText = html;
            browser.Document.OpenNew(true);
            browser.Document.Write(html);
            browser.Refresh();

            HtmlDocument doc = browser.Document;

            HtmlElementCollection tables = doc.GetElementsByTagName("table");
            List<List<string>> data = new List<List<string>>();

            foreach (HtmlElement tbl in tables)
            {
                if (tbl.InnerText.StartsWith("Date"))
                {
                    data = getTableData(tbl).Skip(1).ToList();
                    break;
                }
            }
            //
            foreach (var row in data)
            {
                if (row[0].StartsWith("du")) row[0] = row[0].Substring(row[0].IndexOf("au ") + 3);
                DateTime date = DateTime.Parse(row[0]);

                string comment = row[1];
                if (row[2] != null) comment += Environment.NewLine + row[2];

                if (!stockSerie.StockAnalysis.Comments.ContainsKey(date))
                {
                    stockSerie.StockAnalysis.Comments.Add(date, comment);
                }
                else
                {
                    if (!stockSerie.StockAnalysis.Comments[date].Contains(comment))
                    {
                        stockSerie.StockAnalysis.Comments[date] = stockSerie.StockAnalysis.Comments[date] +
                                                                  Environment.NewLine + comment;
                    }
                }
            }
        }