Esempio n. 1
0
        private void Filter(_InternalData internalData)
        {
            string prefix = "4/5 Filtering...";

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 0), ConsoleColor.Gray);

            try
            {
                // Filter only if we have not parsed
                if (Reload || !File.Exists(SerializedFile))
                {
                    // Order by history length
                    internalData.Products = internalData.Products.OrderBy(x => x.StartDate.Ticks).ToList();

                    Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 50), ConsoleColor.Green);

                    // Take only N best
                    internalData.Products = internalData.Products.Take(DataManager.Config.BestNCompanies).ToList();
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine();
                Utils.DrawMessage("", ex.Message, ConsoleColor.Red);
                Console.WriteLine();

                System.Environment.Exit(1);
            }

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 100), ConsoleColor.Green);
            Console.WriteLine();
        }
Esempio n. 2
0
        public override async Task Prepare(DataContainer dataContainer)
        {
            // Delete old data
            if (Reload)
            {
                Directory.Delete(Folder, true);
            }

            // Create folder
            Directory.CreateDirectory(Folder);

            // Inernal data
            _InternalData internalData = new _InternalData();

            // Draw header
            base.DrawHeader();

            // Download
            await Download();

            // Parse
            Parse(internalData);

            // Find starting dates
            await FindStartDate(internalData);

            // Filter
            Filter(internalData);

            // SeDeSerialize
            Cache(dataContainer, internalData);

            // End info
            Console.WriteLine();
        }
Esempio n. 3
0
        public _Cache(_InternalData internalData)
        {
            Symbols = new List <string>();
            Names   = new List <string>();

            foreach (_Product p in internalData.Products)
            {
                Symbols.Add(p.Symbol);
                Names.Add(p.Name);
            }
        }
Esempio n. 4
0
        private void Cache(DataContainer dataContainer, _InternalData internalData)
        {
            string prefix = "5/5 Caching...";

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 0), ConsoleColor.Gray);

            try
            {
                // Serialize or deserialize
                _Cache cache = null;
                if (Reload || !File.Exists(SerializedFile))
                {
                    cache = new _Cache(internalData);

                    Serializer.Serialize(SerializedFile, cache);
                }
                else
                {
                    cache = (_Cache)Serializer.Deserialize(SerializedFile);
                }

                Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 50), ConsoleColor.Gray);

                // Apply data do global data
                for (int i = 0; i < cache.Symbols.Count; ++i)
                {
                    Product p = new Product()
                    {
                        Symbol = cache.Symbols[i],
                        Name   = cache.Names[i]
                    };

                    dataContainer.Products.Add(p);
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine();
                Utils.DrawMessage("", ex.Message, ConsoleColor.Red);
                Console.WriteLine();

                System.Environment.Exit(1);
            }

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 100), ConsoleColor.Green);
            Console.WriteLine();
        }
Esempio n. 5
0
        private void Parse(_InternalData internalData)
        {
            string prefix = "2/5 Parsing...";

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 0), ConsoleColor.Gray);

            try
            {
                // Parse only if we have not parsed
                if (Reload || !File.Exists(SerializedFile))
                {
                    // Load data
                    string[] lines = File.ReadAllLines(Path.Combine(Folder, "Products.csv"));

                    // Count when we update progress bar
                    int drawEvery = Utils.PercentIntervalByLength(lines.Length);

                    // Check header
                    if (lines[0] != "\"Symbol\",\"Name\",\"LastSale\",\"MarketCap\",\"IPOyear\",\"Sector\",\"industry\",\"Summary Quote\",")
                    {
                        throw new Exception("Unknown file header");
                    }

                    // Parse product names
                    for (int i = 1; i < lines.Length; ++i)
                    {
                        // Get columns
                        string[] columns = lines[i].Split("\",\"");

                        // Extract symbol
                        string tickerSymbol = columns[0];
                        tickerSymbol = tickerSymbol.Substring(1).Trim();

                        // Extract name
                        string name = columns[1];

                        // Extract market capitalization
                        string mcStr = columns[3];
                        double mc    = -1;
                        if (mcStr != "n/a")
                        {
                            if (mcStr[0] != '$')
                            {
                                throw new Exception("Market capitalization - missin $ symbol (" + i + "," + lines[i].Split(new char[] { ',' })[3] + ")");
                            }
                            mcStr = mcStr.Substring(1, mcStr.Length - 1);
                            if (mcStr.Last() != 'B' && mcStr.Last() != 'M' && !Char.IsDigit(mcStr.Last()))
                            {
                                throw new Exception("Market capitalization - not millions, not billions and not normal number (" + i + "," + lines[i].Split(new char[] { ',' })[3] + ")");
                            }

                            int multiple = 1;
                            if (mcStr.Last() == 'B')
                            {
                                multiple = 1000000000;
                                mcStr    = mcStr.Substring(0, mcStr.Length - 1);
                            }
                            else if (mcStr.Last() == 'M')
                            {
                                multiple = 1000000;
                                mcStr    = mcStr.Substring(0, mcStr.Length - 1);
                            }

                            mc  = double.Parse(mcStr, CultureInfo.InvariantCulture);
                            mc *= multiple;
                        }

                        // Only valid products
                        if ((new Regex("^[a-zA-Z0-9]*$")).IsMatch(tickerSymbol))
                        {
                            // Create new products
                            _Product p = new _Product();
                            p.Symbol = tickerSymbol;
                            p.Name   = name;
                            p.MarketCapitalizationInMillions = mc;

                            // Add to list
                            internalData.Products.Add(p);
                        }

                        // Update progress bar
                        if (i % drawEvery == 0)
                        {
                            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, (double)i / lines.Length * 100.0), ConsoleColor.Gray);
                        }
                    }

                    // Check some properties
                    if (internalData.Products.Count < 1000)
                    {
                        throw new Exception("There are just " + internalData.Products.Count + " products");
                    }
                    int yy = internalData.Products.Select(x => x.Symbol).Where(x => x == "IBM" || x == "JPM" || x == "SNHN").Count();
                    if (internalData.Products.Select(x => x.Symbol).Where(x => x == "IBM" || x == "JPM" || x == "SNHN").Count() != 3)
                    {
                        throw new Exception("Some products are missing");
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine();
                Utils.DrawMessage("", ex.Message, ConsoleColor.Red);
                Console.WriteLine();

                System.Environment.Exit(1);
            }

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 100), ConsoleColor.Green);
            Console.WriteLine();
        }
Esempio n. 6
0
        private async Task FindStartDate(_InternalData internalData)
        {
            string prefix = "3/5 Finding first date...";

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 0), ConsoleColor.Gray);

            // Count when we update progress bar
            int drawEvery = Utils.PercentIntervalByLength(internalData.Products.Count);

            // Basic url
            string url = "http://real-chart.finance.yahoo.com/table.csv?s=___&d=0&e=0&f=2100&g=d&a=0&b=0&c=1900&ignore=.csv";

            // Destination folder
            string targetFolder = Path.Combine(Folder, "ProductsHistory");

            // Create folder for product
            Directory.CreateDirectory(targetFolder);

            // Download
            try
            {
                for (int i = 0; i < internalData.Products.Count; ++i)
                {
                    // Product
                    _Product p = internalData.Products[i];

                    // Prepare url
                    string productUrl = url.Replace("___", p.Symbol);

                    // Target file
                    string targetFile = Path.Combine(targetFolder, p.Symbol + ".csv");

                    // Download if needed
                    if (Reload || !File.Exists(targetFile))
                    {
                        try
                        {
                            await Downloader.DownloadFileAsync(productUrl, targetFile, null, null);
                        }
                        catch (Exception ex)
                        {
                            if (ex.Message == "The remote server returned an error: (404) Not Found.")
                            {
                                internalData.Products.RemoveAt(i);
                                --i;
                                if (File.Exists(targetFile))
                                {
                                    File.Delete(targetFile);
                                }
                            }
                            else
                            {
                                throw ex;
                            }
                        }
                    }

                    if (File.Exists(targetFile))
                    {
                        // Parse only if we have not parsed
                        if (Reload || !File.Exists(SerializedFile))
                        {
                            // Store start date
                            string[] lines = File.ReadAllLines(targetFile);
                            string   date  = lines.Last().Split(new char[] { ',' })[0];

                            p.StartDate = DateTime.ParseExact(date, "yyyy-MM-dd", CultureInfo.InvariantCulture);
                        }
                    }

                    // Update progress bar
                    if (i % drawEvery == 0)
                    {
                        Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, (double)i / internalData.Products.Count * 100.0), ConsoleColor.Gray);
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine();
                Utils.DrawMessage("", ex.Message, ConsoleColor.Red);
                Console.WriteLine();

                System.Environment.Exit(1);
            }

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 100), ConsoleColor.Green);
            Console.WriteLine();
        }