Exemple #1
0
        private async Task DownloadDatasets(_InternalData internalData)
        {
            string prefix = "7/11 Downloading DS...";

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 0), ConsoleColor.Gray);

            int drawEvery = Utils.PercentIntervalByLength(internalData.NameToDataUrl.Count);

            Directory.CreateDirectory(Path.Combine(Folder, "Datasets"));

            try
            {
                // Download pages with complete lists
                int i = 0;
                foreach (KeyValuePair <string, List <string> > kv in internalData.NameToDataUrl)
                {
                    string name = kv.Key;

                    // Create directory
                    Directory.CreateDirectory(Path.Combine(Folder, "Datasets", name));

                    foreach (string access in kv.Value)
                    {
                        // Check if dataset exists
                        string searchFile = Path.Combine(Folder, "Searches", name, Path.GetFileName(access) + ".csv");
                        if (File.ReadAllLines(searchFile).Skip(1).Select(x => x.Split(new char[] { ',' })[1].Trim()).Where(x => x == Path.GetFileName(access)).Count() == 0)
                        {
                            continue;
                        }

                        string url        = "https://www.quandl.com/api/v3/datasets" + access + ".csv?api_key=" + DataManager.Config.QuandlApiKey;
                        string targetFile = Path.Combine(Folder, "Datasets", name, Path.GetFileName(access) + ".csv");
                        if (Reload || !File.Exists(targetFile))
                        {
                            await Downloader.DownloadFileAsync(url, targetFile, null, null);
                        }
                    }

                    // Update progress bar
                    if (i++ % drawEvery == 0)
                    {
                        Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, (double)i / internalData.NameToInfoUrl.Count * 100.0), ConsoleColor.Gray);
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine();
                Utils.DrawMessage("", ex.Message, ConsoleColor.Red);
                Console.WriteLine();

                System.Environment.Exit(1);
            }

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 100), ConsoleColor.Green);
            Console.WriteLine();
        }
Exemple #2
0
        private void Cache(DataContainer dataContainer, _InternalData internalData)
        {
            string prefix = "11/11 Caching...";

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 0), ConsoleColor.Gray);

            try
            {
                // Serialize or deserialize
                _Cache cache = null;
                if (Reload || !File.Exists(SerializedFile))
                {
                    cache = new _Cache(internalData);

                    Serializer.Serialize(SerializedFile, cache);
                }
                else
                {
                    cache = (_Cache)Serializer.Deserialize(SerializedFile);
                }

                Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 50), ConsoleColor.Gray);

                int drawEvery = Utils.PercentIntervalByLength(cache.Name.Count * 2);

                // Save to datacontainer
                for (int i = 0; i < cache.Name.Count; ++i)
                {
                    // Create forex data
                    _FuturesData fd = new _FuturesData()
                    {
                        Price = cache.Value[i]
                    };

                    // Add to data container
                    dataContainer.GetEvent(new DateTime(cache.Date[i])).FuturesDatas.Add(cache.Name[i], fd);

                    if (i % drawEvery == 0)
                    {
                        Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 50 + (double)i / cache.Name.Count * 50), ConsoleColor.Gray);
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine();
                Utils.DrawMessage("", ex.Message, ConsoleColor.Red);
                Console.WriteLine();

                System.Environment.Exit(1);
            }

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 100), ConsoleColor.Green);
            Console.WriteLine();
        }
Exemple #3
0
        private void Clean(_InternalData internalData)
        {
            string prefix = "10/11 Cleaning...";

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 0), ConsoleColor.Gray);

            // Get names
            string[] names = internalData.Events.Select(x => x.Name).Distinct().ToArray();

            int drawEvery = Utils.PercentIntervalByLength(names.Length);

            // Get max data length
            int maxDataLength = 0;

            foreach (string n in names)
            {
                maxDataLength = Math.Max(maxDataLength, internalData.Events.Where(x => x.Name == n).Count());
            }

            // Download
            try
            {
                for (int i = 0; i < names.Length; ++i)
                {
                    // Get events of futures
                    List <_Event> futuresEvents = internalData.Events.Where(x => x.Name == names[i]).OrderBy(x => x.Date).ToList();

                    // Remove futures if not many data
                    if (futuresEvents.Count < 0.5 * maxDataLength)
                    {
                        internalData.Events.RemoveAll(x => x.Name == names[i]);
                    }

                    // Update progress bar
                    if (i % drawEvery == 0)
                    {
                        Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, (double)i / names.Length * 100.0), ConsoleColor.Gray);
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine();
                Utils.DrawMessage("", ex.Message, ConsoleColor.Red);
                Console.WriteLine();

                System.Environment.Exit(1);
            }


            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 100), ConsoleColor.Green);
            Console.WriteLine();
        }
Exemple #4
0
        public override async Task Prepare(DataContainer dataContainer)
        {
            // Delete old data
            if (Reload)
            {
                Directory.Delete(Folder, true);
            }

            // Create folder
            Directory.CreateDirectory(Folder);

            // Inernal data
            _InternalData internalData = new _InternalData();

            // Draw header
            base.DrawHeader();

            // Download main futures page
            await DownloadPage();

            // Download futures list
            await DownloadLists();

            // Parse
            ParseNames(internalData);

            // Download info pages
            await DownloadInfo(internalData);

            // Extract datasets address
            ParseDatasetsUrls(internalData);

            // Download searches
            await DownloadSearches(internalData);

            // Download datasets
            await DownloadDatasets(internalData);

            // Choose datasets
            ChooseBestDataset();

            // Parse datasets
            ParseDatasets(internalData);

            // Clean data
            Clean(internalData);

            // SeDeSerialize
            Cache(dataContainer, internalData);

            // End info
            Console.WriteLine();
        }
Exemple #5
0
        private void ParseNames(_InternalData internalData)
        {
            string prefix = "3/11 Parsing names...";

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 0), ConsoleColor.Gray);

            // Files to parse
            string[] files = Directory.GetFiles(Path.Combine(Folder, "Lists")).ToArray();

            // Count when we update progress bar
            int drawEvery = Utils.PercentIntervalByLength(files.Length);

            // Download
            try
            {
                if (Reload || !File.Exists(SerializedFile))
                {
                    for (int i = 0; i < files.Length; ++i)
                    {
                        // Load file
                        string content = File.ReadAllText(files[i]);

                        // Parse name and url
                        CQ dom = new CQ(content);
                        List <IDomObject> rows = dom["#content > noscript > table.sheet-table > tbody > tr td:first-child a"].ToList();
                        foreach (IDomObject r in rows)
                        {
                            string name = r.TextContent;
                            string link = r.Attributes["href"];

                            internalData.NameToInfoUrl[name] = link;
                        }

                        // Update progress bar
                        if (i % drawEvery == 0)
                        {
                            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, (double)i / files.Length * 100.0), ConsoleColor.Gray);
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine();
                Utils.DrawMessage("", ex.Message, ConsoleColor.Red);
                Console.WriteLine();

                System.Environment.Exit(1);
            }

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 100), ConsoleColor.Green);
            Console.WriteLine();
        }
Exemple #6
0
        public _Cache(_InternalData internalData)
        {
            Name  = new List <string>();
            Date  = new List <long>();
            Value = new List <double>();

            for (int i = 0; i < internalData.Events.Count; ++i)
            {
                Name.Add(internalData.Events[i].Name);
                Date.Add(internalData.Events[i].Date.Ticks);
                Value.Add(internalData.Events[i].Value);
            }
        }
Exemple #7
0
        private async Task DownloadSearches(_InternalData internalData)
        {
            string prefix = "6/11 Downloading search...";

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 0), ConsoleColor.Gray);

            int drawEvery = Utils.PercentIntervalByLength(internalData.NameToDataUrl.Count);

            Directory.CreateDirectory(Path.Combine(Folder, "Searches"));

            try
            {
                // Download pages with complete lists
                int i = 0;
                foreach (KeyValuePair <string, List <string> > kv in internalData.NameToDataUrl)
                {
                    string name = kv.Key;

                    // Create directory
                    Directory.CreateDirectory(Path.Combine(Folder, "Searches", name));

                    foreach (string access in kv.Value)
                    {
                        // Check url
                        string url        = "https://www.quandl.com/api/v3/datasets.csv?database_code=CHRIS&query=" + WebUtility.UrlEncode(Path.GetFileName(access)) + "&api_key=" + DataManager.Config.QuandlApiKey;
                        string targetFile = Path.Combine(Folder, "Searches", name, Path.GetFileName(access) + ".csv");
                        if (Reload || !File.Exists(targetFile))
                        {
                            await Downloader.DownloadFileAsync(url, targetFile, null, null);
                        }
                    }

                    // Update progress bar
                    if (i++ % drawEvery == 0)
                    {
                        Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, (double)i / internalData.NameToInfoUrl.Count * 100.0), ConsoleColor.Gray);
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine();
                Utils.DrawMessage("", ex.Message, ConsoleColor.Red);
                Console.WriteLine();

                System.Environment.Exit(1);
            }

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 100), ConsoleColor.Green);
            Console.WriteLine();
        }
Exemple #8
0
        private void ParseDatasetsUrls(_InternalData internalData)
        {
            string prefix = "5/11 Parsing DS urls...";

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 0), ConsoleColor.Gray);

            // Files to parse
            string[] files = Directory.GetFiles(Path.Combine(Folder, "Info")).ToArray();

            // Count when we update progress bar
            int drawEvery = Utils.PercentIntervalByLength(files.Length);

            // Download
            try
            {
                if (Reload || !File.Exists(SerializedFile))
                {
                    for (int i = 0; i < files.Length; ++i)
                    {
                        // Load file
                        string content = File.ReadAllText(files[i]);

                        // Parse name and url
                        CQ            dom  = new CQ(content);
                        List <string> urls = dom["#content > noscript a"].Select(x => x.Attributes["href"]).Where(x => x.StartsWith("/CHRIS/")).Distinct().ToList();

                        // Save
                        internalData.NameToDataUrl.Add(Path.GetFileNameWithoutExtension(files[i]), urls);

                        // Update progress bar
                        if (i % drawEvery == 0)
                        {
                            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, (double)i / files.Length * 100.0), ConsoleColor.Gray);
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine();
                Utils.DrawMessage("", ex.Message, ConsoleColor.Red);
                Console.WriteLine();

                System.Environment.Exit(1);
            }

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 100), ConsoleColor.Green);
            Console.WriteLine();
        }
Exemple #9
0
        private async Task DownloadInfo(_InternalData internalData)
        {
            string prefix = "4/11 Downloading info...";

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 0), ConsoleColor.Gray);

            int drawEvery = Utils.PercentIntervalByLength(internalData.NameToInfoUrl.Count);

            Directory.CreateDirectory(Path.Combine(Folder, "Info"));

            try
            {
                // Download pages with complete lists
                int i = 0;
                foreach (KeyValuePair <string, string> kv in internalData.NameToInfoUrl)
                {
                    string name = kv.Key;

                    string url        = "https://www.quandl.com" + kv.Value;
                    string targetFile = Path.Combine(Folder, "Info", name.Replace("/", " ").Replace("\\", " ") + ".html");
                    if (Reload || !File.Exists(targetFile))
                    {
                        await Downloader.DownloadFileAsync(url, targetFile, null, null);
                    }

                    // Update progress bar
                    if (i++ % drawEvery == 0)
                    {
                        Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, (double)i / internalData.NameToInfoUrl.Count * 100.0), ConsoleColor.Gray);
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine();
                Utils.DrawMessage("", ex.Message, ConsoleColor.Red);
                Console.WriteLine();

                System.Environment.Exit(1);
            }

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 100), ConsoleColor.Green);
            Console.WriteLine();
        }
Exemple #10
0
        private void ParseDatasets(_InternalData internalData)
        {
            string prefix = "9/11 Parsing DS...";

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 0), ConsoleColor.Gray);

            // All files to parse
            string[] files = Directory.GetFiles(Path.Combine(Folder, "Final datasets")).ToArray();

            // Count when we update progress bar
            int drawEvery = Utils.PercentIntervalByLength(files.Length);

            // Download
            try
            {
                if (Reload || !File.Exists(SerializedFile))
                {
                    for (int i = 0; i < files.Length; ++i)
                    {
                        // Info
                        string code = Path.GetFileNameWithoutExtension(files[i]);
                        string name = internalData.NameToDataUrl.Where(x => x.Value.Where(y => y.Contains(code)).Count() > 0).FirstOrDefault().Key;

                        // Load file
                        string[] lines = File.ReadAllLines(files[i]);

                        // Ignore files with not many data
                        if (lines.Length < 500)
                        {
                            continue;
                        }

                        // Find column with price
                        string[] headers = lines[0].Split(new char[] { ',' }).Select(x => x.ToLower()).ToArray();
                        int      index   = -1;
                        if (headers.Where(x => x.Contains("close")).Count() > 0)
                        {
                            index = headers.Select((v, ind) => new { column = v, index = ind }).First(x => x.column.Contains("close")).index;
                        }
                        else if (headers.Where(x => x.Contains("last")).Count() > 0)
                        {
                            index = headers.Select((v, ind) => new { column = v, index = ind }).First(x => x.column.Contains("last")).index;
                        }
                        else if (headers.Where(x => x.Contains("settle")).Count() > 0)
                        {
                            index = headers.Select((v, ind) => new { column = v, index = ind }).First(x => x.column.Contains("settle")).index;
                        }
                        else
                        {
                            continue;
                        }

                        // Parse data
                        for (int k = 1; k < lines.Length; ++k)
                        {
                            string[] parts = lines[k].Split(new char[] { ',' });

                            // Date
                            string[] dateParts = parts[0].Split(new char[] { '-' });
                            int      year      = int.Parse(dateParts[0]);
                            int      month     = int.Parse(dateParts[1]);
                            int      day       = int.Parse(dateParts[2]);
                            DateTime date      = new DateTime(year, month, day);

                            // Value
                            if (String.IsNullOrWhiteSpace(parts[index]))
                            {
                                continue;
                            }
                            double value = double.Parse(parts[index], CultureInfo.InvariantCulture);

                            // Create event
                            internalData.Events.Add(new _Event()
                            {
                                Name  = name,
                                Date  = date,
                                Value = value
                            });
                        }

                        // Update progress bar
                        if (i % drawEvery == 0)
                        {
                            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, (double)i / files.Length * 100.0), ConsoleColor.Gray);
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine();
                Utils.DrawMessage("", ex.Message, ConsoleColor.Red);
                Console.WriteLine();

                System.Environment.Exit(1);
            }

            Utils.DrawMessage(prefix, Utils.CreateProgressBar(Utils.ProgressBarLength, 100), ConsoleColor.Green);
            Console.WriteLine();
        }