Ejemplo n.º 1
0
        private static void QuandlSequentialScrape()
        {
            var api = new QuandlApi();

            List<Company> companies = new List<Company>();

            var scraped = new List<string>();

            using (var db = new CapstoneContext())
            {
                companies = db.Companies.ToList();

                scraped = db.QStockPrices.GroupBy(p => p.CompanyId).Select(p => p.Key).ToList();
            }

            foreach (var company in companies)
            {

                using (var db = new CapstoneContext())
                {
                    scraped = db.QStockPrices.GroupBy(p => p.CompanyId).Select(p => p.Key).ToList();
                }

                if (!scraped.Contains(company.StockId))
                {
                    api.ScrapeAllPrice(company.StockId);
                }
                else
                {
                    Console.WriteLine(company.StockId + " FINISHED");
                }
            }
        }
Ejemplo n.º 2
0
        // Button to start scraping
        private async void Button_Click_1(object sender, RoutedEventArgs e)
        {
            OutputText.Text = string.Empty;

            int overallProgress = 0;

            OverallProgress.Value = overallProgress;
            // Collect Price Data from Quandl
            var api = new QuandlApi();

            var selectedList = new List<INewsScraper>();

            foreach (var n in NewsSourceList.SelectedItems)
            {
                selectedList.Add((INewsScraper)n);
            }
            // Loop through each company to collect data.
            foreach (var c in CompList)
            {
                WriteToOutput("Collecting Data for: " + c);

                await Task.Run(() => api.SaveCompanyCSV(c));

                int sourceProgress = 0;
                SourceProgress.Value = sourceProgress;
                // Loop through each each nes scraper for the company
                foreach (INewsScraper s in selectedList)
                {
                    await Task.Delay(1000);
                    await Task.Run(()=> s.Scrape(c.Trim()));
                    WriteToOutput("\t Collecting from: " + s.ToString());

                    sourceProgress++;
                    SourceProgress.Value = sourceProgress * 100/ selectedList.Count;
                }

                overallProgress++;

                OverallProgress.Value = overallProgress * 100 / CompList.Count;
            }
        }
Ejemplo n.º 3
0
        static void Main(string[] args)
        {
            var api = new QuandlApi();

            var mw = new MarketWatchScraper();

            using (var fileReader = new StreamReader(System.Configuration.ConfigurationManager.AppSettings["DownloadPath"] + "List.txt"))
            {
                while (!fileReader.EndOfStream)
                {
                    string line = fileReader.ReadLine();

                    api.SaveCompanyCSV(line.Trim());
                    mw.GetTop100MarketWatchNews(line.Trim());
                    mw.ScrapeMarketWatch(line);

                    Console.WriteLine(line + " Complete");
                    Thread.Sleep(1000);
                }
            }
        }
Ejemplo n.º 4
0
        private static void scrapOneCompany()
        {
            var api = new QuandlApi();

            List<Company> companies = new List<Company>();

            var scraped = new List<string>();

            using (var db = new CapstoneContext())
            {
                companies = db.Companies.ToList();

                scraped = db.QStockPrices.GroupBy(p => p.CompanyId).Select(p => p.Key).ToList();
            }

            var com = companies.First(c => !(scraped.Contains(c.StockId)));

            api.ScrapeAllPrice(com.StockId);

            Console.WriteLine("Finished");
            Console.ReadLine();
        }