private static void QuandlSequentialScrape()
{
var api = new QuandlApi();
List<Company> companies = new List<Company>();
var scraped = new List<string>();
using (var db = new CapstoneContext())
{
companies = db.Companies.ToList();
scraped = db.QStockPrices.GroupBy(p => p.CompanyId).Select(p => p.Key).ToList();
}
foreach (var company in companies)
{
using (var db = new CapstoneContext())
{
scraped = db.QStockPrices.GroupBy(p => p.CompanyId).Select(p => p.Key).ToList();
}
if (!scraped.Contains(company.StockId))
{
api.ScrapeAllPrice(company.StockId);
}
else
{
Console.WriteLine(company.StockId + " FINISHED");
}
}
}
// Button to start scraping
private async void Button_Click_1(object sender, RoutedEventArgs e)
{
OutputText.Text = string.Empty;
int overallProgress = 0;
OverallProgress.Value = overallProgress;
// Collect Price Data from Quandl
var api = new QuandlApi();
var selectedList = new List<INewsScraper>();
foreach (var n in NewsSourceList.SelectedItems)
{
selectedList.Add((INewsScraper)n);
}
// Loop through each company to collect data.
foreach (var c in CompList)
{
WriteToOutput("Collecting Data for: " + c);
await Task.Run(() => api.SaveCompanyCSV(c));
int sourceProgress = 0;
SourceProgress.Value = sourceProgress;
// Loop through each each nes scraper for the company
foreach (INewsScraper s in selectedList)
{
await Task.Delay(1000);
await Task.Run(()=> s.Scrape(c.Trim()));
WriteToOutput("\t Collecting from: " + s.ToString());
sourceProgress++;
SourceProgress.Value = sourceProgress * 100/ selectedList.Count;
}
overallProgress++;
OverallProgress.Value = overallProgress * 100 / CompList.Count;
}
}
static void Main(string[] args)
{
var api = new QuandlApi();
var mw = new MarketWatchScraper();
using (var fileReader = new StreamReader(System.Configuration.ConfigurationManager.AppSettings["DownloadPath"] + "List.txt"))
{
while (!fileReader.EndOfStream)
{
string line = fileReader.ReadLine();
api.SaveCompanyCSV(line.Trim());
mw.GetTop100MarketWatchNews(line.Trim());
mw.ScrapeMarketWatch(line);
Console.WriteLine(line + " Complete");
Thread.Sleep(1000);
}
}
}
private static void scrapOneCompany()
{
var api = new QuandlApi();
List<Company> companies = new List<Company>();
var scraped = new List<string>();
using (var db = new CapstoneContext())
{
companies = db.Companies.ToList();
scraped = db.QStockPrices.GroupBy(p => p.CompanyId).Select(p => p.Key).ToList();
}
var com = companies.First(c => !(scraped.Contains(c.StockId)));
api.ScrapeAllPrice(com.StockId);
Console.WriteLine("Finished");
Console.ReadLine();
}