static void Main(string[] args)
{
var api = new QuandlApi();
var mw = new MarketWatchScraper();
using (var fileReader = new StreamReader(System.Configuration.ConfigurationManager.AppSettings["DownloadPath"] + "List.txt"))
{
while (!fileReader.EndOfStream)
{
string line = fileReader.ReadLine();
api.SaveCompanyCSV(line.Trim());
mw.GetTop100MarketWatchNews(line.Trim());
mw.ScrapeMarketWatch(line);
Console.WriteLine(line + " Complete");
Thread.Sleep(1000);
}
}
}
// Button to start scraping
private async void Button_Click_1(object sender, RoutedEventArgs e)
{
OutputText.Text = string.Empty;
int overallProgress = 0;
OverallProgress.Value = overallProgress;
// Collect Price Data from Quandl
var api = new QuandlApi();
var selectedList = new List<INewsScraper>();
foreach (var n in NewsSourceList.SelectedItems)
{
selectedList.Add((INewsScraper)n);
}
// Loop through each company to collect data.
foreach (var c in CompList)
{
WriteToOutput("Collecting Data for: " + c);
await Task.Run(() => api.SaveCompanyCSV(c));
int sourceProgress = 0;
SourceProgress.Value = sourceProgress;
// Loop through each each nes scraper for the company
foreach (INewsScraper s in selectedList)
{
await Task.Delay(1000);
await Task.Run(()=> s.Scrape(c.Trim()));
WriteToOutput("\t Collecting from: " + s.ToString());
sourceProgress++;
SourceProgress.Value = sourceProgress * 100/ selectedList.Count;
}
overallProgress++;
OverallProgress.Value = overallProgress * 100 / CompList.Count;
}
}