public void Run()
{
using (App.databaseConnection = new SqliteConnection("" + new SqliteConnectionStringBuilder {
DataSource = "hello.db"
}))
{
App.databaseConnection.Open();
DataRepairRepository.RepairUrlStackFromEdges().Wait();
var allHtml = DataRepairRepository.GetAllHtmlForNodes();
foreach (var html in allHtml)
{
var htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(html);
var links = WebService.GetLinksFromHtmlDocument(htmlDoc);
var newLinks = GetLinksThatDontAppearInDB(links);
UrlStackRepository.AddUrlsToStack(newLinks);
}
}
}
private IEnumerable <string> GetLinksThatDontAppearInDB(IEnumerable <string> links)
{
var urlsThatAppear = DataRepairRepository.FilterUrlsThatAppearInDatabase(links).ToDictionary(x => x);
return(links.Where(x => !urlsThatAppear.ContainsKey(x)));
}
