public void Run()
{
var startPage = _db.Uri.FirstOrDefault(o => o.AbsoluteUri == _startPageUri.AbsoluteUri);
if (startPage == null)
{
Console.WriteLine("Adding START_PAGE to db...");
startPage = NewUriDbModel(_startPageUri);
_db.Uri.Add(startPage);
_db.SaveChanges();
}
Task.Run(PlannerStatic);
for (int i = 0; i < _crawlerThreadCount; i++)
{
Task.Run(CrawlerStatic);
}
Task.Run(Storer);
Task.Run(PlannerBrowser);
for (int i = 0; i < _browserCrawlerThreadCount; i++)
{
Task.Run(CrawlerBrowserWebDriver);
}
//Task.Run(Indexer);
while (true)
{
//Console.WriteLine("----------------------------Start Crawling--------------------------");
//CrawlPages(_db, startPageHost);
//Console.WriteLine("---------------------------- End Crawling--------------------------");
//Console.WriteLine();
//Console.WriteLine();
//Console.WriteLine();
//Console.WriteLine();
Console.WriteLine($"Pending Crawl: {_queuePlan.Count}\tPending Browser: {_queueBrowserPlan.Count}\tPending Store: {_queueCrawlResult.Count}");
Thread.Sleep(TimeSpan.FromSeconds(1));
}
}
