public CrawlerService(IPageScraper pageScraper, IStoredLinkRepository storedLinkRepository)
{
_maxDepth = 3;
_pageScraper = pageScraper;
_storedLinkRepository = storedLinkRepository;
RecordedThisSessionEntries = new List <string>();
}
public CrawlerService(IPageScraper pageScraper, IStoredLinkRepository storedLinkRepository)
{
_maxDepth = 3;
_pageScraper = pageScraper;
_storedLinkRepository = storedLinkRepository;
RecordedThisSessionEntries = new List<string>();
}
public async void ScrapDailyPage(string sportTypeUrl, DateTime day, int numberOfTries = 0)
{
IPageScraper siteReader = DependencyInjector.Retrieve <IPageScraper>();
Task <string> dayGamesPage = siteReader.GetHtmlAsync(sportTypeUrl
+ day.ToString("yyyy-MM-dd") + "/");
string dayGamesPageResult = await dayGamesPage;
if (!string.IsNullOrEmpty(dayGamesPageResult))
{
while (!ParseDataFromRawPageTxt(dayGamesPageResult, day) && numberOfTries < NUM_OF_TRIES)
{
numberOfTries++;
ParseDataFromRawPageTxt(dayGamesPageResult, day);
}
}
}
public CelebritiesReader(IPageScraper siteReader) : base(siteReader)
{
}
public GoogleSearchAnalyzer(IPageScraper scraper, ISearchResultsPageParser parser, IOptions <GoogleSearchAnalyzerOptions> settings)
{
this.scraper = scraper;
this.parser = parser;
this.settings = settings.Value;
}
public Reader(IPageScraper siteReader)
{
this.siteReader = siteReader;
}
