private async Task <IEnumerable <SearchResult> > SearchUsingSearchProvider(string searchTerm, SearchProvider searchProvider)
{
var searchResults = new List <SearchResult>();
try
{
var retriever = _retrievers.SingleOrDefault(r => r.SearchProvider == searchProvider);
if (retriever == null)
{
throw new ArgumentException($"No retriever found for search provider: {searchProvider.ToString()}");
}
var scraper = _scrapers.SingleOrDefault(s => s.SearchProvider == searchProvider);
if (scraper == null)
{
throw new ArgumentException($"No scraper found for search provider: {searchProvider.ToString()}");
}
//retrieve data as html, start new search with this provider
var htmlResult = await retriever.RetrieveResultsFromProvider(searchTerm);
//scrape results from html
var scrapedResults = await scraper.ScrapeResults(htmlResult);
searchResults.AddRange(scrapedResults);
//retrieve and scrape more pages until we have enough results
while (searchResults.Count() < _config.NumberOfResults && scrapedResults.Count() > 0)
{
htmlResult = await retriever.RetrieveResultsFromProviderNextPage();
scrapedResults = await scraper.ScrapeResults(htmlResult);
searchResults.AddRange(scrapedResults);
}
}
catch (Exception ex)
{
_logger.LogError(ex, $"Failed searching provider {searchProvider.ToString()}");
}
return(searchResults);
}
