Beispiel #1
0
        private async Task <IEnumerable <SearchResult> > SearchUsingSearchProvider(string searchTerm, SearchProvider searchProvider)
        {
            var searchResults = new List <SearchResult>();

            try
            {
                var retriever = _retrievers.SingleOrDefault(r => r.SearchProvider == searchProvider);
                if (retriever == null)
                {
                    throw new ArgumentException($"No retriever found for search provider: {searchProvider.ToString()}");
                }

                var scraper = _scrapers.SingleOrDefault(s => s.SearchProvider == searchProvider);
                if (scraper == null)
                {
                    throw new ArgumentException($"No scraper found for search provider: {searchProvider.ToString()}");
                }

                //retrieve data as html, start new search with this provider
                var htmlResult = await retriever.RetrieveResultsFromProvider(searchTerm);

                //scrape results from html
                var scrapedResults = await scraper.ScrapeResults(htmlResult);

                searchResults.AddRange(scrapedResults);

                //retrieve and scrape more pages until we have enough results
                while (searchResults.Count() < _config.NumberOfResults && scrapedResults.Count() > 0)
                {
                    htmlResult = await retriever.RetrieveResultsFromProviderNextPage();

                    scrapedResults = await scraper.ScrapeResults(htmlResult);

                    searchResults.AddRange(scrapedResults);
                }
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, $"Failed searching provider {searchProvider.ToString()}");
            }

            return(searchResults);
        }