public Scraper(SearchEngineDto searchEngine, int maxResultsToProcess)
 {
     Url = searchEngine.Url;
     _pageNamingConvention      = searchEngine.PageNamingConvention;
     _pagesAvailable            = searchEngine.PagesAvailable;
     _tagContainingSearchResult = searchEngine.TagContainingSearchResult;
     _maxResultsToProcess       = maxResultsToProcess;
 }
        public async Task <int> AddSearchEngineToDbIfNotExistAsync(SearchEngineDto searchEngine)
        {
            SearchEngineDto searchEngineDto = _context.SearchEngines.FirstOrDefault(e => e.NameOfSearchEngine == searchEngine.NameOfSearchEngine);

            if (searchEngineDto == null)
            {
                await _context.AddAsync(searchEngine);
                await SaveChangesAsync();

                return(searchEngine.Id);
            }
            else
            {
                return(searchEngineDto.Id);
            }
        }
Exemple #3
0
        /// <summary>
        /// SearchFight
        /// </summary>
        /// <param name="keyWords"></param>
        /// <returns></returns>
        public async Task <SearchFightDto> SearchFight(string[] keyWords)
        {
            var result = new SearchFightDto();
            var searchEngineDtoList = new List <SearchEngineDto>();

            foreach (var keyWord in keyWords)
            {
                var engineDto = new SearchEngineDto();

                #region GOOGLE
                //for every search in google I put dto in result object
                engineDto.GoogleDto = await _googleService.GetResultGoogle(keyWord);

                #endregion

                #region BING
                //for every search in bing I put dto in result object
                engineDto.BingDto = await _bingService.GetResultBing(keyWord);

                #endregion

                //set keyword in result object
                engineDto.KeyWord = keyWord;

                //add object to list
                searchEngineDtoList.Add(engineDto);
            }
            result.EngineDtoList = searchEngineDtoList;

            #region WINNER
            //I find a winner for every search engine
            result.GoogleWinnerDto = await _googleService.GetGoogleWinner(searchEngineDtoList);

            result.BingWinnerDto = await _bingService.GetBingWinner(searchEngineDtoList);

            //winner at all
            result.Winner = await GetWinner(result);

            #endregion

            return(result);
        }
        public async Task <List <SearchResultViewModel> > SearchByPhraseInSearchEngine(string phraseForSearch, SearchEngine searchEngine)
        {
            List <SearchResultViewModel> searchResults = new List <SearchResultViewModel>();

            using (var scope = _serviceProvider.CreateScope())
            {
                var googleCustomSearchClient = scope.ServiceProvider.GetRequiredService <ISearchClient <GoogleSearchResultModel> >();
                var bingCustomSearchClient   = scope.ServiceProvider.GetRequiredService <ISearchClient <BingCustomSearchResponseModel> >();
                var yandexXmlSearchClient    = scope.ServiceProvider.GetRequiredService <ISearchClient <YandexSearchResultModel> >();
                var searchResultRepository   = scope.ServiceProvider.GetRequiredService <ISearchResultRepository>();
                var searchEngineRepository   = scope.ServiceProvider.GetRequiredService <ISearchEngineRepository>();

                if (searchEngine == SearchEngine.Google)
                {
                    //Add search engine to database if not exist and get id of existed search engine
                    var googleDto = new SearchEngineDto {
                        NameOfSearchEngine = "Google"
                    };
                    var searchEngineId = await searchEngineRepository.AddSearchEngineToDbIfNotExistAsync(googleDto);

                    googleDto = await searchEngineRepository.GetSearchEngineByIdAsync(searchEngineId);

                    //Create request to Google custom search api and convert to ViewModel for returning in controller
                    var googleSearchResult = await googleCustomSearchClient.SearchByPhrase(phraseForSearch);

                    var googleSearchResultViewModel = ConvertGoogleResultToViewModel(googleSearchResult);
                    searchResults = googleSearchResultViewModel;

                    //Convert result to Dto and add the first result to database
                    var googleSearchResultDto = ConvertGoogleResultToDto(googleSearchResult, googleDto);
                    await searchResultRepository.AddResultToDbAsync(googleSearchResultDto[0]);
                }

                if (searchEngine == SearchEngine.Bing)
                {
                    var bingDto = new SearchEngineDto {
                        NameOfSearchEngine = "Bing"
                    };
                    var searchEngineId = await searchEngineRepository.AddSearchEngineToDbIfNotExistAsync(bingDto);

                    bingDto = await searchEngineRepository.GetSearchEngineByIdAsync(searchEngineId);

                    var bingSearchResult = await bingCustomSearchClient.SearchByPhrase(phraseForSearch);

                    var bingSearchResultViewModel = ConvertBingResultToViewModel(bingSearchResult);
                    searchResults = bingSearchResultViewModel;

                    var bingSearchResultDto = ConvertBingResultToDto(bingSearchResult, bingDto);
                    await searchResultRepository.AddResultToDbAsync(bingSearchResultDto[0]);
                }

                if (searchEngine == SearchEngine.Yandex)
                {
                    var yandexDto = new SearchEngineDto {
                        NameOfSearchEngine = "Yandex"
                    };
                    var searchEngineId = await searchEngineRepository.AddSearchEngineToDbIfNotExistAsync(yandexDto);

                    yandexDto = await searchEngineRepository.GetSearchEngineByIdAsync(searchEngineId);

                    YandexSearchResultModel yandexSearchResult = await yandexXmlSearchClient.SearchByPhrase(phraseForSearch);

                    var yandexSearchResultViewModel = ConvertYandexResultToViewModel(yandexSearchResult);
                    searchResults = yandexSearchResultViewModel;

                    var yandexSearchResultDto = ConvertYandexResultToDto(yandexSearchResult, yandexDto);
                    await searchResultRepository.AddResultToDbAsync(yandexSearchResultDto[0]);
                }
            }
            return(searchResults);
        }
        private List <SearchResultDto> ConvertYandexResultToDto(YandexSearchResultModel yandexResult, SearchEngineDto searchEngine)
        {
            var searchResults = new List <SearchResultDto>();

            try
            {
                foreach (var item in yandexResult.Response.Results.Grouping.Group)
                {
                    searchResults.Add(new SearchResultDto
                    {
                        ResultName   = item.Doc.Domain,
                        ResultUrl    = item.Doc.Url,
                        SearchEngine = searchEngine
                    });
                }
            }
            catch (NullReferenceException ex)
            {
                Console.WriteLine("There are some problems with converting Yandex search result to Dto - " + $"{ex.Message}");
            }

            return(searchResults);
        }
        private List <SearchResultDto> ConvertBingResultToDto(BingCustomSearchResponseModel bingResponse, SearchEngineDto searchEngine)
        {
            var dtoSearchResults = new List <SearchResultDto>();

            try
            {
                foreach (var item in bingResponse.WebPages.Value)
                {
                    dtoSearchResults.Add(new SearchResultDto
                    {
                        ResultName   = item.Name,
                        ResultUrl    = item.Url,
                        SearchEngine = searchEngine
                    });
                }
            }
            catch (NullReferenceException ex)
            {
                Console.WriteLine("There are some problems with converting Bing search result to Dto - " + $"{ex.Message}");
            }

            return(dtoSearchResults);
        }
        private List <SearchResultDto> ConvertGoogleResultToDto(GoogleSearchResultModel googleResult, SearchEngineDto searchEngine)
        {
            var dtoSearchResults = new List <SearchResultDto>();

            try
            {
                foreach (var item in googleResult.SearchItems)
                {
                    dtoSearchResults.Add(new SearchResultDto
                    {
                        ResultName   = item.Title,
                        ResultUrl    = item.FormattedUrl,
                        SearchEngine = searchEngine
                    });
                }
            }
            catch (NullReferenceException ex)
            {
                Console.WriteLine("There are some problems with converting Google search result to Dto - " + $"{ex.Message}");
            }

            return(dtoSearchResults);
        }