public Scraper(SearchEngineDto searchEngine, int maxResultsToProcess)
{
Url = searchEngine.Url;
_pageNamingConvention = searchEngine.PageNamingConvention;
_pagesAvailable = searchEngine.PagesAvailable;
_tagContainingSearchResult = searchEngine.TagContainingSearchResult;
_maxResultsToProcess = maxResultsToProcess;
}
public async Task <int> AddSearchEngineToDbIfNotExistAsync(SearchEngineDto searchEngine)
{
SearchEngineDto searchEngineDto = _context.SearchEngines.FirstOrDefault(e => e.NameOfSearchEngine == searchEngine.NameOfSearchEngine);
if (searchEngineDto == null)
{
await _context.AddAsync(searchEngine);
await SaveChangesAsync();
return(searchEngine.Id);
}
else
{
return(searchEngineDto.Id);
}
}
/// <summary>
/// SearchFight
/// </summary>
/// <param name="keyWords"></param>
/// <returns></returns>
public async Task <SearchFightDto> SearchFight(string[] keyWords)
{
var result = new SearchFightDto();
var searchEngineDtoList = new List <SearchEngineDto>();
foreach (var keyWord in keyWords)
{
var engineDto = new SearchEngineDto();
#region GOOGLE
//for every search in google I put dto in result object
engineDto.GoogleDto = await _googleService.GetResultGoogle(keyWord);
#endregion
#region BING
//for every search in bing I put dto in result object
engineDto.BingDto = await _bingService.GetResultBing(keyWord);
#endregion
//set keyword in result object
engineDto.KeyWord = keyWord;
//add object to list
searchEngineDtoList.Add(engineDto);
}
result.EngineDtoList = searchEngineDtoList;
#region WINNER
//I find a winner for every search engine
result.GoogleWinnerDto = await _googleService.GetGoogleWinner(searchEngineDtoList);
result.BingWinnerDto = await _bingService.GetBingWinner(searchEngineDtoList);
//winner at all
result.Winner = await GetWinner(result);
#endregion
return(result);
}
public async Task <List <SearchResultViewModel> > SearchByPhraseInSearchEngine(string phraseForSearch, SearchEngine searchEngine)
{
List <SearchResultViewModel> searchResults = new List <SearchResultViewModel>();
using (var scope = _serviceProvider.CreateScope())
{
var googleCustomSearchClient = scope.ServiceProvider.GetRequiredService <ISearchClient <GoogleSearchResultModel> >();
var bingCustomSearchClient = scope.ServiceProvider.GetRequiredService <ISearchClient <BingCustomSearchResponseModel> >();
var yandexXmlSearchClient = scope.ServiceProvider.GetRequiredService <ISearchClient <YandexSearchResultModel> >();
var searchResultRepository = scope.ServiceProvider.GetRequiredService <ISearchResultRepository>();
var searchEngineRepository = scope.ServiceProvider.GetRequiredService <ISearchEngineRepository>();
if (searchEngine == SearchEngine.Google)
{
//Add search engine to database if not exist and get id of existed search engine
var googleDto = new SearchEngineDto {
NameOfSearchEngine = "Google"
};
var searchEngineId = await searchEngineRepository.AddSearchEngineToDbIfNotExistAsync(googleDto);
googleDto = await searchEngineRepository.GetSearchEngineByIdAsync(searchEngineId);
//Create request to Google custom search api and convert to ViewModel for returning in controller
var googleSearchResult = await googleCustomSearchClient.SearchByPhrase(phraseForSearch);
var googleSearchResultViewModel = ConvertGoogleResultToViewModel(googleSearchResult);
searchResults = googleSearchResultViewModel;
//Convert result to Dto and add the first result to database
var googleSearchResultDto = ConvertGoogleResultToDto(googleSearchResult, googleDto);
await searchResultRepository.AddResultToDbAsync(googleSearchResultDto[0]);
}
if (searchEngine == SearchEngine.Bing)
{
var bingDto = new SearchEngineDto {
NameOfSearchEngine = "Bing"
};
var searchEngineId = await searchEngineRepository.AddSearchEngineToDbIfNotExistAsync(bingDto);
bingDto = await searchEngineRepository.GetSearchEngineByIdAsync(searchEngineId);
var bingSearchResult = await bingCustomSearchClient.SearchByPhrase(phraseForSearch);
var bingSearchResultViewModel = ConvertBingResultToViewModel(bingSearchResult);
searchResults = bingSearchResultViewModel;
var bingSearchResultDto = ConvertBingResultToDto(bingSearchResult, bingDto);
await searchResultRepository.AddResultToDbAsync(bingSearchResultDto[0]);
}
if (searchEngine == SearchEngine.Yandex)
{
var yandexDto = new SearchEngineDto {
NameOfSearchEngine = "Yandex"
};
var searchEngineId = await searchEngineRepository.AddSearchEngineToDbIfNotExistAsync(yandexDto);
yandexDto = await searchEngineRepository.GetSearchEngineByIdAsync(searchEngineId);
YandexSearchResultModel yandexSearchResult = await yandexXmlSearchClient.SearchByPhrase(phraseForSearch);
var yandexSearchResultViewModel = ConvertYandexResultToViewModel(yandexSearchResult);
searchResults = yandexSearchResultViewModel;
var yandexSearchResultDto = ConvertYandexResultToDto(yandexSearchResult, yandexDto);
await searchResultRepository.AddResultToDbAsync(yandexSearchResultDto[0]);
}
}
return(searchResults);
}
private List <SearchResultDto> ConvertYandexResultToDto(YandexSearchResultModel yandexResult, SearchEngineDto searchEngine)
{
var searchResults = new List <SearchResultDto>();
try
{
foreach (var item in yandexResult.Response.Results.Grouping.Group)
{
searchResults.Add(new SearchResultDto
{
ResultName = item.Doc.Domain,
ResultUrl = item.Doc.Url,
SearchEngine = searchEngine
});
}
}
catch (NullReferenceException ex)
{
Console.WriteLine("There are some problems with converting Yandex search result to Dto - " + $"{ex.Message}");
}
return(searchResults);
}
private List <SearchResultDto> ConvertBingResultToDto(BingCustomSearchResponseModel bingResponse, SearchEngineDto searchEngine)
{
var dtoSearchResults = new List <SearchResultDto>();
try
{
foreach (var item in bingResponse.WebPages.Value)
{
dtoSearchResults.Add(new SearchResultDto
{
ResultName = item.Name,
ResultUrl = item.Url,
SearchEngine = searchEngine
});
}
}
catch (NullReferenceException ex)
{
Console.WriteLine("There are some problems with converting Bing search result to Dto - " + $"{ex.Message}");
}
return(dtoSearchResults);
}
private List <SearchResultDto> ConvertGoogleResultToDto(GoogleSearchResultModel googleResult, SearchEngineDto searchEngine)
{
var dtoSearchResults = new List <SearchResultDto>();
try
{
foreach (var item in googleResult.SearchItems)
{
dtoSearchResults.Add(new SearchResultDto
{
ResultName = item.Title,
ResultUrl = item.FormattedUrl,
SearchEngine = searchEngine
});
}
}
catch (NullReferenceException ex)
{
Console.WriteLine("There are some problems with converting Google search result to Dto - " + $"{ex.Message}");
}
return(dtoSearchResults);
}