private async Task <string> GetRawPage(ImmoscoutWebClientOptions options, int pageNr) { var response = await _client.GetAsync($"https://www.immobilienscout24.de/Suche/radius/{options.Type}?centerofsearchaddress={options.Location};;;;&geocoordinates={options.Latitude};{options.Longitude};{options.Radius}&pagenumber={pageNr}"); if (response.IsSuccessStatusCode) { var rawPage = await response.Content.ReadAsStringAsync(); return(rawPage.Contains("Mensch aus Fleisch und Blut") ? InvalidPage : rawPage); } return(InvalidPage); }
public async Task <IEnumerable <Property> > GetProperties() { var settingsContainer = await _settingsReader.ReadSettings("providers/Immobilienscout24.yml"); var optionToUse = GetOptionsToUse(settingsContainer.Settings.ToList()); var result = await _webClient.GetObjects(optionToUse.options, optionToUse.page); _optionsToLastPageDictionary[optionToUse.options] = result.NextPageNumber; _lastUsedOptions = optionToUse.options; _logger.LogInfo($"Found {result.ImmoscoutProperties.Count()} properties"); return(_immoscoutConverter.ToProperties(result.ImmoscoutProperties)); }
public async Task <WebClientResult> GetObjects(ImmoscoutWebClientOptions options, int pageNumber) { _logger.LogDebug($"Retrieving properties for page '{pageNumber}' with the following options: {options}"); var rawPage = await GetRawPage(options, pageNumber); if (rawPage != InvalidPage) { var pageCount = GetPageCount(rawPage); var properties = ParseHtml(rawPage).ToList(); var nextPageNumber = pageCount == pageNumber ? 1 : ++pageNumber; _logger.LogInfo($"Found {properties.Count} properties"); return(new WebClientResult(nextPageNumber, properties)); } _logger.LogWarning($"Retrieving properties was not successful. Received an invalid page."); return(new WebClientResult(1, Enumerable.Empty <ImmoscoutProperty>())); }