//Scrapes properties without garden
public async Task ScrapeObjects(int page)
{
try
{
while (!isObjectSearchCompleted)
{
UriBuilder builder = new UriBuilder(_apiSettings.Value.FundaUrl)
{
Query = $"?type=koop&zo=/amsterdam/&page={page}&pagesize=25"
};
var fundaResult = await _apiCaller.GetResponseAsync <FundaResponseModel>(builder.Uri).ConfigureAwait(false);
await Task.Delay(1300);//to be on the safe side I wait a little bit longer than needed
if (fundaResult != null)
{
foreach (var obj in fundaResult.Objects)
{
UpdateDictionary(makelaarsWithPropertiesAmount, obj.MakelaarNaam);
}
}
if (!fundaResult.Objects.Any())
{
isObjectSearchCompleted = true;
}
else
{
page++;
}
}
}
catch (HttpRequestException ex)
{
Log.Error(ex, ex.Message);
//we could have other kinds of http exception
if (ex.Message == "Response status code does not indicate success: 401 (Request limit exceeded).")
{
Thread.Sleep(TimeSpan.FromSeconds(60));
await ScrapeObjects(page);
}
}
catch (Exception ex)
{
Log.Fatal(ex, ex.Message);
}
}
