public virtual async Task CollectData(City city)
{
CurrentCity = city;
await _repository.ClearOffers(CurrentCity, TargetWebsite.Name);
for (int i = 0; i < await GetPagesCount(); i++)
{
var url = _urlGenerator.GeneratePageUrl(city, i);
var htmlDoc = await HtmlClient.LoadFromWebAsync(url);
var nodes = htmlDoc.DocumentNode
.SelectNodes(TargetWebsite.DocumentNodeXPath).ToList();
var offers = GetDetails(nodes);
await _repository.AddMany(offers);
//todo find better anti-ban solution
Thread.Sleep(4000);
}
}
