private Car GetCar(ModelSearchDto model, string adUrl)
{
WebPage carPage = null;
int counter = 0;
while (carPage == null)
{
if (counter == 10)
{
return null;
}
try
{
carPage = browser.NavigateToPage(new Uri(adUrl));
counter++;
}
catch (IOException)
{
logger.LogWarning("Connection failed, retry");
}
catch (SocketException)
{
logger.LogWarning("Connection failed, retry");
}
catch (WebException)
{
logger.LogWarning("Timed out, retry");
}
catch (Exception)
{
logger.LogWarning("Unknow error during navigation, retry");
}
}
#if DEBUG
File.WriteAllText(@"./CarsBg/" + model.BrandName.Trim() + "/" + model.Name.Trim() + "/" + adUrl.Substring(25) + ".html", carPage.Content);
#endif
HtmlNode pageNode = carPage.Html;
Car car = new Car();
car.SourceId = (int)SourceEnum.CarsBg;
car.AdUrl = adUrl;
car.ModelId = model.Id;
foreach (PropertyInfo property in typeof(Car).GetProperties())
{
if (CarSelectors.ContainsKey(property.Name))
{
string xpathSelector = CarSelectors[property.Name];
HtmlNode node = pageNode.SelectSingleNode(xpathSelector);
if (node != null)
{
AssignProperty(car, property.Name, node.InnerText.Trim());
}
}
}
return car;
}
public void AddModel(ModelSearchDto model)
{
if (parameters.ContainsKey("brandId") && model.BrandKey != parameters["brandId"])
{
throw new ArgumentException(
AutoResourceFile.OnlyModelsUnderTheSameBrandCanBeUsedForSearchCriteria);
}
parameters["modelId"] = "0";
parameters.Add("models[]", model.ModelKeys);
}
public IEnumerable <Car> GetAllCars(ModelSearchDto model, bool newCars)
{
#if DEBUG
Directory.CreateDirectory(@"./CarsBg/" + model.BrandName + "/" + model.Name);
#endif
CarsBgSearchCriteria carsBgSearchCriteria = new CarsBgSearchCriteria();
carsBgSearchCriteria.AddModel(model);
var results = new List <Car>();
logger.LogDebug("Exploring" + model.BrandName + " " + model.Name);
while (true)
{
// CarsBg bugs and displays the 50 page for any page after 50
if (carsBgSearchCriteria.Page > 50)
{
break;
}
var url = carsBgSearchCriteria.ToString();
WebPage carsPage = browser.NavigateToPage(new Uri(url));
HtmlNodeCollection links = carsPage.Html.SelectNodes("//table[@class='tableListResults']//tr[contains(@class, 'odd') or contains(@class, 'even')]/td[2]/a");
if (links == null || links.Count == 0)
{
break;
}
foreach (HtmlNode link in links)
{
string href = link.GetAttributeValue("href");
string adUrl = $"https://www.cars.bg/{href}";
if (!existingCars.Contains(adUrl) && existingCars.Add(adUrl))
{
logger.LogDebug("Processing:" + adUrl);
Car car = GetCar(model, adUrl);
results.Add(car);
}
else if (newCars)
{
return(results);
}
}
carsBgSearchCriteria.Page++;
logger.LogDebug("Exploring page number:" + carsBgSearchCriteria.Page);
}
return(results);
}
