protected override string GetSearchUrl(Search search, PropertyWebsite propertyWebsite) { var queryStringParameters = new StringBuilder(propertyWebsite.WebsitePropertyUrl + "/kopen/?"); if (search.TownId != null) { var zipCode = ImmoCache.Towns.FirstOrDefault(p => p.Id == search.TownId).PostCode; queryStringParameters.Append("zips=" + string.Join("%2C", zipCode) + "&"); } if (search.PropertyType.HasValue) { queryStringParameters.Append("WebIdD=" + (search.PropertyType == Domain.Enums.PropertyTypeEnum.House ? 2:1) + "&"); } if (search.MaxPrice.HasValue) { queryStringParameters.Append("priceTo=" + search.MaxPrice + "&"); } if (search.MinRoomsNo.HasValue) { queryStringParameters.Append("minRooms=" + search.MinRoomsNo + "&"); } var result = queryStringParameters.ToString(); return(result); }
public IEnumerable <Property> SaveProperties(Search search, PropertyWebsite propertyWebsite) { var url = GetSearchUrl(search, propertyWebsite); var propertiesHtmls = GetPropertiesDetailsHtmls(url); ConcurrentBag <Property> propertiesBag = new ConcurrentBag <Property>(); Parallel.ForEach(propertiesHtmls, (currentProperty) => { var curentProperty = GetPropertyFromDetailsHtml(currentProperty.Key, currentProperty.Value, search, propertyWebsite); propertiesBag.Add(curentProperty); }); var properties = SaveProperties(propertiesBag.ToList()); return(properties); }
protected abstract Property GetPropertyFromDetailsHtml(string originalUrl, string html, Search search, PropertyWebsite propertyWebsite);
protected abstract string GetSearchUrl(Search search, PropertyWebsite propertyWebsite);
protected override Property GetPropertyFromDetailsHtml(string originalUrl, string html, Search search, PropertyWebsite propertyWebsite) { var document = BrowsingContext.New(Configuration.Default.WithDefaultLoader()).OpenAsync(req => req.Content(html)).Result; var property = new Property(); property.Id = Guid.NewGuid(); property.OriginalURL = originalUrl; property.AddType = search.AddType; property.CreationDate = DateTime.UtcNow; property.PropertyType = search.PropertyType; property.PropertyWebsiteId = propertyWebsite.Id; property.BathroomNo = GetPropertyElement <Int32?>(document, "AANTAL BADKAMERS"); property.BedroomsNo = GetPropertyElement <Int32?>(document, "AANTAL SLAAPKAMERS"); property.ConstructionYear = GetPropertyElement <Int32?>(document, "BOUWJAAR");; property.Description = document.QuerySelector("div.main-content div.content p").TextContent; property.Surface = GetPropertyElement <double?>(document, "BEWOONBARE OPPERVLAKTE", "m2");; property.Number = null; property.Pictures = String.Join(",", document.QuerySelectorAll("section.property-slider a")?.Select(p => p.Attributes["href"]?.Value)); property.Price = ChangeType <double?>(document.QuerySelector("span.property-price").TextContent.Replace("€", string.Empty).Trim()); property.Street = null; property.TownId = search.TownId; property.Title = RemoveFormatting(document.QuerySelector("div.main-content header h1").TextContent); return(property); }