protected void SetGeocodeData(Apartment apartment) { var request = new GeocodingAddressRequest() { Street = apartment.Street, ZipCode = apartment.ZipCode, City = apartment.City.Name, Country = apartment.City.Country, Culture = "en" }; try { var addresses = GeocodingService.GeocodeAddress(request); if (addresses != null && addresses.Count == 1) { var address = addresses.First(); apartment.Latitude = address.Geometry.Location.Latitude; apartment.Longitude = address.Geometry.Location.Longitude; apartment.Neighborhood = address.GetComponent("sublocality_level_1"); apartment.Locality = address.GetComponent("sublocality_level_2"); if (!string.IsNullOrEmpty(apartment.Neighborhood)) { apartment.Neighborhood = apartment.Neighborhood.Replace("Bezirk", string.Empty).Trim(); } // replace the street with the returned info. Google Api return higher quality addresses (i.e. "Sybelstraße" vs "Sybelstr.") apartment.Street = address.GetComponent("route") + " " + address.GetComponent("street_number"); } } catch (Exception) { // ignore error and move on } }
/// <summary> /// Persists the list of scraped Apartments /// </summary> /// <param name="aparmentTasks">List of Apartment tasks</param> private void PersistApartments(List<Task<ApartmentDTO>> aparmentTasks) { UnitOfWork.BeginTransaction(); // Get current city and existing apartments var berlin = CityRepository.Query().Where(c => c.Code == CityCode.Berlin).First(); var existingApartments = ApartmentRepository.Query() .Where(a => !a.IsDeleted && a.PublisherCode == PublisherCode.Akelius && a.City.Id == berlin.Id) .ToDictionary(a => a.PublisherInternalId); // Init auxiliary variables var currentDate = DateTime.Now; var newApartments = 0; var updatedApartments = 0; var deletedApartments = 0; var geocoderUsageCount = 0; foreach (var task in aparmentTasks) { var apartmentDto = task.Result; Apartment apartment; if (existingApartments.ContainsKey(apartmentDto.PublisherInternalId)) { // The apartment already exists. Update its info // Assume that addresses do not change apartment = existingApartments[apartmentDto.PublisherInternalId]; apartment.ModifiedDate = currentDate; apartment.TotalRent = apartmentDto.TotalRent; apartment.NetRent = apartmentDto.NetRent; apartment.Charges = apartmentDto.Charges; apartment.AvailableFrom = apartmentDto.AvailableFrom; apartment.Title = apartmentDto.Title; ApartmentRepository.Update(apartment); existingApartments.Remove(apartmentDto.PublisherInternalId); updatedApartments++; } else { apartment = new Apartment() { City = berlin, CreatedDate = currentDate, ModifiedDate = currentDate, TotalRent = apartmentDto.TotalRent, NetRent = apartmentDto.NetRent, Charges = apartmentDto.Charges, AvailableFrom = apartmentDto.AvailableFrom, Title = apartmentDto.Title, Street = apartmentDto.Street, ZipCode = apartmentDto.ZipCode, PublicationUrl = apartmentDto.PublicationUrl, PublisherCode = apartmentDto.PublisherCode, PublisherInternalId = apartmentDto.PublisherInternalId, Rooms = apartmentDto.Rooms, Size = apartmentDto.Size }; ApartmentRepository.Insert(apartment); newApartments++; } if (apartment.Latitude == null && !string.IsNullOrEmpty(apartment.Street)) { SetGeocodeData(apartment); // Wait 1 second if we hit the API's quota geocoderUsageCount++; if (geocoderUsageCount == 10) { geocoderUsageCount = 0; Thread.Sleep(1000); } } } foreach (var apartment in existingApartments.Values) { // this apartments are not listed anymore, delete them apartment.IsDeleted = true; apartment.ModifiedDate = currentDate; ApartmentRepository.Update(apartment); deletedApartments++; } UnitOfWork.Commit(); LogResults(newApartments, updatedApartments, deletedApartments); }