protected void SetGeocodeData(Apartment apartment)
{
var request = new GeocodingAddressRequest()
{
Street = apartment.Street,
ZipCode = apartment.ZipCode,
City = apartment.City.Name,
Country = apartment.City.Country,
Culture = "en"
};
try
{
var addresses = GeocodingService.GeocodeAddress(request);
if (addresses != null && addresses.Count == 1)
{
var address = addresses.First();
apartment.Latitude = address.Geometry.Location.Latitude;
apartment.Longitude = address.Geometry.Location.Longitude;
apartment.Neighborhood = address.GetComponent("sublocality_level_1");
apartment.Locality = address.GetComponent("sublocality_level_2");
if (!string.IsNullOrEmpty(apartment.Neighborhood))
{
apartment.Neighborhood = apartment.Neighborhood.Replace("Bezirk", string.Empty).Trim();
}
// replace the street with the returned info. Google Api return higher quality addresses (i.e. "Sybelstraße" vs "Sybelstr.")
apartment.Street = address.GetComponent("route") + " " + address.GetComponent("street_number");
}
}
catch (Exception)
{
// ignore error and move on
}
}
/// <summary>
/// Persists the list of scraped Apartments
/// </summary>
/// <param name="aparmentTasks">List of Apartment tasks</param>
private void PersistApartments(List<Task<ApartmentDTO>> aparmentTasks)
{
UnitOfWork.BeginTransaction();
// Get current city and existing apartments
var berlin = CityRepository.Query().Where(c => c.Code == CityCode.Berlin).First();
var existingApartments = ApartmentRepository.Query()
.Where(a => !a.IsDeleted && a.PublisherCode == PublisherCode.Akelius && a.City.Id == berlin.Id)
.ToDictionary(a => a.PublisherInternalId);
// Init auxiliary variables
var currentDate = DateTime.Now;
var newApartments = 0;
var updatedApartments = 0;
var deletedApartments = 0;
var geocoderUsageCount = 0;
foreach (var task in aparmentTasks)
{
var apartmentDto = task.Result;
Apartment apartment;
if (existingApartments.ContainsKey(apartmentDto.PublisherInternalId))
{
// The apartment already exists. Update its info
// Assume that addresses do not change
apartment = existingApartments[apartmentDto.PublisherInternalId];
apartment.ModifiedDate = currentDate;
apartment.TotalRent = apartmentDto.TotalRent;
apartment.NetRent = apartmentDto.NetRent;
apartment.Charges = apartmentDto.Charges;
apartment.AvailableFrom = apartmentDto.AvailableFrom;
apartment.Title = apartmentDto.Title;
ApartmentRepository.Update(apartment);
existingApartments.Remove(apartmentDto.PublisherInternalId);
updatedApartments++;
}
else
{
apartment = new Apartment()
{
City = berlin,
CreatedDate = currentDate,
ModifiedDate = currentDate,
TotalRent = apartmentDto.TotalRent,
NetRent = apartmentDto.NetRent,
Charges = apartmentDto.Charges,
AvailableFrom = apartmentDto.AvailableFrom,
Title = apartmentDto.Title,
Street = apartmentDto.Street,
ZipCode = apartmentDto.ZipCode,
PublicationUrl = apartmentDto.PublicationUrl,
PublisherCode = apartmentDto.PublisherCode,
PublisherInternalId = apartmentDto.PublisherInternalId,
Rooms = apartmentDto.Rooms,
Size = apartmentDto.Size
};
ApartmentRepository.Insert(apartment);
newApartments++;
}
if (apartment.Latitude == null && !string.IsNullOrEmpty(apartment.Street))
{
SetGeocodeData(apartment);
// Wait 1 second if we hit the API's quota
geocoderUsageCount++;
if (geocoderUsageCount == 10)
{
geocoderUsageCount = 0;
Thread.Sleep(1000);
}
}
}
foreach (var apartment in existingApartments.Values)
{
// this apartments are not listed anymore, delete them
apartment.IsDeleted = true;
apartment.ModifiedDate = currentDate;
ApartmentRepository.Update(apartment);
deletedApartments++;
}
UnitOfWork.Commit();
LogResults(newApartments, updatedApartments, deletedApartments);
}