protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
using var scope = _services.CreateScope();
var cities = await _internalLocationService.GetCities();
var listingRepository = scope.ServiceProvider.GetRequiredService <IListingRepository>();
foreach (var city in cities)
{
foreach (var town in city.Towns)
{
var listings = listingRepository.GetByFilter(x => x.City.Name == city.Name &&
x.Town.Name == town.Name).ToList();
var emlakJetListings = listings.Where(x => x.OwnerSite == (int)Owners.EmlakJet &&
!string.IsNullOrEmpty(x.FullDescription)).ToList();
var hurriyetListings = listings.Where(x => x.OwnerSite == (int)Owners.HurriyetEmlak &&
!string.IsNullOrEmpty(x.FullDescription)).ToList();
foreach (var hurriyetListing in hurriyetListings)
{
var roomNumber = !string.IsNullOrEmpty(hurriyetListing.RoomNumber) ?
hurriyetListing.RoomNumber.Replace(" ", string.Empty):"";
var samePropertyListings = emlakJetListings.Where(x =>
x.RoomNumber.Trim() == roomNumber &&
x.Street.Name == hurriyetListing.Street.Name &&
x.AdvertStatus == hurriyetListing.AdvertStatus);
foreach (var samePropertyListing in samePropertyListings)
{
var similarityMeasure = GetSimilarity(hurriyetListing.FullDescription,
samePropertyListing.FullDescription);
if (similarityMeasure > 0.9)
{
var newSku = $"RE{hurriyetListing.AdvertId}{samePropertyListing.AdvertId}";
await listingRepository.UpdateMany(
new List <ObjectId>() { hurriyetListing.Id, samePropertyListing.Id }, newSku);
break;
}
}
}
}
}
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
var advertStatuses = new Dictionary <string, string>()
{
{ "kiralik", "Kiralık" },
{ "satilik", "Satılık" }
};
using (var scope = _services.CreateScope())
{
var listingRepository = scope.ServiceProvider.GetRequiredService <IListingRepository>();
var cities = await _internalLocationService.GetCities();
foreach (var city in cities)
{
foreach (var town in city.Towns)
{
var cityName = GetEnCharactersInString(city.Name.ToLower());
var townName = GetEnCharactersInString(town.Name.ToLower());
foreach (var advertStatus in advertStatuses)
{
try
{
var advertList =
await _emlakJetService.GetAdvertList($"{advertStatus.Key}-konut/{cityName}-{townName}");
var listings =
await listingRepository.GetByFilter(x =>
x.AdvertStatus == advertStatus.Value &&
x.City.Name == city.Name &&
x.Town.Name == town.Name &&
x.OwnerSite == (int)Owners.EmlakJet)
.ToListAsync(cancellationToken: stoppingToken);
var notExistListings =
listings.Where(x => advertList.All(y => y.Id.ToString() != x.AdvertId))
.Select(x => x.Id).ToList();
if (notExistListings.Any())
{
await listingRepository.DeleteBulkAsync(notExistListings);
}
var newAdverts = advertList.Where(x => listings.All(y => y.AdvertId != x.Id.ToString()))
.ToList();
foreach (var ejListing in newAdverts)
{
try
{
var url = $"https://www.emlakjet.com{ejListing.Url}";
var reSku = $"RE{(int)Owners.EmlakJet}{ejListing.Id}";
var listing = new Listing()
{
City = new LocationModel {
Name = city.Name, Id = city.Id
},
Town = new LocationModel {
Name = town.Name, Id = town.Id
},
Name = ejListing.Title.Tr,
Url = url,
OwnerSite = (int)Owners.EmlakJet,
AdvertId = ejListing.Id.ToString(),
AdvertStatus = advertStatus.Value,
ReSku = reSku
};
var listingDetail = await _emlakJetService.GetAdvertDetail(url);
listing.Price = new PriceModel()
{
Price = listingDetail.PriceOrder, Currency = listingDetail.Currency
};
listing.Images = listingDetail.ImagesFull.Select(x => $"https://imaj.emlakjet.com{x}").ToList();
listing.Street = new LocationModel()
{
Name = listingDetail.Properties.Town.Name.Substring(0, listingDetail.Properties.Town.Name.IndexOf("Mah", StringComparison.Ordinal)).TrimEnd()
};
var ownerType = listingDetail.OrderedProperties
.FirstOrDefault(x => x.Title == "Kimden")?.Value;
if (!string.IsNullOrEmpty(ownerType))
{
if (ownerType == "Emlak Ofisinden")
{
listing.AdvertOwnerType = "RealEstateAgent";
}
else if (ownerType == "Sahibinden")
{
listing.AdvertOwnerType = "Personal";
}
else
{
listing.AdvertOwnerType = "Other";
}
}
listing.FullDescriptionInHtml = listingDetail.DescriptionMasked.Tr;
listing.FullDescription = GetClearText(listingDetail.DescriptionMasked.Tr);
listing.AdvertOwnerName = listingDetail.User.FullName;
listing.AdvertOwnerPhone = listingDetail.PhoneNumber;
listing.RoomNumber =
listingDetail.OrderedProperties.FirstOrDefault(x => x.Title == "Oda Sayısı")?.Value;
listing.SquareMeter =
$"{listingDetail.OrderedProperties.FirstOrDefault(x => x.Title == "Brüt m2")?.Value} / " +
$"{listingDetail.OrderedProperties.FirstOrDefault(x => x.Title == "Net m2")?.Value}";
listing.BuildingAge =
listingDetail.OrderedProperties.FirstOrDefault(x => x.Title == "Bina Yaşı")?.Value;
listing.FloorLocation =
listingDetail.OrderedProperties.FirstOrDefault(x => x.Title == "Bulunduğu Kat")?.Value;
listing.NumberOfFloor =
listingDetail.OrderedProperties.FirstOrDefault(x => x.Title == "Kat Sayısı")?.Value;
listing.FurnitureStatus =
listingDetail.OrderedProperties.FirstOrDefault(x => x.Title == "Eşya Durumu")?.Value;
listing.HeatingType =
listingDetail.OrderedProperties.FirstOrDefault(x => x.Title == "Isıtma")?.Value;
await listingRepository.InsertListing(listing);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
}
}
}
_logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now);
await Task.Delay(30 * 60 * 1000, stoppingToken);
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
using (var scope = _services.CreateScope())
{
var listingRepository = scope.ServiceProvider.GetRequiredService <IListingRepository>();
var cities = await _internalLocationService.GetCities();
foreach (var city in cities)
{
foreach (var town in city.Towns)
{
try
{
var townName = GetEnCharactersInString(town.Name.ToLower());
var response = await _hurriyetService.
GetAdvertsList($"{townName}");
var listings =
await listingRepository.GetByFilter(x =>
x.City.Name == city.Name &&
x.Town.Name == town.Name &&
x.OwnerSite == (int)Owners.HurriyetEmlak)
.ToListAsync(cancellationToken: stoppingToken);
var notExistListings =
listings.Where(x => response.All(y => GetAdvertId(y.Url.ToString()) != x.AdvertId))
.Select(x => x.Id).ToList();
if (notExistListings.Any())
{
await listingRepository.DeleteBulkAsync(notExistListings);
}
var newAdverts = response.Where(x => listings
.All(y => y.AdvertId != GetAdvertId(x.Url.ToString())))
.ToList();
foreach (var item in newAdverts)
{
try
{
var itemUrl = item.Url.ToString();
var advertId = GetAdvertId(itemUrl);
var reSku = $"RE{(int)Owners.HurriyetEmlak}{advertId}";
var listing = new Listing
{
City = new LocationModel {
Name = city.Name, Id = city.Id
},
Town = new LocationModel {
Name = town.Name, Id = town.Id
},
Name = item.Name,
Url = itemUrl,
OwnerSite = (int)Owners.HurriyetEmlak,
AdvertId = advertId,
ReSku = reSku
};
var detail = await _hurriyetService.GetAdvertDetail(itemUrl);
listing.Images = detail.Offers.Image.Select(x => x.ContentUrl.ToString()).ToList();
listing.Street = new LocationModel()
{
Name = detail.Offers.ItemOfferedDetail.Address.StreetAddress
};
listing.Price = new PriceModel()
{
Price = detail.Offers.Price, Currency = detail.Offers.PriceCurrency
};
listing.ShortDescription = detail.Offers.Description;
listing.FullDescription = detail.FullDescription;
listing.FullDescriptionInHtml = detail.FullDescriptionInHtml;
listing.AdvertOwnerType = detail.IsPersonal ? "Personal" : GetAdvertiseOwner(detail.Offers.Seller);
listing.AdvertOwnerName = detail.Offers.Seller.Name;
listing.AdvertOwnerPhone = detail.Offers.Seller.Telephone;
var furnitureStatus =
detail.AdvertFeatures.FirstOrDefault(x => x.Name == "Eşya Durumu")?.Value;
var advertStatus = detail.AdvertFeatures.FirstOrDefault(x => x.Name == "İlan Durumu")
?.Value;
if (!string.IsNullOrEmpty(advertStatus))
{
listing.AdvertStatus = advertStatus == "Günlük Kiralık" ? "Kiralık" : advertStatus;
}
if (!string.IsNullOrEmpty(furnitureStatus))
{
listing.FurnitureStatus =
furnitureStatus == "Eşyalı Değil" ? "Boş" : furnitureStatus;
}
listing.RoomNumber =
detail.AdvertFeatures.FirstOrDefault(x => x.Name == "Oda + Salon Sayısı")?.Value;
listing.SquareMeter =
detail.AdvertFeatures.FirstOrDefault(x => x.Name == "Brüt / Net M2")?.Value;
listing.BuildingAge =
detail.AdvertFeatures.FirstOrDefault(x => x.Name == "Bina Yaşı")?.Value;
listing.FloorLocation =
detail.AdvertFeatures.FirstOrDefault(x => x.Name == "Bulunduğu Kat")?.Value;
listing.NumberOfFloor =
detail.AdvertFeatures.FirstOrDefault(x => x.Name == "Kat Sayısı")?.Value;
listing.HeatingType =
detail.AdvertFeatures.FirstOrDefault(x => x.Name == "Yakıt Tipi")?.Value;
await listingRepository.InsertListing(listing);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
}
}
_logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now);
await Task.Delay(30 * 60 * 1000, stoppingToken);
}
}
