protected override async Task ExecuteAsync(CancellationToken stoppingToken) { while (!stoppingToken.IsCancellationRequested) { using var scope = _services.CreateScope(); var cities = await _internalLocationService.GetCities(); var listingRepository = scope.ServiceProvider.GetRequiredService <IListingRepository>(); foreach (var city in cities) { foreach (var town in city.Towns) { var listings = listingRepository.GetByFilter(x => x.City.Name == city.Name && x.Town.Name == town.Name).ToList(); var emlakJetListings = listings.Where(x => x.OwnerSite == (int)Owners.EmlakJet && !string.IsNullOrEmpty(x.FullDescription)).ToList(); var hurriyetListings = listings.Where(x => x.OwnerSite == (int)Owners.HurriyetEmlak && !string.IsNullOrEmpty(x.FullDescription)).ToList(); foreach (var hurriyetListing in hurriyetListings) { var roomNumber = !string.IsNullOrEmpty(hurriyetListing.RoomNumber) ? hurriyetListing.RoomNumber.Replace(" ", string.Empty):""; var samePropertyListings = emlakJetListings.Where(x => x.RoomNumber.Trim() == roomNumber && x.Street.Name == hurriyetListing.Street.Name && x.AdvertStatus == hurriyetListing.AdvertStatus); foreach (var samePropertyListing in samePropertyListings) { var similarityMeasure = GetSimilarity(hurriyetListing.FullDescription, samePropertyListing.FullDescription); if (similarityMeasure > 0.9) { var newSku = $"RE{hurriyetListing.AdvertId}{samePropertyListing.AdvertId}"; await listingRepository.UpdateMany( new List <ObjectId>() { hurriyetListing.Id, samePropertyListing.Id }, newSku); break; } } } } } } }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) { while (!stoppingToken.IsCancellationRequested) { var advertStatuses = new Dictionary <string, string>() { { "kiralik", "Kiralık" }, { "satilik", "Satılık" } }; using (var scope = _services.CreateScope()) { var listingRepository = scope.ServiceProvider.GetRequiredService <IListingRepository>(); var cities = await _internalLocationService.GetCities(); foreach (var city in cities) { foreach (var town in city.Towns) { var cityName = GetEnCharactersInString(city.Name.ToLower()); var townName = GetEnCharactersInString(town.Name.ToLower()); foreach (var advertStatus in advertStatuses) { try { var advertList = await _emlakJetService.GetAdvertList($"{advertStatus.Key}-konut/{cityName}-{townName}"); var listings = await listingRepository.GetByFilter(x => x.AdvertStatus == advertStatus.Value && x.City.Name == city.Name && x.Town.Name == town.Name && x.OwnerSite == (int)Owners.EmlakJet) .ToListAsync(cancellationToken: stoppingToken); var notExistListings = listings.Where(x => advertList.All(y => y.Id.ToString() != x.AdvertId)) .Select(x => x.Id).ToList(); if (notExistListings.Any()) { await listingRepository.DeleteBulkAsync(notExistListings); } var newAdverts = advertList.Where(x => listings.All(y => y.AdvertId != x.Id.ToString())) .ToList(); foreach (var ejListing in newAdverts) { try { var url = $"https://www.emlakjet.com{ejListing.Url}"; var reSku = $"RE{(int)Owners.EmlakJet}{ejListing.Id}"; var listing = new Listing() { City = new LocationModel { Name = city.Name, Id = city.Id }, Town = new LocationModel { Name = town.Name, Id = town.Id }, Name = ejListing.Title.Tr, Url = url, OwnerSite = (int)Owners.EmlakJet, AdvertId = ejListing.Id.ToString(), AdvertStatus = advertStatus.Value, ReSku = reSku }; var listingDetail = await _emlakJetService.GetAdvertDetail(url); listing.Price = new PriceModel() { Price = listingDetail.PriceOrder, Currency = listingDetail.Currency }; listing.Images = listingDetail.ImagesFull.Select(x => $"https://imaj.emlakjet.com{x}").ToList(); listing.Street = new LocationModel() { Name = listingDetail.Properties.Town.Name.Substring(0, listingDetail.Properties.Town.Name.IndexOf("Mah", StringComparison.Ordinal)).TrimEnd() }; var ownerType = listingDetail.OrderedProperties .FirstOrDefault(x => x.Title == "Kimden")?.Value; if (!string.IsNullOrEmpty(ownerType)) { if (ownerType == "Emlak Ofisinden") { listing.AdvertOwnerType = "RealEstateAgent"; } else if (ownerType == "Sahibinden") { listing.AdvertOwnerType = "Personal"; } else { listing.AdvertOwnerType = "Other"; } } listing.FullDescriptionInHtml = listingDetail.DescriptionMasked.Tr; listing.FullDescription = GetClearText(listingDetail.DescriptionMasked.Tr); listing.AdvertOwnerName = listingDetail.User.FullName; listing.AdvertOwnerPhone = listingDetail.PhoneNumber; listing.RoomNumber = listingDetail.OrderedProperties.FirstOrDefault(x => x.Title == "Oda Sayısı")?.Value; listing.SquareMeter = $"{listingDetail.OrderedProperties.FirstOrDefault(x => x.Title == "Brüt m2")?.Value} / " + $"{listingDetail.OrderedProperties.FirstOrDefault(x => x.Title == "Net m2")?.Value}"; listing.BuildingAge = listingDetail.OrderedProperties.FirstOrDefault(x => x.Title == "Bina Yaşı")?.Value; listing.FloorLocation = listingDetail.OrderedProperties.FirstOrDefault(x => x.Title == "Bulunduğu Kat")?.Value; listing.NumberOfFloor = listingDetail.OrderedProperties.FirstOrDefault(x => x.Title == "Kat Sayısı")?.Value; listing.FurnitureStatus = listingDetail.OrderedProperties.FirstOrDefault(x => x.Title == "Eşya Durumu")?.Value; listing.HeatingType = listingDetail.OrderedProperties.FirstOrDefault(x => x.Title == "Isıtma")?.Value; await listingRepository.InsertListing(listing); } catch (Exception e) { Console.WriteLine(e); } } } catch (Exception e) { Console.WriteLine(e); } } } } } _logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now); await Task.Delay(30 * 60 * 1000, stoppingToken); } }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) { while (!stoppingToken.IsCancellationRequested) { using (var scope = _services.CreateScope()) { var listingRepository = scope.ServiceProvider.GetRequiredService <IListingRepository>(); var cities = await _internalLocationService.GetCities(); foreach (var city in cities) { foreach (var town in city.Towns) { try { var townName = GetEnCharactersInString(town.Name.ToLower()); var response = await _hurriyetService. GetAdvertsList($"{townName}"); var listings = await listingRepository.GetByFilter(x => x.City.Name == city.Name && x.Town.Name == town.Name && x.OwnerSite == (int)Owners.HurriyetEmlak) .ToListAsync(cancellationToken: stoppingToken); var notExistListings = listings.Where(x => response.All(y => GetAdvertId(y.Url.ToString()) != x.AdvertId)) .Select(x => x.Id).ToList(); if (notExistListings.Any()) { await listingRepository.DeleteBulkAsync(notExistListings); } var newAdverts = response.Where(x => listings .All(y => y.AdvertId != GetAdvertId(x.Url.ToString()))) .ToList(); foreach (var item in newAdverts) { try { var itemUrl = item.Url.ToString(); var advertId = GetAdvertId(itemUrl); var reSku = $"RE{(int)Owners.HurriyetEmlak}{advertId}"; var listing = new Listing { City = new LocationModel { Name = city.Name, Id = city.Id }, Town = new LocationModel { Name = town.Name, Id = town.Id }, Name = item.Name, Url = itemUrl, OwnerSite = (int)Owners.HurriyetEmlak, AdvertId = advertId, ReSku = reSku }; var detail = await _hurriyetService.GetAdvertDetail(itemUrl); listing.Images = detail.Offers.Image.Select(x => x.ContentUrl.ToString()).ToList(); listing.Street = new LocationModel() { Name = detail.Offers.ItemOfferedDetail.Address.StreetAddress }; listing.Price = new PriceModel() { Price = detail.Offers.Price, Currency = detail.Offers.PriceCurrency }; listing.ShortDescription = detail.Offers.Description; listing.FullDescription = detail.FullDescription; listing.FullDescriptionInHtml = detail.FullDescriptionInHtml; listing.AdvertOwnerType = detail.IsPersonal ? "Personal" : GetAdvertiseOwner(detail.Offers.Seller); listing.AdvertOwnerName = detail.Offers.Seller.Name; listing.AdvertOwnerPhone = detail.Offers.Seller.Telephone; var furnitureStatus = detail.AdvertFeatures.FirstOrDefault(x => x.Name == "Eşya Durumu")?.Value; var advertStatus = detail.AdvertFeatures.FirstOrDefault(x => x.Name == "İlan Durumu") ?.Value; if (!string.IsNullOrEmpty(advertStatus)) { listing.AdvertStatus = advertStatus == "Günlük Kiralık" ? "Kiralık" : advertStatus; } if (!string.IsNullOrEmpty(furnitureStatus)) { listing.FurnitureStatus = furnitureStatus == "Eşyalı Değil" ? "Boş" : furnitureStatus; } listing.RoomNumber = detail.AdvertFeatures.FirstOrDefault(x => x.Name == "Oda + Salon Sayısı")?.Value; listing.SquareMeter = detail.AdvertFeatures.FirstOrDefault(x => x.Name == "Brüt / Net M2")?.Value; listing.BuildingAge = detail.AdvertFeatures.FirstOrDefault(x => x.Name == "Bina Yaşı")?.Value; listing.FloorLocation = detail.AdvertFeatures.FirstOrDefault(x => x.Name == "Bulunduğu Kat")?.Value; listing.NumberOfFloor = detail.AdvertFeatures.FirstOrDefault(x => x.Name == "Kat Sayısı")?.Value; listing.HeatingType = detail.AdvertFeatures.FirstOrDefault(x => x.Name == "Yakıt Tipi")?.Value; await listingRepository.InsertListing(listing); } catch (Exception e) { Console.WriteLine(e); } } } catch (Exception e) { Console.WriteLine(e); } } } } _logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now); await Task.Delay(30 * 60 * 1000, stoppingToken); } }