private List <ShortItemDtoModel> _getItemShortFromNodes(IEnumerable <HtmlNode> nodes) { var result = new List <ShortItemDtoModel>(); foreach (var node in nodes) { var itemId = _getIdFromTr(node); var apportamentType = _getApportamentTypeFromTr(node); var city = _getCityFromTr(node); var address = _getAddressFromTr(node); var checkHour = _getCheckHourFromTr(node); var price = _getPriceFromTr(node); var dateUpdate = _getDateUpdateFromTr(node); var item = new ShortItemDtoModel() { ItemId = itemId, ApportamentType = apportamentType, City = city, Address = address, CheckHour = checkHour, Price = price, DateUpdate = dateUpdate, }; result.Add(item); } return(result); }
private bool _isDownloadedItem(ShortItemDtoModel shortItem, ScraperWinWinStateModel state) { var filename = $"{state.ItemsPath}/{shortItem.ItemId}.json"; var result = File.Exists(filename); return(result); }
public async Task <AdItemWinWinDtoModel> GetItemDtoAsync(string itemId) { var state = (ScraperWinWinStateModel)_state; var shortItem = new ShortItemDtoModel() { ItemId = itemId }; var itemPage = await _loadItemPageAsync(shortItem, state); var itemDto = await _parseItemPageAsync(itemPage, shortItem); return(itemDto); }
private async Task <bool> _downloadItemAsync(ShortItemDtoModel shortItem, ScraperWinWinStateModel state) { var result = false; try { var itemPage = await _loadItemPageAsync(shortItem, state); var itemDto = await _parseItemPageAsync(itemPage, shortItem); await _saveItemDtoAsync(itemDto, state); result = true; } catch (Exception exception) { _log($"Error-z2. {exception.Message}"); } return(result); }
private async Task <HtmlDocument> _loadItemPageAsync(ShortItemDtoModel shortItem, ScraperWinWinStateModel state) { var itemId = shortItem.ItemId; _log($"Loading item-id:{itemId}"); var url = $"https://www.winwin.co.il/RealEstate/ForRent/Ads/RealEstateAds,{itemId}.aspx"; var webGet = new HtmlWeb(); HtmlDocument result = null; try { result = await webGet.LoadFromWebAsync(url); } catch (Exception exception) { _log($"Error-z3. {exception.Message}"); } //_log($"Load item-id:{itemId} is done"); return(result); }
private async Task <AdItemWinWinDtoModel> _parseItemPageAsync(HtmlDocument itemPage, ShortItemDtoModel shortItem) { DataCoordinatesLatLng coordinates = new DataCoordinatesLatLng(); var iframe = _parseItemPage_GetMediaFrameAsync(itemPage).Result; var itemId = shortItem.ItemId; var city = shortItem.City; var dateUpdate = shortItem.DateUpdate; var area = _parseItemPage_GetArea(itemPage); var streetAddress = _parseItemPage_GetStreetAddress(itemPage); var rooms = _parseItemPage_GetRooms(itemPage); var floor = _parseItemPage_GetFloor(itemPage); var state = _parseItemPage_GetState(itemPage); var dateEnter = _parseItemPage_GetDateEnter(itemPage); var square = _parseItemPage_GetSquare(itemPage); var isPartners = _parseItemPage_GetIsPartners(itemPage); var amountPayment = _parseItemPage_GetAmountPayments(itemPage); var description = _parseItemPage_GetDescription(itemPage); var price = _parseItemPage_GetPrice(itemPage); var isAgent = _parseItemPage_IsAgency(itemPage); string phone1, phone2, contactName; if (isAgent) { contactName = _parseItemPage_GetContactName_Agent(itemPage); phone1 = _parseItemPage_GetPhone1_Agent(itemPage); phone2 = _parseItemPage_GetPhone2_Agent(itemPage); } else { contactName = _parseItemPage_GetContactName_Private(itemPage); phone1 = _parseItemPage_GetPhone1_Private(itemPage); phone2 = _parseItemPage_GetPhone2_Private(itemPage); } var isOk = _parseItemPage_GetCoordinates(itemPage, coordinates); if (!isOk) { isOk = _parseItemPage_GetCoordinates2(iframe, coordinates); } if (!isOk) { isOk = await GetCoordinates_WebClientAsync(city, area, coordinates); } var images = _parseItemPage_GetImages(iframe); var result = new AdItemWinWinDtoModel() { ItemId = itemId, DateUpdate = dateUpdate, Longitude = coordinates.lng, Latitude = coordinates.lat, City = city, Area = area, StreetAddress = streetAddress, Rooms = rooms, Floor = floor, State = state, DateEnter = dateEnter, Square = square, IsPartners = isPartners, AmountPayment = amountPayment, Description = description, Price = price, ContactName = contactName, Phone1 = phone1, Phone2 = phone2, IsAgent = isAgent ? "yes":"no", Images = images, }; return(result); }