private static List <BaseHouseInfo> GetHouseData(string result) { List <BaseHouseInfo> houseList = new List <BaseHouseInfo>(); var resultJObject = JsonConvert.DeserializeObject <JObject>(result); if (resultJObject == null || resultJObject["head"] == null || !resultJObject["head"]["success"].ToObject <Boolean>()) { return(houseList); } foreach (var item in resultJObject["houseList"]) { BaseHouseInfo houseInfo = new BaseHouseInfo(); var houseDesc = item["houseDescript"].ToObject <string>().Replace("😄", ""); var houseURL = $"http://www.huzhumaifang.com/Renting/house_detail/id/{item["houseId"]}.html"; houseInfo.HouseOnlineURL = houseURL; houseInfo.HouseTitle = houseDesc; houseInfo.HouseLocation = houseDesc; houseInfo.HouseText = item.ToString(); houseInfo.HousePrice = item["houseRentPrice"].ToObject <Int32>(); houseInfo.DisPlayPrice = item["houseRentPrice"].ToString(); houseInfo.LocationCityName = "上海"; houseInfo.PubTime = item["houseCreateTime"].ToObject <DateTime>(); houseInfo.PicURLs = item["bigPicUrls"].ToString(); houseInfo.Source = ConstConfigName.HuZhuZuFang; houseList.Add(houseInfo); } return(houseList); }
private static List <BaseHouseInfo> GetHouseData(string cityShortCutName, string result) { var houseList = new List <BaseHouseInfo>(); if (string.IsNullOrEmpty(result)) { return(houseList); } var resultJObject = JsonConvert.DeserializeObject <JObject>(result); foreach (var item in resultJObject["items"]) { BaseHouseInfo houseInfo = new BaseHouseInfo(); string houseURL = GetHouseOnlineURL(cityShortCutName, item); houseInfo.HouseOnlineURL = houseURL; houseInfo.HouseLocation = item["headline"].ToObject <string>(); houseInfo.HouseTitle = item["headline"].ToObject <string>(); houseInfo.LocationCityName = item["cityName"].ToObject <string>(); houseInfo.HouseText = item.ToString(); houseInfo.HousePrice = item["totalPrice"].ToObject <Int32>(); houseInfo.DisPlayPrice = item["totalPrice"].ToString(); houseInfo.PubTime = item["publishTime"].ToObject <DateTime>(); houseInfo.Source = ConstConfigName.CCBHouse; houseList.Add(houseInfo); } return(houseList); }
public override List <BaseHouseInfo> ParseHouses(JToken config, string data) { var lstHouseInfo = new List <BaseHouseInfo>(); if (string.IsNullOrEmpty(data)) { return(lstHouseInfo); } var cityName = config["cityName"]?.ToString(); var citySortName = config["citySortName"]?.ToString(); var resultJObject = JsonConvert.DeserializeObject <JObject>(data); foreach (var rentHouse in resultJObject["data"]["list"]) { var housePrice = GetPrice(rentHouse); var house = new BaseHouseInfo() { HouseLocation = rentHouse["house_title"].ToString(), HouseTitle = rentHouse["house_title"].ToString() + rentHouse["desc"].ToString(), HouseOnlineURL = GetHouseURL(rentHouse, citySortName), HouseText = rentHouse.ToString(), HousePrice = housePrice, IsAnalyzed = true, DisPlayPrice = housePrice > 0 ? $"{housePrice}元" : "", Source = ConstConfigName.Beike, LocationCityName = cityName, Status = 1, PicURLs = JsonConvert.SerializeObject(new List <string>()), PubTime = DateTime.Now }; lstHouseInfo.Add(house); } return(lstHouseInfo); }
public override List <BaseHouseInfo> ParseHouses(JToken config, string data) { var houses = new List <BaseHouseInfo>(); var resultJObject = JsonConvert.DeserializeObject <JObject>(data); var cityName = config["cityname"].ToString(); foreach (var topic in resultJObject["topics"]) { var housePrice = JiebaTools.GetHousePrice(topic["content"].ToString()); var photos = topic["photos"]?.Select(photo => photo["alt"].ToString()).ToList(); var house = new BaseHouseInfo() { HouseLocation = topic["title"].ToString(), HouseTitle = topic["title"].ToString(), HouseOnlineURL = topic["share_url"].ToString(), HouseText = topic["content"].ToString(), HousePrice = housePrice, IsAnalyzed = true, DisPlayPrice = housePrice > 0 ? $"{housePrice}元" : "", Source = ConstConfigName.Douban, LocationCityName = cityName, Status = 1, PicURLs = JsonConvert.SerializeObject(photos), PubTime = topic["created"].ToObject <DateTime>() }; houses.Add(house); } return(houses); }
private static BaseHouseInfo ConvertToHouse(string shortCutName, string cityName, JToken info) { var onlineUrl = $"https://{shortCutName}.58.com/pinpaigongyu/{info["infoID"].ToString()}x.shtml"; var housePrice = decimal.Parse(info["minPrice"].ToString()); var houseInfo = new BaseHouseInfo { HouseTitle = $"{info["title"].ToString()}-{info["layout"].ToString()}", HouseOnlineURL = onlineUrl, DisPlayPrice = info["priceTitle"].ToString(), HouseLocation = GetHouseLocation(info), Source = ConstConfigName.PinPaiGongYu, HousePrice = housePrice, HouseText = info.ToString(), LocationCityName = cityName, PubTime = new DateTime(info["postDate"]["year"].ToObject <int>(), info["postDate"]["mon"].ToObject <int>(), info["postDate"]["mday"].ToObject <int>(), info["postDate"]["hours"].ToObject <int>(), info["postDate"]["minutes"].ToObject <int>(), info["postDate"]["seconds"].ToObject <int>()), PicURLs = info["picsUrl"].ToString() }; return(houseInfo); }
private static List <BaseHouseInfo> GetHouses(string shortCutName, string cityName, string json) { List <BaseHouseInfo> houseList = new List <BaseHouseInfo>(); var resultJObject = JsonConvert.DeserializeObject <JObject>(json); foreach (var info in resultJObject["result"]["getListInfo"]["infolist"]) { var onlineUrl = $"https://{shortCutName}.58.com/pinpaigongyu/{info["infoID"].ToString()}x.shtml"; var housePrice = decimal.Parse(info["minPrice"].ToString()); var houseInfo = new BaseHouseInfo { HouseTitle = $"{info["title"].ToString()}-{info["layout"].ToString()}", HouseOnlineURL = onlineUrl, DisPlayPrice = info["priceTitle"].ToString(), HouseLocation = GetHouseLocation(info), Source = ConstConfigName.PinPaiGongYu, HousePrice = housePrice, HouseText = info.ToString(), LocationCityName = cityName, PubTime = new DateTime(info["postDate"]["year"].ToObject <int>(), info["postDate"]["mon"].ToObject <int>(), info["postDate"]["mday"].ToObject <int>(), info["postDate"]["hours"].ToObject <int>(), info["postDate"]["minutes"].ToObject <int>(), info["postDate"]["seconds"].ToObject <int>()), PicURLs = info["picsUrl"].ToString() }; houseList.Add(houseInfo); } return(houseList); }
public override List <BaseHouseInfo> ParseHouses(JToken config, string jsonOrHTML) { var houseList = new List <BaseHouseInfo>(); if (string.IsNullOrEmpty(jsonOrHTML)) { return(houseList); } var htmlDoc = htmlParser.Parse(jsonOrHTML); var houseItems = htmlDoc.QuerySelectorAll("li.listing-ad.item-regular"); if (!houseItems.Any()) { return(houseList); } var cityName = config["cityname"]?.ToString(); foreach (var item in houseItems) { var element = item.QuerySelector("div.media-body"); if (element == null) { continue; } var disPlayPrice = element.QuerySelector("span.highlight").TextContent; int.TryParse(disPlayPrice.Replace("元", ""), out var housePrice); var adTitle = element.QuerySelector("a.ad-title"); var detailList = element.QuerySelectorAll("div.ad-item-detail"); var descTitle = detailList[0]; var localTitle = detailList[1]; if (localTitle.QuerySelector("a.source") != null) { localTitle.QuerySelector("a.source").Remove(); } if (localTitle.QuerySelector("time") != null) { localTitle.QuerySelector("time").Remove(); } var houseLocation = localTitle.TextContent.Replace("来自:", "").Trim(); var houseInfo = new BaseHouseInfo { HouseTitle = adTitle.TextContent + houseLocation, HouseOnlineURL = adTitle.GetAttribute("href"), DisPlayPrice = disPlayPrice, HouseLocation = houseLocation, Source = ConstConfigName.BaiXing, HousePrice = housePrice, HouseText = element.InnerHtml, LocationCityName = cityName, PubTime = DateTime.Now, PicURLs = GetPhotos(item) }; houseList.Add(houseInfo); } return(houseList); }
public static List <BaseHouseInfo> GetHouseData(string cityName, int cityID, int currentPage, int rentTypes = 2) { List <BaseHouseInfo> lstHouse = new List <BaseHouseInfo>(); var result = GetAPIResult(cityID, currentPage, rentTypes); if (string.IsNullOrEmpty(result)) { return(lstHouse); } var resultJObject = JsonConvert.DeserializeObject <JObject>(result); if (resultJObject["message"].ToString() != "success" || resultJObject["content"] == null || resultJObject["content"]["roomInfos"] == null) { return(lstHouse); } foreach (var room in resultJObject["content"]["roomInfos"]) { try { var housePrice = room["maxShowPrice"].ToObject <decimal>(); var lastPublishTime = GetPublishTime(room); string location = room["address"] != null ? room["address"].ToString() : room["title"].ToString(); var house = new BaseHouseInfo() { HouseLocation = location, HouseTitle = $"{room["title"].ToString()}【{room["subtitleNew"].ToString()}】", HouseOnlineURL = $"https://h5.mgzf.com/houseDetail/{room["roomId"].ToString()}", HouseText = room.ToString(), HousePrice = housePrice, DisPlayPrice = housePrice > 0 ? $"{housePrice}元" : "", Source = ConstConfigName.MoguHouse, LocationCityName = cityName, PicURLs = GetPhotos(room), PubTime = lastPublishTime, Status = 1, IsAnalyzed = true }; lstHouse.Add(house); } catch (Exception ex) { LogHelper.Error("Convert to House error", ex, room); } } return(lstHouse); }
private static Tuple <List <BaseHouseInfo>, string> GetHouseData(string cityName, string sequence) { List <BaseHouseInfo> lstHouse = new List <BaseHouseInfo>(); LogHelper.Debug($"city:{cityName},sequence:{sequence}"); var result = GetAPIResult(cityName, sequence); if (string.IsNullOrEmpty(result)) { return(Tuple.Create <List <BaseHouseInfo>, string>(lstHouse, "")); } var nextSequence = ""; var resultJObject = JsonConvert.DeserializeObject <JObject>(result); if (resultJObject["code"].ToString() == "0" && resultJObject["result"] != null) { if (resultJObject["result"]["has_next_page"].ToObject <bool>()) { nextSequence = resultJObject["result"]["sequence"].ToObject <string>(); } foreach (var item in resultJObject["result"]["items"]) { var room = item["room"]; var housePrice = room["cost1"].ToObject <decimal>(); var house = new BaseHouseInfo() { HouseLocation = room["address"].ToString(), HouseTitle = room["summary"].ToString(), HouseOnlineURL = $"http://www.zuber.im/app/room/{room["id"].ToString()}", HouseText = item.ToString(), HousePrice = housePrice, IsAnalyzed = true, DisPlayPrice = housePrice > 0 ? $"{housePrice}元" : "", Source = ConstConfigName.Zuber, LocationCityName = cityName, Status = 1, PicURLs = GetPhotos(room), PubTime = room["last_modify_time"].ToObject <DateTime>() }; lstHouse.Add(house); } } return(Tuple.Create <List <BaseHouseInfo>, string>(lstHouse, nextSequence)); }
public override List <BaseHouseInfo> ParseHouses(JToken config, string data) { var houseList = new List <BaseHouseInfo>(); if (string.IsNullOrEmpty(data)) { return(houseList); } var htmlDoc = htmlParser.Parse(data); var list = htmlDoc.QuerySelectorAll("div").Where(element => element.ClassList.Contains("cell-listings-rows-for-cards--row")); if (list == null || list.Count() == 0) { return(houseList); } foreach (var element in list) { string houseLocation = GetLocation(element); string houseTitle, disPlayPrice; decimal housePrice; FillTitlePrice(element, houseLocation, out houseTitle, out housePrice, out disPlayPrice); DateTime pubTime = GetPubtime(element); string onlineUrl = GetOnlineURL(element); var houseInfo = new BaseHouseInfo { HouseTitle = houseTitle, HouseOnlineURL = onlineUrl, DisPlayPrice = disPlayPrice, HouseLocation = houseLocation, Source = ConstConfigName.HKSpacious, HousePrice = housePrice, HouseText = element.InnerHtml, LocationCityName = "香港", PubTime = pubTime }; houseList.Add(houseInfo); } return(houseList); }
private static BaseHouseInfo ConvertHouse(string cityName, JToken item) { var room = item["room"]; var housePrice = room["cost1"].ToObject <decimal>(); var house = new BaseHouseInfo() { HouseLocation = room["address"].ToString(), HouseTitle = room["summary"].ToString(), HouseOnlineURL = $"http://www.zuber.im/app/room/{room["id"].ToString()}", HouseText = item.ToString(), HousePrice = housePrice, IsAnalyzed = true, DisPlayPrice = housePrice > 0 ? $"{housePrice}元" : "", Source = ConstConfigName.Zuber, LocationCityName = cityName, Status = 1, PicURLs = GetPhotos(room), PubTime = room["last_modify_time"].ToObject <DateTime>() }; return(house); }
private static List <BaseHouseInfo> GetDataFromHMTL(string cityName, string houseHTML) { List <BaseHouseInfo> houseList = new List <BaseHouseInfo>(); if (string.IsNullOrEmpty(houseHTML)) { return(houseList); } var htmlDoc = htmlParser.Parse(houseHTML); var houseItems = htmlDoc.QuerySelectorAll("div.pan-item.clearfix"); if (!houseItems.Any()) { return(houseList); } foreach (var item in houseItems) { int housePrice = GetHousePrice(item); string houseLocation = GetLocation(item); var titleItem = item.QuerySelector("h2"); var pubTime = GetPubTime(item); string houseURL = GetHouseURL(titleItem); var houseInfo = new BaseHouseInfo { HouseTitle = titleItem.TextContent.Replace("\n", "").Trim() + houseLocation, HouseOnlineURL = "http://zf.cdfgj.gov.cn" + houseURL, DisPlayPrice = housePrice + "元/月", HouseLocation = houseLocation, Source = ConstConfigName.Chengdufgj, HousePrice = housePrice, HouseText = item.InnerHtml, LocationCityName = cityName, PubTime = pubTime, PicURLs = GetPhotos(item) }; houseList.Add(houseInfo); } return(houseList); }
public static List <BaseHouseInfo> GetHouseData(string groupID, string cityName, int pageIndex) { List <BaseHouseInfo> lstHouseInfo = new List <BaseHouseInfo>(); var apiURL = $"https://api.douban.com/v2/group/{groupID}/topics?start={pageIndex * 50}&count=50"; LogHelper.Debug($"url:{apiURL},groupID:{groupID}, city:{cityName}"); var result = GetAPIResult(apiURL); if (string.IsNullOrEmpty(result)) { return(lstHouseInfo); } var resultJObject = JsonConvert.DeserializeObject <JObject>(result); foreach (var topic in resultJObject["topics"]) { var housePrice = JiebaTools.GetHousePrice(topic["content"].ToString()); var photos = topic["photos"]?.Select(photo => photo["alt"].ToString()).ToList(); var house = new BaseHouseInfo() { HouseLocation = topic["title"].ToString(), HouseTitle = topic["title"].ToString(), HouseOnlineURL = topic["share_url"].ToString(), HouseText = topic["content"].ToString(), HousePrice = housePrice, IsAnalyzed = true, DisPlayPrice = housePrice > 0 ? $"{housePrice}元" : "", Source = ConstConfigName.Douban, LocationCityName = cityName, Status = 1, PicURLs = JsonConvert.SerializeObject(photos), PubTime = topic["created"].ToObject <DateTime>() }; lstHouseInfo.Add(house); } return(lstHouseInfo); }
public override List <BaseHouseInfo> ParseHouses(JToken config, string data) { var cityName = config["cityname"]?.ToString(); List <BaseHouseInfo> houseList = new List <BaseHouseInfo>(); if (string.IsNullOrEmpty(data)) { return(houseList); } var htmlDoc = htmlParser.Parse(data); var houseItems = htmlDoc.QuerySelectorAll("div.pan-item.clearfix"); if (!houseItems.Any()) { return(houseList); } foreach (var item in houseItems) { BaseHouseInfo houseInfo = ConvertToHouse(cityName, item); houseList.Add(houseInfo); } return(houseList); }
private static BaseHouseInfo ConvertToHouse(string cityName, IElement item) { int housePrice = GetHousePrice(item); string houseLocation = GetLocation(item); var titleItem = item.QuerySelector("h2"); var pubTime = GetPubTime(item); string houseURL = GetHouseURL(titleItem); var houseInfo = new BaseHouseInfo { HouseTitle = titleItem.TextContent.Replace("\n", "").Trim() + houseLocation, HouseOnlineURL = "http://zf.cdfgj.gov.cn" + houseURL, DisPlayPrice = housePrice + "元/月", HouseLocation = houseLocation, Source = ConstConfigName.Chengdufgj, HousePrice = housePrice, HouseText = item.InnerHtml, LocationCityName = cityName, PubTime = pubTime, PicURLs = GetPhotos(item) }; return(houseInfo); }
public static List <BaseHouseInfo> GetHouseData(string citySortName, string cityID, string cityName, int pageIndex) { List <BaseHouseInfo> lstHouseInfo = new List <BaseHouseInfo>(); var apiURL = $"https://app.api.ke.com/Rentplat/v1/house/list?city_id={cityID}&offset={pageIndex * 30}&limit=30"; var result = GetAPIResult(apiURL, cityID); if (string.IsNullOrEmpty(result)) { return(lstHouseInfo); } var resultJObject = JsonConvert.DeserializeObject <JObject>(result); foreach (var rentHouse in resultJObject["data"]["list"]) { var housePrice = GetPrice(rentHouse); var house = new BaseHouseInfo() { HouseLocation = rentHouse["house_title"].ToString(), HouseTitle = rentHouse["house_title"].ToString() + rentHouse["desc"].ToString(), HouseOnlineURL = GetHouseURL(rentHouse, citySortName), HouseText = rentHouse.ToString(), HousePrice = housePrice, IsAnalyzed = true, DisPlayPrice = housePrice > 0 ? $"{housePrice}元" : "", Source = ConstConfigName.Beike, LocationCityName = cityName, Status = 1, PicURLs = JsonConvert.SerializeObject(new List <string>()), PubTime = DateTime.Now }; lstHouseInfo.Add(house); } return(lstHouseInfo); }
private static List <BaseHouseInfo> GetHouseDataFromHTML(string result) { var houseList = new List <BaseHouseInfo>(); if (string.IsNullOrEmpty(result)) { return(houseList); } var htmlDoc = htmlParser.Parse(result); var list = htmlDoc.QuerySelectorAll("div").Where(element => element.ClassList.Contains("cell-listings-rows-for-cards--row")); if (list == null || list.Count() == 0) { return(houseList); } foreach (var element in list) { string houseLocation = element.QuerySelectorAll("div") .Where(div => div.ClassList.Contains("carousel-inner")) .FirstOrDefault()? .QuerySelector("img")? .GetAttribute("alt"); string houseTitle = ""; decimal housePrice = 0; string disPlayPrice = ""; var titleItem = element.QuerySelectorAll("div") .Where(div => div.ClassList.Contains("cell-listing-carousel--infobox-text")).FirstOrDefault(); if (titleItem != null) { houseTitle = houseLocation + titleItem.TextContent; disPlayPrice = titleItem.FirstElementChild.TextContent; var textPrice = titleItem.FirstElementChild.TextContent.Replace("HKD$", "").Replace("萬", ""); decimal.TryParse(textPrice, out housePrice); //为了显示效果,此处乘以1000即可,本意应该是10000的 housePrice = housePrice * 1000; } var timeText = element.QuerySelectorAll("div") .Where(div => div.ClassList.Contains("cell--listings--row--card__info-contact-response_posted_time")) .FirstOrDefault()?.FirstElementChild.GetAttribute("datetime"); var pubTime = DateTime.Now; DateTime.TryParse(timeText, out pubTime); var onlineUrl = "https://www.spacious.hk" + element.QuerySelectorAll("a") .Where(item => item.ClassList.Contains("GTM-tracking-full-listing")) .FirstOrDefault()?.GetAttribute("href"); var houseInfo = new BaseHouseInfo { HouseTitle = houseTitle, HouseOnlineURL = onlineUrl, DisPlayPrice = disPlayPrice, HouseLocation = houseLocation, Source = ConstConfigName.HKSpacious, HousePrice = housePrice, HouseText = element.InnerHtml, LocationCityName = "香港", PubTime = pubTime }; houseList.Add(houseInfo); } return(houseList); }