private static List <BaseHouseInfo> GetHouseData(string result)
{
List <BaseHouseInfo> houseList = new List <BaseHouseInfo>();
var resultJObject = JsonConvert.DeserializeObject <JObject>(result);
if (resultJObject == null || resultJObject["head"] == null || !resultJObject["head"]["success"].ToObject <Boolean>())
{
return(houseList);
}
foreach (var item in resultJObject["houseList"])
{
BaseHouseInfo houseInfo = new BaseHouseInfo();
var houseDesc = item["houseDescript"].ToObject <string>().Replace("😄", "");
var houseURL = $"http://www.huzhumaifang.com/Renting/house_detail/id/{item["houseId"]}.html";
houseInfo.HouseOnlineURL = houseURL;
houseInfo.HouseTitle = houseDesc;
houseInfo.HouseLocation = houseDesc;
houseInfo.HouseText = item.ToString();
houseInfo.HousePrice = item["houseRentPrice"].ToObject <Int32>();
houseInfo.DisPlayPrice = item["houseRentPrice"].ToString();
houseInfo.LocationCityName = "上海";
houseInfo.PubTime = item["houseCreateTime"].ToObject <DateTime>();
houseInfo.PicURLs = item["bigPicUrls"].ToString();
houseInfo.Source = ConstConfigName.HuZhuZuFang;
houseList.Add(houseInfo);
}
return(houseList);
}
private static List <BaseHouseInfo> GetHouseData(string cityShortCutName, string result)
{
var houseList = new List <BaseHouseInfo>();
if (string.IsNullOrEmpty(result))
{
return(houseList);
}
var resultJObject = JsonConvert.DeserializeObject <JObject>(result);
foreach (var item in resultJObject["items"])
{
BaseHouseInfo houseInfo = new BaseHouseInfo();
string houseURL = GetHouseOnlineURL(cityShortCutName, item);
houseInfo.HouseOnlineURL = houseURL;
houseInfo.HouseLocation = item["headline"].ToObject <string>();
houseInfo.HouseTitle = item["headline"].ToObject <string>();
houseInfo.LocationCityName = item["cityName"].ToObject <string>();
houseInfo.HouseText = item.ToString();
houseInfo.HousePrice = item["totalPrice"].ToObject <Int32>();
houseInfo.DisPlayPrice = item["totalPrice"].ToString();
houseInfo.PubTime = item["publishTime"].ToObject <DateTime>();
houseInfo.Source = ConstConfigName.CCBHouse;
houseList.Add(houseInfo);
}
return(houseList);
}
public override List <BaseHouseInfo> ParseHouses(JToken config, string data)
{
var lstHouseInfo = new List <BaseHouseInfo>();
if (string.IsNullOrEmpty(data))
{
return(lstHouseInfo);
}
var cityName = config["cityName"]?.ToString();
var citySortName = config["citySortName"]?.ToString();
var resultJObject = JsonConvert.DeserializeObject <JObject>(data);
foreach (var rentHouse in resultJObject["data"]["list"])
{
var housePrice = GetPrice(rentHouse);
var house = new BaseHouseInfo()
{
HouseLocation = rentHouse["house_title"].ToString(),
HouseTitle = rentHouse["house_title"].ToString() + rentHouse["desc"].ToString(),
HouseOnlineURL = GetHouseURL(rentHouse, citySortName),
HouseText = rentHouse.ToString(),
HousePrice = housePrice,
IsAnalyzed = true,
DisPlayPrice = housePrice > 0 ? $"{housePrice}元" : "",
Source = ConstConfigName.Beike,
LocationCityName = cityName,
Status = 1,
PicURLs = JsonConvert.SerializeObject(new List <string>()),
PubTime = DateTime.Now
};
lstHouseInfo.Add(house);
}
return(lstHouseInfo);
}
public override List <BaseHouseInfo> ParseHouses(JToken config, string data)
{
var houses = new List <BaseHouseInfo>();
var resultJObject = JsonConvert.DeserializeObject <JObject>(data);
var cityName = config["cityname"].ToString();
foreach (var topic in resultJObject["topics"])
{
var housePrice = JiebaTools.GetHousePrice(topic["content"].ToString());
var photos = topic["photos"]?.Select(photo => photo["alt"].ToString()).ToList();
var house = new BaseHouseInfo()
{
HouseLocation = topic["title"].ToString(),
HouseTitle = topic["title"].ToString(),
HouseOnlineURL = topic["share_url"].ToString(),
HouseText = topic["content"].ToString(),
HousePrice = housePrice,
IsAnalyzed = true,
DisPlayPrice = housePrice > 0 ? $"{housePrice}元" : "",
Source = ConstConfigName.Douban,
LocationCityName = cityName,
Status = 1,
PicURLs = JsonConvert.SerializeObject(photos),
PubTime = topic["created"].ToObject <DateTime>()
};
houses.Add(house);
}
return(houses);
}
private static BaseHouseInfo ConvertToHouse(string shortCutName, string cityName, JToken info)
{
var onlineUrl = $"https://{shortCutName}.58.com/pinpaigongyu/{info["infoID"].ToString()}x.shtml";
var housePrice = decimal.Parse(info["minPrice"].ToString());
var houseInfo = new BaseHouseInfo
{
HouseTitle = $"{info["title"].ToString()}-{info["layout"].ToString()}",
HouseOnlineURL = onlineUrl,
DisPlayPrice = info["priceTitle"].ToString(),
HouseLocation = GetHouseLocation(info),
Source = ConstConfigName.PinPaiGongYu,
HousePrice = housePrice,
HouseText = info.ToString(),
LocationCityName = cityName,
PubTime = new DateTime(info["postDate"]["year"].ToObject <int>(),
info["postDate"]["mon"].ToObject <int>(),
info["postDate"]["mday"].ToObject <int>(),
info["postDate"]["hours"].ToObject <int>(),
info["postDate"]["minutes"].ToObject <int>(),
info["postDate"]["seconds"].ToObject <int>()),
PicURLs = info["picsUrl"].ToString()
};
return(houseInfo);
}
private static List <BaseHouseInfo> GetHouses(string shortCutName, string cityName, string json)
{
List <BaseHouseInfo> houseList = new List <BaseHouseInfo>();
var resultJObject = JsonConvert.DeserializeObject <JObject>(json);
foreach (var info in resultJObject["result"]["getListInfo"]["infolist"])
{
var onlineUrl = $"https://{shortCutName}.58.com/pinpaigongyu/{info["infoID"].ToString()}x.shtml";
var housePrice = decimal.Parse(info["minPrice"].ToString());
var houseInfo = new BaseHouseInfo
{
HouseTitle = $"{info["title"].ToString()}-{info["layout"].ToString()}",
HouseOnlineURL = onlineUrl,
DisPlayPrice = info["priceTitle"].ToString(),
HouseLocation = GetHouseLocation(info),
Source = ConstConfigName.PinPaiGongYu,
HousePrice = housePrice,
HouseText = info.ToString(),
LocationCityName = cityName,
PubTime = new DateTime(info["postDate"]["year"].ToObject <int>(),
info["postDate"]["mon"].ToObject <int>(),
info["postDate"]["mday"].ToObject <int>(),
info["postDate"]["hours"].ToObject <int>(),
info["postDate"]["minutes"].ToObject <int>(),
info["postDate"]["seconds"].ToObject <int>()),
PicURLs = info["picsUrl"].ToString()
};
houseList.Add(houseInfo);
}
return(houseList);
}
public override List <BaseHouseInfo> ParseHouses(JToken config, string jsonOrHTML)
{
var houseList = new List <BaseHouseInfo>();
if (string.IsNullOrEmpty(jsonOrHTML))
{
return(houseList);
}
var htmlDoc = htmlParser.Parse(jsonOrHTML);
var houseItems = htmlDoc.QuerySelectorAll("li.listing-ad.item-regular");
if (!houseItems.Any())
{
return(houseList);
}
var cityName = config["cityname"]?.ToString();
foreach (var item in houseItems)
{
var element = item.QuerySelector("div.media-body");
if (element == null)
{
continue;
}
var disPlayPrice = element.QuerySelector("span.highlight").TextContent;
int.TryParse(disPlayPrice.Replace("元", ""), out var housePrice);
var adTitle = element.QuerySelector("a.ad-title");
var detailList = element.QuerySelectorAll("div.ad-item-detail");
var descTitle = detailList[0];
var localTitle = detailList[1];
if (localTitle.QuerySelector("a.source") != null)
{
localTitle.QuerySelector("a.source").Remove();
}
if (localTitle.QuerySelector("time") != null)
{
localTitle.QuerySelector("time").Remove();
}
var houseLocation = localTitle.TextContent.Replace("来自：", "").Trim();
var houseInfo = new BaseHouseInfo
{
HouseTitle = adTitle.TextContent + houseLocation,
HouseOnlineURL = adTitle.GetAttribute("href"),
DisPlayPrice = disPlayPrice,
HouseLocation = houseLocation,
Source = ConstConfigName.BaiXing,
HousePrice = housePrice,
HouseText = element.InnerHtml,
LocationCityName = cityName,
PubTime = DateTime.Now,
PicURLs = GetPhotos(item)
};
houseList.Add(houseInfo);
}
return(houseList);
}
public static List <BaseHouseInfo> GetHouseData(string cityName, int cityID, int currentPage, int rentTypes = 2)
{
List <BaseHouseInfo> lstHouse = new List <BaseHouseInfo>();
var result = GetAPIResult(cityID, currentPage, rentTypes);
if (string.IsNullOrEmpty(result))
{
return(lstHouse);
}
var resultJObject = JsonConvert.DeserializeObject <JObject>(result);
if (resultJObject["message"].ToString() != "success" ||
resultJObject["content"] == null ||
resultJObject["content"]["roomInfos"] == null)
{
return(lstHouse);
}
foreach (var room in resultJObject["content"]["roomInfos"])
{
try
{
var housePrice = room["maxShowPrice"].ToObject <decimal>();
var lastPublishTime = GetPublishTime(room);
string location = room["address"] != null ? room["address"].ToString() : room["title"].ToString();
var house = new BaseHouseInfo()
{
HouseLocation = location,
HouseTitle = $"{room["title"].ToString()}【{room["subtitleNew"].ToString()}】",
HouseOnlineURL = $"https://h5.mgzf.com/houseDetail/{room["roomId"].ToString()}",
HouseText = room.ToString(),
HousePrice = housePrice,
DisPlayPrice = housePrice > 0 ? $"{housePrice}元" : "",
Source = ConstConfigName.MoguHouse,
LocationCityName = cityName,
PicURLs = GetPhotos(room),
PubTime = lastPublishTime,
Status = 1,
IsAnalyzed = true
};
lstHouse.Add(house);
}
catch (Exception ex)
{
LogHelper.Error("Convert to House error", ex, room);
}
}
return(lstHouse);
}
private static Tuple <List <BaseHouseInfo>, string> GetHouseData(string cityName, string sequence)
{
List <BaseHouseInfo> lstHouse = new List <BaseHouseInfo>();
LogHelper.Debug($"city:{cityName},sequence:{sequence}");
var result = GetAPIResult(cityName, sequence);
if (string.IsNullOrEmpty(result))
{
return(Tuple.Create <List <BaseHouseInfo>, string>(lstHouse, ""));
}
var nextSequence = "";
var resultJObject = JsonConvert.DeserializeObject <JObject>(result);
if (resultJObject["code"].ToString() == "0" && resultJObject["result"] != null)
{
if (resultJObject["result"]["has_next_page"].ToObject <bool>())
{
nextSequence = resultJObject["result"]["sequence"].ToObject <string>();
}
foreach (var item in resultJObject["result"]["items"])
{
var room = item["room"];
var housePrice = room["cost1"].ToObject <decimal>();
var house = new BaseHouseInfo()
{
HouseLocation = room["address"].ToString(),
HouseTitle = room["summary"].ToString(),
HouseOnlineURL = $"http://www.zuber.im/app/room/{room["id"].ToString()}",
HouseText = item.ToString(),
HousePrice = housePrice,
IsAnalyzed = true,
DisPlayPrice = housePrice > 0 ? $"{housePrice}元" : "",
Source = ConstConfigName.Zuber,
LocationCityName = cityName,
Status = 1,
PicURLs = GetPhotos(room),
PubTime = room["last_modify_time"].ToObject <DateTime>()
};
lstHouse.Add(house);
}
}
return(Tuple.Create <List <BaseHouseInfo>, string>(lstHouse, nextSequence));
}
public override List <BaseHouseInfo> ParseHouses(JToken config, string data)
{
var houseList = new List <BaseHouseInfo>();
if (string.IsNullOrEmpty(data))
{
return(houseList);
}
var htmlDoc = htmlParser.Parse(data);
var list = htmlDoc.QuerySelectorAll("div").Where(element => element.ClassList.Contains("cell-listings-rows-for-cards--row"));
if (list == null || list.Count() == 0)
{
return(houseList);
}
foreach (var element in list)
{
string houseLocation = GetLocation(element);
string houseTitle, disPlayPrice;
decimal housePrice;
FillTitlePrice(element, houseLocation, out houseTitle, out housePrice, out disPlayPrice);
DateTime pubTime = GetPubtime(element);
string onlineUrl = GetOnlineURL(element);
var houseInfo = new BaseHouseInfo
{
HouseTitle = houseTitle,
HouseOnlineURL = onlineUrl,
DisPlayPrice = disPlayPrice,
HouseLocation = houseLocation,
Source = ConstConfigName.HKSpacious,
HousePrice = housePrice,
HouseText = element.InnerHtml,
LocationCityName = "香港",
PubTime = pubTime
};
houseList.Add(houseInfo);
}
return(houseList);
}
private static BaseHouseInfo ConvertHouse(string cityName, JToken item)
{
var room = item["room"];
var housePrice = room["cost1"].ToObject <decimal>();
var house = new BaseHouseInfo()
{
HouseLocation = room["address"].ToString(),
HouseTitle = room["summary"].ToString(),
HouseOnlineURL = $"http://www.zuber.im/app/room/{room["id"].ToString()}",
HouseText = item.ToString(),
HousePrice = housePrice,
IsAnalyzed = true,
DisPlayPrice = housePrice > 0 ? $"{housePrice}元" : "",
Source = ConstConfigName.Zuber,
LocationCityName = cityName,
Status = 1,
PicURLs = GetPhotos(room),
PubTime = room["last_modify_time"].ToObject <DateTime>()
};
return(house);
}
private static List <BaseHouseInfo> GetDataFromHMTL(string cityName, string houseHTML)
{
List <BaseHouseInfo> houseList = new List <BaseHouseInfo>();
if (string.IsNullOrEmpty(houseHTML))
{
return(houseList);
}
var htmlDoc = htmlParser.Parse(houseHTML);
var houseItems = htmlDoc.QuerySelectorAll("div.pan-item.clearfix");
if (!houseItems.Any())
{
return(houseList);
}
foreach (var item in houseItems)
{
int housePrice = GetHousePrice(item);
string houseLocation = GetLocation(item);
var titleItem = item.QuerySelector("h2");
var pubTime = GetPubTime(item);
string houseURL = GetHouseURL(titleItem);
var houseInfo = new BaseHouseInfo
{
HouseTitle = titleItem.TextContent.Replace("\n", "").Trim() + houseLocation,
HouseOnlineURL = "http://zf.cdfgj.gov.cn" + houseURL,
DisPlayPrice = housePrice + "元/月",
HouseLocation = houseLocation,
Source = ConstConfigName.Chengdufgj,
HousePrice = housePrice,
HouseText = item.InnerHtml,
LocationCityName = cityName,
PubTime = pubTime,
PicURLs = GetPhotos(item)
};
houseList.Add(houseInfo);
}
return(houseList);
}
public static List <BaseHouseInfo> GetHouseData(string groupID, string cityName, int pageIndex)
{
List <BaseHouseInfo> lstHouseInfo = new List <BaseHouseInfo>();
var apiURL = $"https://api.douban.com/v2/group/{groupID}/topics?start={pageIndex * 50}&count=50";
LogHelper.Debug($"url:{apiURL},groupID:{groupID}, city:{cityName}");
var result = GetAPIResult(apiURL);
if (string.IsNullOrEmpty(result))
{
return(lstHouseInfo);
}
var resultJObject = JsonConvert.DeserializeObject <JObject>(result);
foreach (var topic in resultJObject["topics"])
{
var housePrice = JiebaTools.GetHousePrice(topic["content"].ToString());
var photos = topic["photos"]?.Select(photo => photo["alt"].ToString()).ToList();
var house = new BaseHouseInfo()
{
HouseLocation = topic["title"].ToString(),
HouseTitle = topic["title"].ToString(),
HouseOnlineURL = topic["share_url"].ToString(),
HouseText = topic["content"].ToString(),
HousePrice = housePrice,
IsAnalyzed = true,
DisPlayPrice = housePrice > 0 ? $"{housePrice}元" : "",
Source = ConstConfigName.Douban,
LocationCityName = cityName,
Status = 1,
PicURLs = JsonConvert.SerializeObject(photos),
PubTime = topic["created"].ToObject <DateTime>()
};
lstHouseInfo.Add(house);
}
return(lstHouseInfo);
}
public override List <BaseHouseInfo> ParseHouses(JToken config, string data)
{
var cityName = config["cityname"]?.ToString();
List <BaseHouseInfo> houseList = new List <BaseHouseInfo>();
if (string.IsNullOrEmpty(data))
{
return(houseList);
}
var htmlDoc = htmlParser.Parse(data);
var houseItems = htmlDoc.QuerySelectorAll("div.pan-item.clearfix");
if (!houseItems.Any())
{
return(houseList);
}
foreach (var item in houseItems)
{
BaseHouseInfo houseInfo = ConvertToHouse(cityName, item);
houseList.Add(houseInfo);
}
return(houseList);
}
private static BaseHouseInfo ConvertToHouse(string cityName, IElement item)
{
int housePrice = GetHousePrice(item);
string houseLocation = GetLocation(item);
var titleItem = item.QuerySelector("h2");
var pubTime = GetPubTime(item);
string houseURL = GetHouseURL(titleItem);
var houseInfo = new BaseHouseInfo
{
HouseTitle = titleItem.TextContent.Replace("\n", "").Trim() + houseLocation,
HouseOnlineURL = "http://zf.cdfgj.gov.cn" + houseURL,
DisPlayPrice = housePrice + "元/月",
HouseLocation = houseLocation,
Source = ConstConfigName.Chengdufgj,
HousePrice = housePrice,
HouseText = item.InnerHtml,
LocationCityName = cityName,
PubTime = pubTime,
PicURLs = GetPhotos(item)
};
return(houseInfo);
}
public static List <BaseHouseInfo> GetHouseData(string citySortName, string cityID, string cityName, int pageIndex)
{
List <BaseHouseInfo> lstHouseInfo = new List <BaseHouseInfo>();
var apiURL = $"https://app.api.ke.com/Rentplat/v1/house/list?city_id={cityID}&offset={pageIndex * 30}&limit=30";
var result = GetAPIResult(apiURL, cityID);
if (string.IsNullOrEmpty(result))
{
return(lstHouseInfo);
}
var resultJObject = JsonConvert.DeserializeObject <JObject>(result);
foreach (var rentHouse in resultJObject["data"]["list"])
{
var housePrice = GetPrice(rentHouse);
var house = new BaseHouseInfo()
{
HouseLocation = rentHouse["house_title"].ToString(),
HouseTitle = rentHouse["house_title"].ToString() + rentHouse["desc"].ToString(),
HouseOnlineURL = GetHouseURL(rentHouse, citySortName),
HouseText = rentHouse.ToString(),
HousePrice = housePrice,
IsAnalyzed = true,
DisPlayPrice = housePrice > 0 ? $"{housePrice}元" : "",
Source = ConstConfigName.Beike,
LocationCityName = cityName,
Status = 1,
PicURLs = JsonConvert.SerializeObject(new List <string>()),
PubTime = DateTime.Now
};
lstHouseInfo.Add(house);
}
return(lstHouseInfo);
}
private static List <BaseHouseInfo> GetHouseDataFromHTML(string result)
{
var houseList = new List <BaseHouseInfo>();
if (string.IsNullOrEmpty(result))
{
return(houseList);
}
var htmlDoc = htmlParser.Parse(result);
var list = htmlDoc.QuerySelectorAll("div").Where(element => element.ClassList.Contains("cell-listings-rows-for-cards--row"));
if (list == null || list.Count() == 0)
{
return(houseList);
}
foreach (var element in list)
{
string houseLocation = element.QuerySelectorAll("div")
.Where(div => div.ClassList.Contains("carousel-inner"))
.FirstOrDefault()?
.QuerySelector("img")?
.GetAttribute("alt");
string houseTitle = "";
decimal housePrice = 0;
string disPlayPrice = "";
var titleItem = element.QuerySelectorAll("div")
.Where(div => div.ClassList.Contains("cell-listing-carousel--infobox-text")).FirstOrDefault();
if (titleItem != null)
{
houseTitle = houseLocation + titleItem.TextContent;
disPlayPrice = titleItem.FirstElementChild.TextContent;
var textPrice = titleItem.FirstElementChild.TextContent.Replace("HKD$", "").Replace("萬", "");
decimal.TryParse(textPrice, out housePrice);
//为了显示效果,此处乘以1000即可,本意应该是10000的
housePrice = housePrice * 1000;
}
var timeText = element.QuerySelectorAll("div")
.Where(div => div.ClassList.Contains("cell--listings--row--card__info-contact-response_posted_time"))
.FirstOrDefault()?.FirstElementChild.GetAttribute("datetime");
var pubTime = DateTime.Now;
DateTime.TryParse(timeText, out pubTime);
var onlineUrl = "https://www.spacious.hk" + element.QuerySelectorAll("a")
.Where(item => item.ClassList.Contains("GTM-tracking-full-listing"))
.FirstOrDefault()?.GetAttribute("href");
var houseInfo = new BaseHouseInfo
{
HouseTitle = houseTitle,
HouseOnlineURL = onlineUrl,
DisPlayPrice = disPlayPrice,
HouseLocation = houseLocation,
Source = ConstConfigName.HKSpacious,
HousePrice = housePrice,
HouseText = element.InnerHtml,
LocationCityName = "香港",
PubTime = pubTime
};
houseList.Add(houseInfo);
}
return(houseList);
}
