private static List <BizHouseInfo> GetDataFromOnlineWeb(string groupID, string cityName, int pageIndex)
{
List <BizHouseInfo> lstHouseInfo = new List <BizHouseInfo>();
var url = $"https://www.douban.com/group/{groupID}/discussion?start={pageIndex * 25}";
var htmlResult = DoubanHTTPHelper.GetHTMLForDouban(url);
if (string.IsNullOrEmpty(htmlResult))
{
return(lstHouseInfo);
}
var page = HtmlParser.Parse(htmlResult);
var tableElement = page.QuerySelector("table.olt");
if (tableElement == null)
{
return(lstHouseInfo);
}
foreach (var trItem in tableElement.QuerySelectorAll("tr"))
{
var titleItem = trItem.QuerySelector("td.title");
if (titleItem == null || DataContent.HouseInfos.Find(titleItem.QuerySelector("a").GetAttribute("href")) != null)
{
continue;
}
var houseTitle = titleItem.QuerySelector("a").GetAttribute("title");
var housePrice = JiebaTools.GetHousePrice(houseTitle);
var houseInfo = new BizHouseInfo()
{
HouseTitle = houseTitle,
HouseOnlineURL = titleItem.QuerySelector("a").GetAttribute("href"),
HouseLocation = houseTitle,
HouseText = houseTitle,
DataCreateTime = DateTime.Now,
PubTime = titleItem.QuerySelector("td.time") != null
? DateTime.Parse(DateTime.Now.ToString("yyyy-") + titleItem.QuerySelector("td.time").InnerHtml)
: DateTime.Now,
DisPlayPrice = housePrice > 0 ? $"{housePrice}元":"",
Source = ConstConfigurationName.Douban,
HousePrice = housePrice,
LocationCityName = cityName,
IsAnalyzed = housePrice > 0,
Status = housePrice > 0 ? 1 : 0,
};
lstHouseInfo.Add(houseInfo);
}
return(lstHouseInfo);
}
private static void AnalyzeFromWebPage(Web.Model.DBHouseInfo houseInfo,
ref decimal housePrice, ref string houseTextContent)
{
var htmlResult = DoubanHTTPHelper.GetHTMLForDouban(houseInfo.HouseOnlineURL);
//没有页面信息
if (string.IsNullOrEmpty(htmlResult))
{
//404页面
houseInfo.Status = 2;
}
else
{
var page = HtmlParser.Parse(htmlResult);
var topicContent = page.QuerySelector("div.topic-content");
//没有帖子内容
if (topicContent == null || topicContent.QuerySelector("p") == null || topicContent.QuerySelector("p") == null)
{
houseInfo.Status = 3;
}
else
{
//获取帖子内容
houseTextContent = topicContent.QuerySelector("p").TextContent;
//获取价格信息
housePrice = JiebaTools.GetHousePrice(houseTextContent);
if (housePrice != 0 || !string.IsNullOrEmpty(houseTextContent))
{
houseInfo.Status = 1;
}
houseInfo.DisPlayPrice = housePrice.ToString(CultureInfo.InvariantCulture);
houseInfo.HousePrice = housePrice;
houseInfo.HouseText = houseTextContent;
}
}
}