public static void CaptureHouseInfoFromConfig()
{
try
{
int captrueHouseCount = 0;
foreach (var doubanConf in dataContent.CrawlerConfigurations
.Where(c => c.ConfigurationName == ConstConfigurationName.Douban).ToList())
{
var confInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(doubanConf.ConfigurationValue);
for (var pageIndex = 0; pageIndex < confInfo.pagecount.Value; pageIndex++)
{
var lstHouseInfo = GetDataFromOnlineWeb(confInfo.groupid.Value, confInfo.cityname.Value, pageIndex);
dataContent.AddRange(lstHouseInfo);
dataContent.SaveChanges();
captrueHouseCount = captrueHouseCount + lstHouseInfo.Count;
}
}
HouseSourceInfo.RefreshHouseSourceInfo();
BizCrawlerLog.SaveLog("爬取豆瓣租房数据", $"本次共爬取到{captrueHouseCount}条数据。", 1);
}
catch (Exception ex)
{
LogHelper.Error("DoubanHouseCrawler CrawlerHouseInfo Exception", ex);
}
}
public static void SaveLog(string logTitle, string logContent, int logType)
{
BizCrawlerLog log = new BizCrawlerLog();
log.LogTitle = logTitle;
log.LogContent = logContent;
log.LogType = logType;
dataContent.Add(log);
dataContent.SaveChanges();
}