public static void CaptureHouseInfoFromConfig()
{
try
{
int captrueHouseCount = 0;
foreach (var doubanConf in DataContent.CrawlerConfigurations
.Where(c => c.ConfigurationName == ConstConfigurationName.Douban).ToList())
{
LogHelper.RunActionNotThrowEx(() =>
{
var confInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(doubanConf.ConfigurationValue);
for (var pageIndex = 0; pageIndex < confInfo.pagecount.Value; pageIndex++)
{
var lstHouseInfo = GetDataFromAPI(confInfo.groupid.Value,
confInfo.cityname.Value, pageIndex);
DataContent.DoubanHouseInfos.AddRange(lstHouseInfo);
DataContent.SaveChanges();
captrueHouseCount = captrueHouseCount + lstHouseInfo.Count;
}
}, "CaptureHouseInfoFromConfig", doubanConf);
}
HouseSourceInfo.RefreshHouseSourceInfo();
BizCrawlerLog.SaveLog("爬取豆瓣租房数据", $"本次共爬取到{captrueHouseCount}条数据。", 1);
}
catch (Exception ex)
{
LogHelper.Error("DoubanHouseCrawler CrawlerHouseInfo Exception", ex);
}
}
public static void CaptureHouseInfo()
{
try
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
foreach (var doubanConf in DataContent.CrawlerConfigurations
.Where(c => c.ConfigurationName == ConstConfigurationName.Douban).ToList())
{
LogHelper.RunActionNotThrowEx(() =>
{
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
var confInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(doubanConf.ConfigurationValue);
for (var pageIndex = 0; pageIndex < confInfo.pagecount.Value; pageIndex++)
{
var lstHouseInfo = GetHouseData(confInfo.groupid.Value,
confInfo.cityname.Value, pageIndex);
houses.AddRange(lstHouseInfo);
}
captrueHouseCount = captrueHouseCount + houses.Count;
CrawlerDataDapper.BulkInsertHouses(houses);
}, "DoubanHouseCrawler CaptureHouseInfo ", doubanConf);
}
BizCrawlerLog.SaveLog($"爬取豆瓣租房小组数据",
$"本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1);
}
catch (Exception ex)
{
LogHelper.Error("DoubanHouseCrawler CrawlerHouseInfo Exception", ex);
}
}
public static void CapturePinPaiHouseInfo()
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
foreach (var crawlerConfiguration in DataContent.CrawlerConfigurations.Where(c => c.ConfigurationName
== ConstConfigurationName.PinPaiGongYu && c.IsEnabled).ToList())
{
LogHelper.RunActionNotThrowEx(() =>
{
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
var confInfo = JsonConvert.DeserializeObject <dynamic>(crawlerConfiguration.ConfigurationValue);
for (var index = 0; index < confInfo.pagecount.Value; index++)
{
var url = $"http://{confInfo.shortcutname.Value}.58.com/pinpaigongyu/pn/{index}";
var houseHTML = GetHouseHTML(url);
houses.AddRange(GetDataFromHMTL(confInfo.shortcutname.Value, confInfo.cityname.Value, houseHTML));
}
CrawlerDataDapper.BulkInsertHouses(houses);
captrueHouseCount = captrueHouseCount + houses.Count;
}, "CapturPinPaiHouseInfo", crawlerConfiguration);
}
BizCrawlerLog.SaveLog($"爬取58品牌公寓租房数据",
$"本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1);
}
public static void Run()
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
var peopleRentingConf = CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.HuZhuZuFang)
.FirstOrDefault();
var pageCount = peopleRentingConf != null
? JsonConvert.DeserializeObject <dynamic>(peopleRentingConf.ConfigurationValue).pagecount.Value
: 10;
var hsHouseOnlineUrl = new HashSet <string>();
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
for (var pageNum = 1; pageNum < pageCount; pageNum++)
{
string result = getResultFromAPI(pageNum);
houses.AddRange(GetHouseData(result));
}
CrawlerDataDapper.BulkInsertHouses(houses);
captrueHouseCount = captrueHouseCount + houses.Count;
BizCrawlerLog.SaveLog($"爬取上海互助租房数据",
$"本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1);
}
public static void SaveLog(string logTitle, string logContent, int logType)
{
var log = new BizCrawlerLog
{
LogTitle = logTitle,
LogContent = logContent,
LogType = logType
};
}
public static void Run()
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
LogHelper.RunActionNotThrowEx(() =>
{
captrueHouseCount = captrueHouseCount + CaptureHouse();
}, "CapturPinPaiHouseInfo", "香港");
BizCrawlerLog.SaveLog($"爬取HKSpacious数据", $"本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1);
}
public static void SaveLog(string logTitle, string logContent, int logType)
{
BizCrawlerLog log = new BizCrawlerLog();
log.LogTitle = logTitle;
log.LogContent = logContent;
log.LogType = logType;
dataContent.Add(log);
dataContent.SaveChanges();
}
public static void SaveLog(string logTitle, string logContent, int logType)
{
var log = new BizCrawlerLog
{
LogTitle = logTitle,
LogContent = logContent,
LogType = logType
};
DataContent.Add(log);
DataContent.SaveChanges();
}
public static void Run()
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
foreach (var crawlerConfiguration in CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.CCBHouse).ToList())
{
LogHelper.RunActionNotThrowEx(() =>
{
captrueHouseCount = captrueHouseCount + CaptureHouse(crawlerConfiguration);
}, "CapturPinPaiHouseInfo", crawlerConfiguration);
}
BizCrawlerLog.SaveLog($"爬取CBBHouse数据",
$"本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1);
}