public void Run()
{
foreach (var conf in configDapper.GetList(ConstConfigName.MoguHouse))
{
LogHelper.RunActionNotThrowEx(() =>
{
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
var confInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(conf.ConfigurationValue);
var cityName = confInfo.cityname.Value;
var cityId = (int)confInfo.cityid.Value;
// 2:合租 3:整租 5:业主房源
var rentTypes = new List <int>()
{
2, 3, 5
};
foreach (var rentType in rentTypes)
{
for (var pageIndex = 1; pageIndex <= confInfo.pagecount.Value; pageIndex++)
{
var list = GetHouseData(cityName, cityId, pageIndex, rentType);
houses.AddRange(list);
}
}
houseDapper.BulkInsertHouses(houses);
}, "MoGuHouseCrawler Run ", conf);
}
}
public void Run()
{
try
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
foreach (var config in configDapper.GetList(ConstConfigName.Beike))
{
LogHelper.RunActionNotThrowEx(() =>
{
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
var confInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(config.ConfigurationValue);
for (var pageIndex = 0; pageIndex < confInfo.pagecount.Value; pageIndex++)
{
var lstHouseInfo = GetHouseData(confInfo.citySortName.Value, confInfo.cityID.Value,
confInfo.cityName.Value, pageIndex);
houses.AddRange(lstHouseInfo);
}
captrueHouseCount = captrueHouseCount + houses.Count;
houseDapper.BulkInsertHouses(houses);
}, "BeikeHouseCrawler CaptureHouseInfo ", config);
}
LogHelper.Info($"BeikeHouseCrawler finish.本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。");
}
catch (Exception ex)
{
LogHelper.Error("BeikeHouseCrawler CrawlerHouseInfo Exception", ex);
}
}
public void Run()
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
var peopleRentingConf = configDapper.GetList(ConstConfigName.HuZhuZuFang)
.FirstOrDefault();
var pageCount = peopleRentingConf != null
? JsonConvert.DeserializeObject <dynamic>(peopleRentingConf.ConfigurationValue).pagecount.Value
: 10;
var hsHouseOnlineUrl = new HashSet <string>();
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
for (var pageNum = 1; pageNum < pageCount; pageNum++)
{
string result = getResultFromAPI(pageNum);
houses.AddRange(GetHouseData(result));
}
houseDapper.BulkInsertHouses(houses);
captrueHouseCount = captrueHouseCount + houses.Count;
LogHelper.Info($"PeopleRentingCrawler finish.本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。");
}
public void Run()
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
foreach (var crawlerConfiguration in configDapper.GetList(ConstConfigName.CCBHouse).ToList())
{
LogHelper.RunActionNotThrowEx(() =>
{
captrueHouseCount = captrueHouseCount + CaptureHouse(crawlerConfiguration);
}, "CapturPinPaiHouseInfo", crawlerConfiguration);
}
LogHelper.Info($"CCBHouesCrawler finish.本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。");
}
public void Run()
{
foreach (var doubanConf in configDapper.GetList(ConstConfigName.Zuber))
{
LogHelper.RunActionNotThrowEx(() =>
{
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
var confInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(doubanConf.ConfigurationValue);
var cityName = confInfo.cityname.Value;
var sequence = "";
for (var i = 0; i <= confInfo.pagecount.Value; i++)
{
var tupleResult = GetHouseData(cityName, sequence);
sequence = tupleResult.Item2;
houses.AddRange(tupleResult.Item1);
}
houseDapper.BulkInsertHouses(houses);
}, "DoubanHouseCrawler CaptureHouseInfo ", doubanConf);
}
}
public void Run()
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
foreach (var crawlerConfiguration in configDapper.GetList(ConstConfigName.Chengdufgj).ToList())
{
LogHelper.RunActionNotThrowEx(() =>
{
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
var confInfo = JsonConvert.DeserializeObject <dynamic>(crawlerConfiguration.ConfigurationValue);
for (var pageNum = 1; pageNum < confInfo.pagecount.Value; pageNum++)
{
var url = $"http://zf.cdfgj.gov.cn/{confInfo.path.Value}page={pageNum}";
var houseHTML = GetHouseHTML(url);
houses.AddRange(GetDataFromHMTL(confInfo.cityname.Value, houseHTML));
}
houseDapper.BulkInsertHouses(houses);
captrueHouseCount = captrueHouseCount + houses.Count;
}, "CapturBaiXing", crawlerConfiguration);
}
LogHelper.Info($"ChengduZufangCrawler finish.本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。");
}
public void Run()
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
foreach (var crawlerConfiguration in configDapper.GetList(ConstConfigName.PinPaiGongYu)
.Where(c => c.IsEnabled).ToList())
{
LogHelper.RunActionNotThrowEx(() =>
{
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
var confInfo = JsonConvert.DeserializeObject <dynamic>(crawlerConfiguration.ConfigurationValue);
for (var page = 0; page < confInfo.pagecount.Value; page++)
{
var jsonDate = GetDataFromAPI(confInfo.shortcutname.Value, page);
houses.AddRange(GetHouses(confInfo.shortcutname.Value, confInfo.cityname.Value, jsonDate));
}
houseDapper.BulkInsertHouses(houses);
captrueHouseCount = captrueHouseCount + houses.Count;
}, "CapturPinPaiHouseInfo", crawlerConfiguration);
}
LogHelper.Info($"PinPaiGongYuHouseCrawler finish.本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。");
}
