public static void Run()
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
foreach (var crawlerConfiguration in CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.PinPaiGongYu).Where(c => c.IsEnabled).ToList())
{
LogHelper.RunActionNotThrowEx(() =>
{
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
var confInfo = JsonConvert.DeserializeObject <dynamic>(crawlerConfiguration.ConfigurationValue);
for (var index = 0; index < confInfo.pagecount.Value; index++)
{
var url = $"http://{confInfo.shortcutname.Value}.58.com/pinpaigongyu/pn/{index}";
var houseHTML = GetHouseHTML(url);
houses.AddRange(GetDataFromHMTL(confInfo.shortcutname.Value, confInfo.cityname.Value, houseHTML));
}
CrawlerDataDapper.BulkInsertHouses(houses);
captrueHouseCount = captrueHouseCount + houses.Count;
}, "CapturPinPaiHouseInfo", crawlerConfiguration);
}
BizCrawlerLog.SaveLog($"爬取58品牌公寓租房数据",
$"本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1);
}
public static void Run()
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
var peopleRentingConf = CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.HuZhuZuFang)
.FirstOrDefault();
var pageCount = peopleRentingConf != null
? JsonConvert.DeserializeObject <dynamic>(peopleRentingConf.ConfigurationValue).pagecount.Value
: 10;
var hsHouseOnlineUrl = new HashSet <string>();
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
for (var pageNum = 1; pageNum < pageCount; pageNum++)
{
string result = getResultFromAPI(pageNum);
houses.AddRange(GetHouseData(result));
}
CrawlerDataDapper.BulkInsertHouses(houses);
captrueHouseCount = captrueHouseCount + houses.Count;
BizCrawlerLog.SaveLog($"爬取上海互助租房数据",
$"本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1);
}
public static void Run()
{
foreach (var doubanConf in CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.MoguHouse))
{
LogHelper.RunActionNotThrowEx(() =>
{
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
var confInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(doubanConf.ConfigurationValue);
var cityName = confInfo.cityname.Value;
var cityId = (int)confInfo.cityid.Value;
// 2:合租 3:整租 5:业主房源
var rentTypes = new List <int>()
{
2, 3, 5
};
foreach (var rentType in rentTypes)
{
for (var pageIndex = 1; pageIndex <= confInfo.pagecount.Value; pageIndex++)
{
var list = GetHouseData(cityName, cityId, pageIndex, rentType);
houses.AddRange(list);
}
}
CrawlerDataDapper.BulkInsertHouses(houses);
}, "MoGuHouseCrawler Run ", doubanConf);
}
}
public static void Run()
{
try
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
foreach (var doubanConf in CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.Douban))
{
LogHelper.RunActionNotThrowEx(() =>
{
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
var confInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(doubanConf.ConfigurationValue);
for (var pageIndex = 0; pageIndex < confInfo.pagecount.Value; pageIndex++)
{
var lstHouseInfo = GetHouseData(confInfo.groupid.Value,
confInfo.cityname.Value, pageIndex);
houses.AddRange(lstHouseInfo);
}
captrueHouseCount = captrueHouseCount + houses.Count;
CrawlerDataDapper.BulkInsertHouses(houses);
}, "DoubanHouseCrawler CaptureHouseInfo ", doubanConf);
}
BizCrawlerLog.SaveLog($"爬取豆瓣租房小组数据",
$"本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1);
}
catch (Exception ex)
{
LogHelper.Error("DoubanHouseCrawler CrawlerHouseInfo Exception", ex);
}
}
public static void Run()
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
foreach (var crawlerConfiguration in CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.CCBHouse).ToList())
{
LogHelper.RunActionNotThrowEx(() =>
{
captrueHouseCount = captrueHouseCount + CaptureHouse(crawlerConfiguration);
}, "CapturPinPaiHouseInfo", crawlerConfiguration);
}
BizCrawlerLog.SaveLog($"爬取CBBHouse数据",
$"本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1);
}
public static void Run()
{
foreach (var doubanConf in CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.Zuber))
{
LogHelper.RunActionNotThrowEx(() =>
{
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
var confInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(doubanConf.ConfigurationValue);
var cityName = confInfo.cityname.Value;
var sequence = "";
for (var i = 0; i <= confInfo.pagecount.Value; i++)
{
var tupleResult = GetHouseData(cityName, sequence);
sequence = tupleResult.Item2;
houses.AddRange(tupleResult.Item1);
}
CrawlerDataDapper.BulkInsertHouses(houses);
}, "DoubanHouseCrawler CaptureHouseInfo ", doubanConf);
}
}
