public static void Run()
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
var peopleRentingConf = CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.HuZhuZuFang)
.FirstOrDefault();
var pageCount = peopleRentingConf != null
? JsonConvert.DeserializeObject <dynamic>(peopleRentingConf.ConfigurationValue).pagecount.Value
: 10;
var hsHouseOnlineUrl = new HashSet <string>();
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
for (var pageNum = 1; pageNum < pageCount; pageNum++)
{
string result = getResultFromAPI(pageNum);
houses.AddRange(GetHouseData(result));
}
CrawlerDataDapper.BulkInsertHouses(houses);
captrueHouseCount = captrueHouseCount + houses.Count;
BizCrawlerLog.SaveLog($"爬取上海互助租房数据",
$"本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1);
}
public static void Run()
{
foreach (var doubanConf in CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.MoguHouse))
{
LogHelper.RunActionNotThrowEx(() =>
{
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
var confInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(doubanConf.ConfigurationValue);
var cityName = confInfo.cityname.Value;
var cityId = (int)confInfo.cityid.Value;
// 2:合租 3:整租 5:业主房源
var rentTypes = new List <int>()
{
2, 3, 5
};
foreach (var rentType in rentTypes)
{
for (var pageIndex = 1; pageIndex <= confInfo.pagecount.Value; pageIndex++)
{
var list = GetHouseData(cityName, cityId, pageIndex, rentType);
houses.AddRange(list);
}
}
CrawlerDataDapper.BulkInsertHouses(houses);
}, "MoGuHouseCrawler Run ", doubanConf);
}
}
public static void Run()
{
try
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
foreach (var doubanConf in CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.Douban))
{
LogHelper.RunActionNotThrowEx(() =>
{
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
var confInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(doubanConf.ConfigurationValue);
for (var pageIndex = 0; pageIndex < confInfo.pagecount.Value; pageIndex++)
{
var lstHouseInfo = GetHouseData(confInfo.groupid.Value,
confInfo.cityname.Value, pageIndex);
houses.AddRange(lstHouseInfo);
}
captrueHouseCount = captrueHouseCount + houses.Count;
CrawlerDataDapper.BulkInsertHouses(houses);
}, "DoubanHouseCrawler CaptureHouseInfo ", doubanConf);
}
BizCrawlerLog.SaveLog($"爬取豆瓣租房小组数据",
$"本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1);
}
catch (Exception ex)
{
LogHelper.Error("DoubanHouseCrawler CrawlerHouseInfo Exception", ex);
}
}
private static int CaptureHouse()
{
int captrueHouseCount = 0;
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
for (var pageNum = 1; pageNum < 20; pageNum++)
{
var result = GetHTML(pageNum);
houses.AddRange(GetHouseDataFromHTML(result));
}
captrueHouseCount = captrueHouseCount + houses.Count;
CrawlerDataDapper.BulkInsertHouses(houses);
return(captrueHouseCount);
}
public static void Run()
{
foreach (var doubanConf in CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.Zuber))
{
LogHelper.RunActionNotThrowEx(() =>
{
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
var confInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(doubanConf.ConfigurationValue);
var cityName = confInfo.cityname.Value;
var sequence = "";
for (var i = 0; i <= confInfo.pagecount.Value; i++)
{
var tupleResult = GetHouseData(cityName, sequence);
sequence = tupleResult.Item2;
houses.AddRange(tupleResult.Item1);
}
CrawlerDataDapper.BulkInsertHouses(houses);
}, "DoubanHouseCrawler CaptureHouseInfo ", doubanConf);
}
}
private static int CaptureHouse(BizCrawlerConfiguration crawlerConfiguration)
{
var confInfo = JsonConvert.DeserializeObject <dynamic>(crawlerConfiguration.ConfigurationValue);
if (confInfo.shortcutname == null || string.IsNullOrEmpty(confInfo.shortcutname.Value))
{
return(0);
}
int captrueHouseCount = 0;
string cityShortCutName = confInfo.shortcutname.Value;
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
for (var pageNum = 1; pageNum < confInfo.pagecount.Value; pageNum++)
{
var result = GetResultByAPI(cityShortCutName, pageNum);
houses.AddRange(GetHouseData(cityShortCutName, result));
}
captrueHouseCount = captrueHouseCount + houses.Count;
CrawlerDataDapper.BulkInsertHouses(houses);
return(captrueHouseCount);
}