public IActionResult GetHouseInfo(string cityName, string source = "", int houseCount = 100,
int intervalDay = 7, string keyword = "", bool refresh = false)
{
try
{
var houseList = CrawlerDataDapper.SearchHouseInfo(cityName, source, houseCount, intervalDay, keyword, refresh);
var rooms = houseList.Select(house =>
{
var markBGType = string.Empty;
int housePrice = (int)house.HousePrice;
if (housePrice > 0)
{
markBGType = LocationMarkBGType.SelectColor(housePrice / 1000);
}
return(new HouseInfo
{
Money = house.DisPlayPrice,
HouseURL = house.HouseOnlineURL,
HouseLocation = house.HouseLocation,
HouseTime = house.PubTime.ToString(),
HousePrice = housePrice,
LocationMarkBG = markBGType,
DisplaySource = ConstConfigurationName.ConvertToDisPlayName(house.Source)
});
});
return(Json(new { IsSuccess = true, HouseInfos = rooms }));
}
catch (Exception ex)
{
return(Json(new { IsSuccess = false, error = ex.ToString() }));
}
}
public void Run()
{
var today = DateTime.Now.ToLocalTime();
EmailInfo email = new EmailInfo();
email.Receiver = configuration.ReceiverAddress;
email.ReceiverName = configuration.ReceiverName;
email.Subject = $"地图搜租房每日数据汇总({today.ToString("yyyy-MM-dd")})";
var statList = CrawlerDataDapper.GetHouseStatList();
string bodyHTML = @"<table border='1' cellpadding='0' cellspacing='0' width='100%'>
<tr>
<td>来源</td>
<td>数量</td>
<td>最晚发布时间</td>
<td>最晚入库时间</td>
</tr>";
foreach (var stat in statList)
{
bodyHTML = bodyHTML + $" <tr> <td>{stat.Source}</td><td>{stat.HouseSum}</td><td>{stat.LastPubTime}</td> <td>{stat.LastCreateTime}</td> </tr>";
}
bodyHTML = bodyHTML + $" <tr> <td>共计</td><td>{statList.Sum(s => s.HouseSum)}</td> </tr>";
bodyHTML = bodyHTML + " </table>";
email.Body = bodyHTML;
emailService.Send(email);
}
public static void CapturePinPaiHouseInfo()
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
foreach (var crawlerConfiguration in DataContent.CrawlerConfigurations.Where(c => c.ConfigurationName
== ConstConfigurationName.PinPaiGongYu && c.IsEnabled).ToList())
{
LogHelper.RunActionNotThrowEx(() =>
{
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
var confInfo = JsonConvert.DeserializeObject <dynamic>(crawlerConfiguration.ConfigurationValue);
for (var index = 0; index < confInfo.pagecount.Value; index++)
{
var url = $"http://{confInfo.shortcutname.Value}.58.com/pinpaigongyu/pn/{index}";
var houseHTML = GetHouseHTML(url);
houses.AddRange(GetDataFromHMTL(confInfo.shortcutname.Value, confInfo.cityname.Value, houseHTML));
}
CrawlerDataDapper.BulkInsertHouses(houses);
captrueHouseCount = captrueHouseCount + houses.Count;
}, "CapturPinPaiHouseInfo", crawlerConfiguration);
}
BizCrawlerLog.SaveLog($"爬取58品牌公寓租房数据",
$"本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1);
}
public static void RefreshDashboard()
{
var lstCityHouse = CrawlerDataDapper.GetHouseSourceInfoList();
var config = DataContent.CrawlerConfigurations.FirstOrDefault(c => c.ConfigurationName == ConstConfigurationName.CityHouseInfo);
if (config == null)
{
config = new BizCrawlerConfiguration
{
ConfigurationKey = 0,
ConfigurationName = ConstConfigurationName.CityHouseInfo,
ConfigurationValue = Newtonsoft.Json.JsonConvert.SerializeObject(lstCityHouse),
IsEnabled = true
};
DataContent.Add(config);
DataContent.SaveChanges();
}
else
{
config.ConfigurationKey = 0;
config.ConfigurationName = ConstConfigurationName.CityHouseInfo;
config.ConfigurationValue = Newtonsoft.Json.JsonConvert.SerializeObject(lstCityHouse);
config.IsEnabled = true;
DataContent.SaveChanges();
}
}
public static void CaptureHouseInfo()
{
try
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
foreach (var doubanConf in DataContent.CrawlerConfigurations
.Where(c => c.ConfigurationName == ConstConfigurationName.Douban).ToList())
{
LogHelper.RunActionNotThrowEx(() =>
{
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
var confInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(doubanConf.ConfigurationValue);
for (var pageIndex = 0; pageIndex < confInfo.pagecount.Value; pageIndex++)
{
var lstHouseInfo = GetHouseData(confInfo.groupid.Value,
confInfo.cityname.Value, pageIndex);
houses.AddRange(lstHouseInfo);
}
captrueHouseCount = captrueHouseCount + houses.Count;
CrawlerDataDapper.BulkInsertHouses(houses);
}, "DoubanHouseCrawler CaptureHouseInfo ", doubanConf);
}
BizCrawlerLog.SaveLog($"爬取豆瓣租房小组数据",
$"本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1);
}
catch (Exception ex)
{
LogHelper.Error("DoubanHouseCrawler CrawlerHouseInfo Exception", ex);
}
}
public static void Run()
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
var peopleRentingConf = CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.HuZhuZuFang)
.FirstOrDefault();
var pageCount = peopleRentingConf != null
? JsonConvert.DeserializeObject <dynamic>(peopleRentingConf.ConfigurationValue).pagecount.Value
: 10;
var hsHouseOnlineUrl = new HashSet <string>();
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
for (var pageNum = 1; pageNum < pageCount; pageNum++)
{
string result = getResultFromAPI(pageNum);
houses.AddRange(GetHouseData(result));
}
CrawlerDataDapper.BulkInsertHouses(houses);
captrueHouseCount = captrueHouseCount + houses.Count;
BizCrawlerLog.SaveLog($"爬取上海互助租房数据",
$"本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1);
}
public IActionResult GetHouseInfo(string cityName, string source = "", int houseCount = 500,
int withAnyDays = 7, string keyword = "")
{
var houses = CrawlerDataDapper.SearchHouseInfo(cityName, source, houseCount, withAnyDays, keyword);
var rooms = houses.Select(house =>
{
var markBGType = string.Empty;
int housePrice = (int)house.HousePrice;
if (housePrice > 0)
{
markBGType = LocationMarkBGType.SelectColor(housePrice / 1000);
}
return(new HouseInfo
{
Source = house.Source,
Money = house.DisPlayPrice,
HouseURL = house.HouseOnlineURL,
HouseLocation = house.HouseLocation,
HouseTime = house.PubTime.ToString(),
HouseTitle = house.HouseTitle,
HousePrice = housePrice,
LocationMarkBG = markBGType,
DisplaySource = ConstConfigurationName.ConvertToDisPlayName(house.Source)
});
});
return(Json(new { IsSuccess = true, HouseInfos = rooms }));
}
public static void Run()
{
foreach (var doubanConf in CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.MoguHouse))
{
LogHelper.RunActionNotThrowEx(() =>
{
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
var confInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(doubanConf.ConfigurationValue);
var cityName = confInfo.cityname.Value;
var cityId = (int)confInfo.cityid.Value;
// 2:合租 3:整租 5:业主房源
var rentTypes = new List <int>()
{
2, 3, 5
};
foreach (var rentType in rentTypes)
{
for (var pageIndex = 1; pageIndex <= confInfo.pagecount.Value; pageIndex++)
{
var list = GetHouseData(cityName, cityId, pageIndex, rentType);
houses.AddRange(list);
}
}
CrawlerDataDapper.BulkInsertHouses(houses);
}, "MoGuHouseCrawler Run ", doubanConf);
}
}
private static int CaptureHouse()
{
int captrueHouseCount = 0;
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
for (var pageNum = 1; pageNum < 20; pageNum++)
{
var result = GetHTML(pageNum);
houses.AddRange(GetHouseDataFromHTML(result));
}
captrueHouseCount = captrueHouseCount + houses.Count;
CrawlerDataDapper.BulkInsertHouses(houses);
return(captrueHouseCount);
}
public static List <HouseDashboard> LoadDashboard()
{
string houseDashboardJson = RedisService.ReadCache("HouseDashboard");
if (string.IsNullOrEmpty(houseDashboardJson))
{
List <HouseDashboard> dashboards = CrawlerDataDapper.GetHouseDashboard();
RedisService.WriteCache("HouseDashboard", Newtonsoft.Json.JsonConvert.SerializeObject(dashboards));
return(dashboards);
}
else
{
return(Newtonsoft.Json.JsonConvert.DeserializeObject <List <HouseDashboard> >(houseDashboardJson));
}
}
public static void Run()
{
int captrueHouseCount = 0;
DateTime startTime = DateTime.Now;
foreach (var crawlerConfiguration in CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.CCBHouse).ToList())
{
LogHelper.RunActionNotThrowEx(() =>
{
captrueHouseCount = captrueHouseCount + CaptureHouse(crawlerConfiguration);
}, "CapturPinPaiHouseInfo", crawlerConfiguration);
}
BizCrawlerLog.SaveLog($"爬取CBBHouse数据",
$"本次共爬取到{captrueHouseCount}条数据，耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1);
}
public static void Run()
{
foreach (var doubanConf in CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.Zuber))
{
LogHelper.RunActionNotThrowEx(() =>
{
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
var confInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(doubanConf.ConfigurationValue);
var cityName = confInfo.cityname.Value;
var sequence = "";
for (var i = 0; i <= confInfo.pagecount.Value; i++)
{
var tupleResult = GetHouseData(cityName, sequence);
sequence = tupleResult.Item2;
houses.AddRange(tupleResult.Item1);
}
CrawlerDataDapper.BulkInsertHouses(houses);
}, "DoubanHouseCrawler CaptureHouseInfo ", doubanConf);
}
}
private static int CaptureHouse(BizCrawlerConfiguration crawlerConfiguration)
{
var confInfo = JsonConvert.DeserializeObject <dynamic>(crawlerConfiguration.ConfigurationValue);
if (confInfo.shortcutname == null || string.IsNullOrEmpty(confInfo.shortcutname.Value))
{
return(0);
}
int captrueHouseCount = 0;
string cityShortCutName = confInfo.shortcutname.Value;
List <BaseHouseInfo> houses = new List <BaseHouseInfo>();
for (var pageNum = 1; pageNum < confInfo.pagecount.Value; pageNum++)
{
var result = GetResultByAPI(cityShortCutName, pageNum);
houses.AddRange(GetHouseData(cityShortCutName, result));
}
captrueHouseCount = captrueHouseCount + houses.Count;
CrawlerDataDapper.BulkInsertHouses(houses);
return(captrueHouseCount);
}
public IActionResult AddUserCollection(string onlineURL)
{
var house = CrawlerDataDapper.GetHouseByOnlineURL(onlineURL);
return(null);
}
