public IActionResult GetHouseInfo(string cityName, string source = "", int houseCount = 100, int intervalDay = 7, string keyword = "", bool refresh = false) { try { var houseList = CrawlerDataDapper.SearchHouseInfo(cityName, source, houseCount, intervalDay, keyword, refresh); var rooms = houseList.Select(house => { var markBGType = string.Empty; int housePrice = (int)house.HousePrice; if (housePrice > 0) { markBGType = LocationMarkBGType.SelectColor(housePrice / 1000); } return(new HouseInfo { Money = house.DisPlayPrice, HouseURL = house.HouseOnlineURL, HouseLocation = house.HouseLocation, HouseTime = house.PubTime.ToString(), HousePrice = housePrice, LocationMarkBG = markBGType, DisplaySource = ConstConfigurationName.ConvertToDisPlayName(house.Source) }); }); return(Json(new { IsSuccess = true, HouseInfos = rooms })); } catch (Exception ex) { return(Json(new { IsSuccess = false, error = ex.ToString() })); } }
public void Run() { var today = DateTime.Now.ToLocalTime(); EmailInfo email = new EmailInfo(); email.Receiver = configuration.ReceiverAddress; email.ReceiverName = configuration.ReceiverName; email.Subject = $"地图搜租房每日数据汇总({today.ToString("yyyy-MM-dd")})"; var statList = CrawlerDataDapper.GetHouseStatList(); string bodyHTML = @"<table border='1' cellpadding='0' cellspacing='0' width='100%'> <tr> <td>来源</td> <td>数量</td> <td>最晚发布时间</td> <td>最晚入库时间</td> </tr>"; foreach (var stat in statList) { bodyHTML = bodyHTML + $" <tr> <td>{stat.Source}</td><td>{stat.HouseSum}</td><td>{stat.LastPubTime}</td> <td>{stat.LastCreateTime}</td> </tr>"; } bodyHTML = bodyHTML + $" <tr> <td>共计</td><td>{statList.Sum(s => s.HouseSum)}</td> </tr>"; bodyHTML = bodyHTML + " </table>"; email.Body = bodyHTML; emailService.Send(email); }
public static void CapturePinPaiHouseInfo() { int captrueHouseCount = 0; DateTime startTime = DateTime.Now; foreach (var crawlerConfiguration in DataContent.CrawlerConfigurations.Where(c => c.ConfigurationName == ConstConfigurationName.PinPaiGongYu && c.IsEnabled).ToList()) { LogHelper.RunActionNotThrowEx(() => { List <BaseHouseInfo> houses = new List <BaseHouseInfo>(); var confInfo = JsonConvert.DeserializeObject <dynamic>(crawlerConfiguration.ConfigurationValue); for (var index = 0; index < confInfo.pagecount.Value; index++) { var url = $"http://{confInfo.shortcutname.Value}.58.com/pinpaigongyu/pn/{index}"; var houseHTML = GetHouseHTML(url); houses.AddRange(GetDataFromHMTL(confInfo.shortcutname.Value, confInfo.cityname.Value, houseHTML)); } CrawlerDataDapper.BulkInsertHouses(houses); captrueHouseCount = captrueHouseCount + houses.Count; }, "CapturPinPaiHouseInfo", crawlerConfiguration); } BizCrawlerLog.SaveLog($"爬取58品牌公寓租房数据", $"本次共爬取到{captrueHouseCount}条数据,耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1); }
public static void RefreshDashboard() { var lstCityHouse = CrawlerDataDapper.GetHouseSourceInfoList(); var config = DataContent.CrawlerConfigurations.FirstOrDefault(c => c.ConfigurationName == ConstConfigurationName.CityHouseInfo); if (config == null) { config = new BizCrawlerConfiguration { ConfigurationKey = 0, ConfigurationName = ConstConfigurationName.CityHouseInfo, ConfigurationValue = Newtonsoft.Json.JsonConvert.SerializeObject(lstCityHouse), IsEnabled = true }; DataContent.Add(config); DataContent.SaveChanges(); } else { config.ConfigurationKey = 0; config.ConfigurationName = ConstConfigurationName.CityHouseInfo; config.ConfigurationValue = Newtonsoft.Json.JsonConvert.SerializeObject(lstCityHouse); config.IsEnabled = true; DataContent.SaveChanges(); } }
public static void CaptureHouseInfo() { try { int captrueHouseCount = 0; DateTime startTime = DateTime.Now; foreach (var doubanConf in DataContent.CrawlerConfigurations .Where(c => c.ConfigurationName == ConstConfigurationName.Douban).ToList()) { LogHelper.RunActionNotThrowEx(() => { List <BaseHouseInfo> houses = new List <BaseHouseInfo>(); var confInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(doubanConf.ConfigurationValue); for (var pageIndex = 0; pageIndex < confInfo.pagecount.Value; pageIndex++) { var lstHouseInfo = GetHouseData(confInfo.groupid.Value, confInfo.cityname.Value, pageIndex); houses.AddRange(lstHouseInfo); } captrueHouseCount = captrueHouseCount + houses.Count; CrawlerDataDapper.BulkInsertHouses(houses); }, "DoubanHouseCrawler CaptureHouseInfo ", doubanConf); } BizCrawlerLog.SaveLog($"爬取豆瓣租房小组数据", $"本次共爬取到{captrueHouseCount}条数据,耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1); } catch (Exception ex) { LogHelper.Error("DoubanHouseCrawler CrawlerHouseInfo Exception", ex); } }
public static void Run() { int captrueHouseCount = 0; DateTime startTime = DateTime.Now; var peopleRentingConf = CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.HuZhuZuFang) .FirstOrDefault(); var pageCount = peopleRentingConf != null ? JsonConvert.DeserializeObject <dynamic>(peopleRentingConf.ConfigurationValue).pagecount.Value : 10; var hsHouseOnlineUrl = new HashSet <string>(); List <BaseHouseInfo> houses = new List <BaseHouseInfo>(); for (var pageNum = 1; pageNum < pageCount; pageNum++) { string result = getResultFromAPI(pageNum); houses.AddRange(GetHouseData(result)); } CrawlerDataDapper.BulkInsertHouses(houses); captrueHouseCount = captrueHouseCount + houses.Count; BizCrawlerLog.SaveLog($"爬取上海互助租房数据", $"本次共爬取到{captrueHouseCount}条数据,耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1); }
public IActionResult GetHouseInfo(string cityName, string source = "", int houseCount = 500, int withAnyDays = 7, string keyword = "") { var houses = CrawlerDataDapper.SearchHouseInfo(cityName, source, houseCount, withAnyDays, keyword); var rooms = houses.Select(house => { var markBGType = string.Empty; int housePrice = (int)house.HousePrice; if (housePrice > 0) { markBGType = LocationMarkBGType.SelectColor(housePrice / 1000); } return(new HouseInfo { Source = house.Source, Money = house.DisPlayPrice, HouseURL = house.HouseOnlineURL, HouseLocation = house.HouseLocation, HouseTime = house.PubTime.ToString(), HouseTitle = house.HouseTitle, HousePrice = housePrice, LocationMarkBG = markBGType, DisplaySource = ConstConfigurationName.ConvertToDisPlayName(house.Source) }); }); return(Json(new { IsSuccess = true, HouseInfos = rooms })); }
public static void Run() { foreach (var doubanConf in CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.MoguHouse)) { LogHelper.RunActionNotThrowEx(() => { List <BaseHouseInfo> houses = new List <BaseHouseInfo>(); var confInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(doubanConf.ConfigurationValue); var cityName = confInfo.cityname.Value; var cityId = (int)confInfo.cityid.Value; // 2:合租 3:整租 5:业主房源 var rentTypes = new List <int>() { 2, 3, 5 }; foreach (var rentType in rentTypes) { for (var pageIndex = 1; pageIndex <= confInfo.pagecount.Value; pageIndex++) { var list = GetHouseData(cityName, cityId, pageIndex, rentType); houses.AddRange(list); } } CrawlerDataDapper.BulkInsertHouses(houses); }, "MoGuHouseCrawler Run ", doubanConf); } }
private static int CaptureHouse() { int captrueHouseCount = 0; List <BaseHouseInfo> houses = new List <BaseHouseInfo>(); for (var pageNum = 1; pageNum < 20; pageNum++) { var result = GetHTML(pageNum); houses.AddRange(GetHouseDataFromHTML(result)); } captrueHouseCount = captrueHouseCount + houses.Count; CrawlerDataDapper.BulkInsertHouses(houses); return(captrueHouseCount); }
public static List <HouseDashboard> LoadDashboard() { string houseDashboardJson = RedisService.ReadCache("HouseDashboard"); if (string.IsNullOrEmpty(houseDashboardJson)) { List <HouseDashboard> dashboards = CrawlerDataDapper.GetHouseDashboard(); RedisService.WriteCache("HouseDashboard", Newtonsoft.Json.JsonConvert.SerializeObject(dashboards)); return(dashboards); } else { return(Newtonsoft.Json.JsonConvert.DeserializeObject <List <HouseDashboard> >(houseDashboardJson)); } }
public static void Run() { int captrueHouseCount = 0; DateTime startTime = DateTime.Now; foreach (var crawlerConfiguration in CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.CCBHouse).ToList()) { LogHelper.RunActionNotThrowEx(() => { captrueHouseCount = captrueHouseCount + CaptureHouse(crawlerConfiguration); }, "CapturPinPaiHouseInfo", crawlerConfiguration); } BizCrawlerLog.SaveLog($"爬取CBBHouse数据", $"本次共爬取到{captrueHouseCount}条数据,耗时{ (DateTime.Now - startTime).TotalSeconds}秒。", 1); }
public static void Run() { foreach (var doubanConf in CrawlerDataDapper.GetConfigurationList(ConstConfigurationName.Zuber)) { LogHelper.RunActionNotThrowEx(() => { List <BaseHouseInfo> houses = new List <BaseHouseInfo>(); var confInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(doubanConf.ConfigurationValue); var cityName = confInfo.cityname.Value; var sequence = ""; for (var i = 0; i <= confInfo.pagecount.Value; i++) { var tupleResult = GetHouseData(cityName, sequence); sequence = tupleResult.Item2; houses.AddRange(tupleResult.Item1); } CrawlerDataDapper.BulkInsertHouses(houses); }, "DoubanHouseCrawler CaptureHouseInfo ", doubanConf); } }
private static int CaptureHouse(BizCrawlerConfiguration crawlerConfiguration) { var confInfo = JsonConvert.DeserializeObject <dynamic>(crawlerConfiguration.ConfigurationValue); if (confInfo.shortcutname == null || string.IsNullOrEmpty(confInfo.shortcutname.Value)) { return(0); } int captrueHouseCount = 0; string cityShortCutName = confInfo.shortcutname.Value; List <BaseHouseInfo> houses = new List <BaseHouseInfo>(); for (var pageNum = 1; pageNum < confInfo.pagecount.Value; pageNum++) { var result = GetResultByAPI(cityShortCutName, pageNum); houses.AddRange(GetHouseData(cityShortCutName, result)); } captrueHouseCount = captrueHouseCount + houses.Count; CrawlerDataDapper.BulkInsertHouses(houses); return(captrueHouseCount); }
public IActionResult AddUserCollection(string onlineURL) { var house = CrawlerDataDapper.GetHouseByOnlineURL(onlineURL); return(null); }