// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerFactory loggerFactory)
{
//add NLog to .NET Core
loggerFactory.AddNLog();
//Enable ASP.NET Core features (NLog.web) - only needed for ASP.NET Core users
app.AddNLogWeb();
//needed for non-NETSTANDARD platforms: configure nlog.config in your project root. NB: you need NLog.Web.AspNetCore package for this.
env.ConfigureNLog("./wwwroot/nlog.config");
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
loggerFactory.AddConsole();
}
//使用TimedJob
app.UseTimedJob();
app.UseStaticFiles();
app.UseMvcWithDefaultRoute();
app.UseMvc(routes =>
{
routes.MapRoute(
name: "default",
template: "{controller=House}/{action=Index}/{id?}");
});
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
AppSettings.CityJsonFilePath = Path.Combine(env.WebRootPath, "DomainJS//pv.json");
ConnectionStrings.MySQLConnectionString = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json").Build()["ConnectionStrings:MySQLConnectionString"];
AppSettings.DoubanAccount = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json").Build()["DoubanAccount"];
AppSettings.DoubanPassword = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json").Build()["DoubanPassword"];
DoubanHTTPHelper.InitCookieCollection();
DomainProxyInfo.InitDomainProxyInfo(Path.Combine(env.WebRootPath, "availableProxy.json"));
}
private static List <BizHouseInfo> GetDataFromOnlineWeb(string groupID, string cityName, int pageIndex)
{
List <BizHouseInfo> lstHouseInfo = new List <BizHouseInfo>();
var url = $"https://www.douban.com/group/{groupID}/discussion?start={pageIndex * 25}";
var htmlResult = DoubanHTTPHelper.GetHTMLForDouban(url);
if (string.IsNullOrEmpty(htmlResult))
{
return(lstHouseInfo);
}
var page = HtmlParser.Parse(htmlResult);
var tableElement = page.QuerySelector("table.olt");
if (tableElement == null)
{
return(lstHouseInfo);
}
foreach (var trItem in tableElement.QuerySelectorAll("tr"))
{
var titleItem = trItem.QuerySelector("td.title");
if (titleItem == null || DataContent.HouseInfos.Find(titleItem.QuerySelector("a").GetAttribute("href")) != null)
{
continue;
}
var houseTitle = titleItem.QuerySelector("a").GetAttribute("title");
var housePrice = JiebaTools.GetHousePrice(houseTitle);
var houseInfo = new BizHouseInfo()
{
HouseTitle = houseTitle,
HouseOnlineURL = titleItem.QuerySelector("a").GetAttribute("href"),
HouseLocation = houseTitle,
HouseText = houseTitle,
DataCreateTime = DateTime.Now,
PubTime = titleItem.QuerySelector("td.time") != null
? DateTime.Parse(DateTime.Now.ToString("yyyy-") + titleItem.QuerySelector("td.time").InnerHtml)
: DateTime.Now,
DisPlayPrice = housePrice > 0 ? $"{housePrice}元":"",
Source = ConstConfigurationName.Douban,
HousePrice = housePrice,
LocationCityName = cityName,
IsAnalyzed = housePrice > 0,
Status = housePrice > 0 ? 1 : 0,
};
lstHouseInfo.Add(houseInfo);
}
return(lstHouseInfo);
}
private static void AnalyzeFromWebPage(Web.Model.DBHouseInfo houseInfo,
ref decimal housePrice, ref string houseTextContent)
{
var htmlResult = DoubanHTTPHelper.GetHTMLForDouban(houseInfo.HouseOnlineURL);
//没有页面信息
if (string.IsNullOrEmpty(htmlResult))
{
//404页面
houseInfo.Status = 2;
}
else
{
var page = HtmlParser.Parse(htmlResult);
var topicContent = page.QuerySelector("div.topic-content");
//没有帖子内容
if (topicContent == null || topicContent.QuerySelector("p") == null || topicContent.QuerySelector("p") == null)
{
houseInfo.Status = 3;
}
else
{
//获取帖子内容
houseTextContent = topicContent.QuerySelector("p").TextContent;
//获取价格信息
housePrice = JiebaTools.GetHousePrice(houseTextContent);
if (housePrice != 0 || !string.IsNullOrEmpty(houseTextContent))
{
houseInfo.Status = 1;
}
houseInfo.DisPlayPrice = housePrice.ToString(CultureInfo.InvariantCulture);
houseInfo.HousePrice = housePrice;
houseInfo.HouseText = houseTextContent;
}
}
}