// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerFactory loggerFactory)
{
//add NLog to .NET Core
loggerFactory.AddNLog();
//Enable ASP.NET Core features (NLog.web) - only needed for ASP.NET Core users
app.AddNLogWeb();
//needed for non-NETSTANDARD platforms: configure nlog.config in your project root. NB: you need NLog.Web.AspNetCore package for this.
env.ConfigureNLog("./wwwroot/nlog.config");
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
loggerFactory.AddConsole();
}
//使用TimedJob
app.UseTimedJob();
app.UseStaticFiles();
app.UseMvcWithDefaultRoute();
app.UseMvc(routes =>
{
routes.MapRoute(
name: "default",
template: "{controller=House}/{action=Index}/{id?}");
});
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
AppSettings.CityJsonFilePath = Path.Combine(env.WebRootPath, "DomainJS//pv.json");
ConnectionStrings.MySQLConnectionString = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json").Build()["ConnectionStrings:MySQLConnectionString"];
AppSettings.DoubanAccount = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json").Build()["DoubanAccount"];
AppSettings.DoubanPassword = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json").Build()["DoubanPassword"];
DoubanHTTPHelper.InitCookieCollection();
DomainProxyInfo.InitDomainProxyInfo(Path.Combine(env.WebRootPath, "availableProxy.json"));
}
public static string GetHTMLForDouban(string url)
{
var proxyIPItem = DomainProxyInfo.GetRandomProxyIPItem();
HttpClientHandler httpClientHandler = new HttpClientHandler();
if (proxyIPItem != null)
{
httpClientHandler = new HttpClientHandler
{
Proxy = new CrawlerProxyInfo($"http://{proxyIPItem.ip}:{proxyIPItem.port}"),
UseProxy = true
};
LogHelper.Info("URL:" + url + ";ProxyIP:" + Newtonsoft.Json.JsonConvert.SerializeObject(proxyIPItem));
Console.WriteLine("URL:" + url + ";ProxyIP:" + Newtonsoft.Json.JsonConvert.SerializeObject(proxyIPItem));
}
try
{
var httpClient = new HttpClient(httpClientHandler);
httpClient.Timeout = new TimeSpan(0, 3, 0);
httpClient.DefaultRequestHeaders.ExpectContinue = true;
var task = httpClient.GetStringAsync(url);
var htmlDoc = task.Result;
if (htmlDoc != null)
{
LogHelper.Info("GetStringAsync Success:" + url + ";ProxyIP:" + Newtonsoft.Json.JsonConvert.SerializeObject(proxyIPItem));
}
return(htmlDoc);
}
catch (System.Exception ex)
{
if (proxyIPItem != null)
{
DomainProxyInfo.GetDomainProxyInfo().lstDoubanProxyIP.Remove(proxyIPItem);
Console.WriteLine("Remove ProxyIP:" + Newtonsoft.Json.JsonConvert.SerializeObject(proxyIPItem));
}
LogHelper.Error("GetHTMLByURL Exception", ex.InnerException, new { URL = url });
Console.WriteLine(ex.InnerException.ToString());
return(string.Empty);
}
}
