public TianYanCrawler() : base(1003, "天眼查")
{
//logger = LoggerManager.GetLogger(GetType());
dal = AppServiceProvider.ServiceProvider.GetService <CrawlerDAL>();
RequestUrl = "https://bj.tianyancha.com/search/ocE-e015-s1-la3";
CookiesContainer = new CookieContainer();
}
public static void CapturHouseInfo()
{
var peopleRentingConf = dataContent.CrawlerConfigurations.FirstOrDefault(conf => conf.ConfigurationName == ConstConfigurationName.HuZhuZuFang);
var pageCount = peopleRentingConf != null?Newtonsoft.Json.JsonConvert.DeserializeObject <dynamic>(peopleRentingConf.ConfigurationValue).pagecount.Value : 10;
HashSet <string> hsHouseOnlineURL = new CrawlerDAL().GetAllHuzhuzufangHouseOnlineURL();
for (var pageIndex = 1; pageIndex < pageCount; pageIndex++)
{
LogHelper.RunActionNotThrowEx(() =>
{
GetDataByWebAPI(pageIndex, hsHouseOnlineURL);
}, "CapturHouseInfo", pageIndex);
}
}
/// <summary>
/// 注入服务
/// </summary>
/// <param name="services"></param>
public void ConfigureServices(IServiceCollection services)
{
//services.AddSingleLogServiceCollectionExtensions();
ConfigurationBuilder builder = new ConfigurationBuilder();
builder.AddJsonFile("appsettings.json");
IConfiguration configuration = builder.Build();
services.AddSingleton <IConfiguration>(configuration);
string connectionString = configuration.GetSection("ConnectionStrings")["SqlServerConnection"];
DbHelperSQLP DbHelperSQL = new DbHelperSQLP(connectionString);
CrawlerDAL crawlerDAL = new CrawlerDAL(DbHelperSQL);
services.AddSingleton <CrawlerDAL>(crawlerDAL);
}
public GcxxCrawler() : base(1001, "跑项目")
{
//logger = LoggerManager.GetLogger(GetType());
dal = AppServiceProvider.ServiceProvider.GetService <CrawlerDAL>();
RequestUrl = "http://www.gcxx.com/project/index?provinceOrCity=110000";
}
