static void Main(string[] args)
{
//Common.Logger.LoggerHost = "http://localhost/";
var logger = new Common.Logger(typeof(Program));
//编码注册
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
//添加Nugget包Microsoft.Extensions.Configuration(ConfigurationBuilder) 和 Microsoft.Extensions.Configuration.Json(AddJsonFile)
var config = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory()) //指定配置文件所在的目录
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) //指定加载的配置文件
.Build(); //编译成对象
var service = new ServiceCollection()
.AddOptions()
.AddSingleton <IConfiguration>(config) //单例
//.AddTransient<ILoggerFactory, LoggerFactory>()
.AddTransient <DataContext, ShiweijiaContext>()
.AddTransient <BaseSpider, ShiweijiaCategory>()
.AddTransient <BaseSpider, ShiweijiaProduct>() //注入服务
.BuildServiceProvider(); //编译
Task.Run(() => BaseSpider.Init(new DataContext(config))).GetAwaiter().GetResult();
logger.Info("开始抓取");
foreach (var spider in service.GetServices <BaseSpider>())
{
spider.Run();
}
logger.Info("抓取完成");
Task.Run(() => BaseSpider.Finish(new DataContext(config))).GetAwaiter().GetResult();
Console.Read();
}
public DefaultRequestPipelines(BaseSpider spider)
{
this.spider = spider;
this.bloomFilter = new BloomFilterHelper(16, 20480000);
spider.http.RegRequestPipelines(this.UrlFilter);
spider.http.RegRequestPipelines(this.DeepControl);
}
public DefaultResponsePipelines(BaseSpider spider)
{
this.spider = spider;
//this.spider.http.RegResponsePipelines(this.PageUrlGetter);
}
