public static SpiderBuilder ConfigureAppConfiguration(this SpiderBuilder builder, string config = null, string[] args = null, bool loadCommandLine = true) { builder.Services.ConfigureAppConfiguration(config, args, loadCommandLine); return(builder); }
public static SpiderBuilder UseDefaultDownloaderAllocator(this SpiderBuilder builder) { Check.NotNull(builder, nameof(builder)); builder.Services.AddSingleton <IDownloaderAllocator, DownloaderAllocator>(); return(builder); }
public static SpiderBuilder UseConfiguration(this SpiderBuilder builder, string config = null, string[] args = null) { var configurationBuilder = Framework.CreateConfigurationBuilder(config, args); builder.Services.AddSingleton <IConfiguration>(configurationBuilder.Build()); return(builder); }
/// <summary> /// Create a Spider object /// </summary> /// <typeparam name="T"></typeparam> /// <returns></returns> public static T Create <T>() where T : Spider { var builder = new SpiderBuilder(); builder.AddSerilog(); builder.ConfigureAppConfiguration(); builder.UseStandalone(); builder.AddSpider <T>(); var factory = builder.Build(); return(factory.Create <T>()); }
public static SpiderBuilder AddSpider(this SpiderBuilder builder, Type type) { Check.NotNull(builder, nameof(builder)); Check.NotNull(type, nameof(type)); if (!typeof(Spider).IsAssignableFrom(type)) { throw new SpiderException("不能添加非爬虫类型"); } builder.Services.AddTransient(type); return(builder); }
public static IServiceCollection AddDotnetSpider(this IServiceCollection services, Action <SpiderBuilder> configureBuilder = null) { SpiderBuilder builder = new SpiderBuilder(services); configureBuilder?.Invoke(builder); services.AddSingleton(provider => builder.Build()); services.AddScoped <SpiderOptions>(); // Add all storage services.AddSingleton <MongoEntityStorage>(); return(services); }
/// <summary> /// 单机模式 /// 在单机模式下,使用内存型消息队列,因此只有在此作用域 SpiderBuilder 下构建的的爬虫才会共用一个消息队列。 /// </summary> /// <param name="builder">爬虫构造器</param> /// <returns>爬虫构造器</returns> public static SpiderBuilder UseStandalone(this SpiderBuilder builder) { Check.NotNull(builder, nameof(builder)); builder.Services.AddSingleton <IMessageQueue, LocalMessageQueue>(); builder.Services.AddSingleton <IDownloaderAgent, LocalDownloaderAgent>(); builder.Services.AddSingleton <IDownloadCenter, LocalDownloadCenter>(); builder.Services.AddSingleton <IDownloaderAgentStore, LocalDownloaderAgentStore>(); builder.Services.AddSingleton <IDownloadService, LocalDownloadCenter>(); builder.Services.AddSingleton <IStatisticsService, StatisticsService>(); builder.Services.AddSingleton <IStatisticsStore, MemoryStatisticsStore>(); builder.Services.AddSingleton <IStatisticsCenter, StatisticsCenter>(); return(builder); }
/// <summary> /// 单机模式 /// 在单机模式下,使用内存型消息队列,因此只有在此作用域 SpiderBuilder 下构建的的爬虫才会共用一个消息队列。 /// </summary> /// <param name="builder">爬虫构造器</param> /// <returns>爬虫构造器</returns> public static SpiderBuilder UseStandalone(this SpiderBuilder builder) { Check.NotNull(builder, nameof(builder)); var dotnetSpiderBuilder = new DotnetSpiderBuilder(builder.Services); dotnetSpiderBuilder.UseLocalMessageQueue(); dotnetSpiderBuilder.AddLocalDownloaderAgent(x => { x.UseFileLocker(); x.UseDefaultAdslRedialer(); x.UseDefaultInternetDetector(); }); dotnetSpiderBuilder.AddLocalDownloadCenter(); dotnetSpiderBuilder.AddSpiderStatisticsCenter(x => x.UseMemory()); return(builder); }
public static SpiderBuilder UseSerilog(this SpiderBuilder builder, LoggerConfiguration configuration = null) { Check.NotNull(builder, nameof(builder)); builder.Services.AddSingleton <ILoggerFactory, SerilogLoggerFactory>(); if (configuration == null) { configuration = new LoggerConfiguration() .MinimumLevel.Verbose() .MinimumLevel.Override("Microsoft", LogEventLevel.Warning) .Enrich.FromLogContext() .WriteTo.Console().WriteTo .RollingFile("dotnet-spider.log"); } Log.Logger = configuration.CreateLogger(); return(builder); }
public static SpiderBuilder AddSpider <T>(this SpiderBuilder builder) where T : Spider { builder.Services.AddTransient(typeof(T)); return(builder); }
public static SpiderBuilder UseLocalMessageQueue(this SpiderBuilder builder) { builder.Services.AddSingleton <IMessageQueue, LocalMessageQueue>(); return(builder); }
public static SpiderBuilder ConfigureAppConfiguration(this SpiderBuilder builder, IConfigurationRoot configuration) { builder.Services.AddSingleton <IConfiguration>(configuration); return(builder); }
/// <summary> /// 使用 Serilog 日志 /// </summary> /// <param name="builder"></param> /// <param name="configure"></param> /// <returns></returns> public static SpiderBuilder AddSerilog(this SpiderBuilder builder, LoggerConfiguration configure = null) { builder.Services.AddSerilog(configure); return(builder); }
public static SpiderBuilder UserKafka(this SpiderBuilder builder) { return(builder); }
public static SpiderBuilder UseDynamicMessageQueue(this SpiderBuilder builder) { builder.Services.AddSingleton <IDynamicMessageQueue, InMemoryMessageQueue>(); return(builder); }