#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. protected Spider(ILogger <Spider <T> > logger, INetConnection connection, IOptions <ConfigOptions> config, IParallelManager parallelManager) #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. { _config = config.Value; _logger = logger; Connection = connection; ParallelManager = parallelManager; FileUrlsLog = new FileJson($"{GetType().Name}{"ErrorItens.json"}", _config.LogFolder); }
#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. public ThreadManager(IOptions <ConfigOptions> config) #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. { _config = config.Value; _totalItensToProcessCount = _itensProcessDoneCount = 0; AutoEvent = new AutoResetEvent(false); //init threadpool ThreadPool.SetMaxThreads(_config.MaximumThreads, _config.MaximumThreads); }
public static void AddMisterSpider(this IServiceCollection services, IConfiguration configuration) { ConfigOptions config = configuration.GetSection(ConfigOptions.Position).Get <ConfigOptions>(); if (string.IsNullOrWhiteSpace(config.LogFolder)) { throw new Exception("Are you missing the appsettings section Spider?"); } // Add our Config object so it can be injected services.Configure <ConfigOptions>(configuration.GetSection(ConfigOptions.Position)); services.AddSingleton <INetConnection, NetConnection>(); services.AddTransient <IParallelManager, ThreadManager>(); services.AddSingleton <ISpiderFactory, SpiderFactory>(); services.AddScoped <IWebScrapperManager, WebScrapperManager>(); services.AddHttpClient(Options.DefaultName, httpClient => { httpClient.DefaultRequestHeaders.Add(HeaderNames.UserAgent, "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36"); }); }
public NetConnection(ILogger <NetConnection> logger, IOptions <ConfigOptions> config, IHttpClientFactory clientFactory) { _logger = logger; _httpClientFactory = clientFactory; _config = config.Value; }