#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
protected Spider(ILogger <Spider <T> > logger, INetConnection connection, IOptions <ConfigOptions> config, IParallelManager parallelManager)
#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
{
_config = config.Value;
_logger = logger;
Connection = connection;
ParallelManager = parallelManager;
FileUrlsLog = new FileJson($"{GetType().Name}{"ErrorItens.json"}", _config.LogFolder);
}
#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
public ThreadManager(IOptions <ConfigOptions> config)
#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
{
_config = config.Value;
_totalItensToProcessCount = _itensProcessDoneCount = 0;
AutoEvent = new AutoResetEvent(false);
//init threadpool
ThreadPool.SetMaxThreads(_config.MaximumThreads, _config.MaximumThreads);
}
public static void AddMisterSpider(this IServiceCollection services, IConfiguration configuration)
{
ConfigOptions config = configuration.GetSection(ConfigOptions.Position).Get <ConfigOptions>();
if (string.IsNullOrWhiteSpace(config.LogFolder))
{
throw new Exception("Are you missing the appsettings section Spider?");
}
// Add our Config object so it can be injected
services.Configure <ConfigOptions>(configuration.GetSection(ConfigOptions.Position));
services.AddSingleton <INetConnection, NetConnection>();
services.AddTransient <IParallelManager, ThreadManager>();
services.AddSingleton <ISpiderFactory, SpiderFactory>();
services.AddScoped <IWebScrapperManager, WebScrapperManager>();
services.AddHttpClient(Options.DefaultName, httpClient =>
{
httpClient.DefaultRequestHeaders.Add(HeaderNames.UserAgent, "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36");
});
}
public NetConnection(ILogger <NetConnection> logger, IOptions <ConfigOptions> config, IHttpClientFactory clientFactory)
{
_logger = logger;
_httpClientFactory = clientFactory;
_config = config.Value;
}
