public static SpiderBuilder ConfigureAppConfiguration(this SpiderBuilder builder,
string config = null,
string[] args = null, bool loadCommandLine = true)
{
builder.Services.ConfigureAppConfiguration(config, args, loadCommandLine);
return(builder);
}
public static SpiderBuilder UseDefaultDownloaderAllocator(this SpiderBuilder builder)
{
Check.NotNull(builder, nameof(builder));
builder.Services.AddSingleton <IDownloaderAllocator, DownloaderAllocator>();
return(builder);
}
public static SpiderBuilder UseConfiguration(this SpiderBuilder builder, string config = null,
string[] args = null)
{
var configurationBuilder = Framework.CreateConfigurationBuilder(config, args);
builder.Services.AddSingleton <IConfiguration>(configurationBuilder.Build());
return(builder);
}
/// <summary>
/// Create a Spider object
/// </summary>
/// <typeparam name="T"></typeparam>
/// <returns></returns>
public static T Create <T>() where T : Spider
{
var builder = new SpiderBuilder();
builder.AddSerilog();
builder.ConfigureAppConfiguration();
builder.UseStandalone();
builder.AddSpider <T>();
var factory = builder.Build();
return(factory.Create <T>());
}
public static SpiderBuilder AddSpider(this SpiderBuilder builder, Type type)
{
Check.NotNull(builder, nameof(builder));
Check.NotNull(type, nameof(type));
if (!typeof(Spider).IsAssignableFrom(type))
{
throw new SpiderException("不能添加非爬虫类型");
}
builder.Services.AddTransient(type);
return(builder);
}
public static IServiceCollection AddDotnetSpider(this IServiceCollection services,
Action <SpiderBuilder> configureBuilder = null)
{
SpiderBuilder builder = new SpiderBuilder(services);
configureBuilder?.Invoke(builder);
services.AddSingleton(provider => builder.Build());
services.AddScoped <SpiderOptions>();
// Add all storage
services.AddSingleton <MongoEntityStorage>();
return(services);
}
/// <summary>
/// 单机模式
/// 在单机模式下，使用内存型消息队列，因此只有在此作用域 SpiderBuilder 下构建的的爬虫才会共用一个消息队列。
/// </summary>
/// <param name="builder">爬虫构造器</param>
/// <returns>爬虫构造器</returns>
public static SpiderBuilder UseStandalone(this SpiderBuilder builder)
{
Check.NotNull(builder, nameof(builder));
builder.Services.AddSingleton <IMessageQueue, LocalMessageQueue>();
builder.Services.AddSingleton <IDownloaderAgent, LocalDownloaderAgent>();
builder.Services.AddSingleton <IDownloadCenter, LocalDownloadCenter>();
builder.Services.AddSingleton <IDownloaderAgentStore, LocalDownloaderAgentStore>();
builder.Services.AddSingleton <IDownloadService, LocalDownloadCenter>();
builder.Services.AddSingleton <IStatisticsService, StatisticsService>();
builder.Services.AddSingleton <IStatisticsStore, MemoryStatisticsStore>();
builder.Services.AddSingleton <IStatisticsCenter, StatisticsCenter>();
return(builder);
}
/// <summary>
/// 单机模式
/// 在单机模式下，使用内存型消息队列，因此只有在此作用域 SpiderBuilder 下构建的的爬虫才会共用一个消息队列。
/// </summary>
/// <param name="builder">爬虫构造器</param>
/// <returns>爬虫构造器</returns>
public static SpiderBuilder UseStandalone(this SpiderBuilder builder)
{
Check.NotNull(builder, nameof(builder));
var dotnetSpiderBuilder = new DotnetSpiderBuilder(builder.Services);
dotnetSpiderBuilder.UseLocalMessageQueue();
dotnetSpiderBuilder.AddLocalDownloaderAgent(x =>
{
x.UseFileLocker();
x.UseDefaultAdslRedialer();
x.UseDefaultInternetDetector();
});
dotnetSpiderBuilder.AddLocalDownloadCenter();
dotnetSpiderBuilder.AddSpiderStatisticsCenter(x => x.UseMemory());
return(builder);
}
public static SpiderBuilder UseSerilog(this SpiderBuilder builder, LoggerConfiguration configuration = null)
{
Check.NotNull(builder, nameof(builder));
builder.Services.AddSingleton <ILoggerFactory, SerilogLoggerFactory>();
if (configuration == null)
{
configuration = new LoggerConfiguration()
.MinimumLevel.Verbose()
.MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
.Enrich.FromLogContext()
.WriteTo.Console().WriteTo
.RollingFile("dotnet-spider.log");
}
Log.Logger = configuration.CreateLogger();
return(builder);
}
public static SpiderBuilder AddSpider <T>(this SpiderBuilder builder) where T : Spider
{
builder.Services.AddTransient(typeof(T));
return(builder);
}
public static SpiderBuilder UseLocalMessageQueue(this SpiderBuilder builder)
{
builder.Services.AddSingleton <IMessageQueue, LocalMessageQueue>();
return(builder);
}
public static SpiderBuilder ConfigureAppConfiguration(this SpiderBuilder builder,
IConfigurationRoot configuration)
{
builder.Services.AddSingleton <IConfiguration>(configuration);
return(builder);
}
/// <summary>
/// 使用 Serilog 日志
/// </summary>
/// <param name="builder"></param>
/// <param name="configure"></param>
/// <returns></returns>
public static SpiderBuilder AddSerilog(this SpiderBuilder builder, LoggerConfiguration configure = null)
{
builder.Services.AddSerilog(configure);
return(builder);
}
public static SpiderBuilder UserKafka(this SpiderBuilder builder)
{
return(builder);
}
public static SpiderBuilder UseDynamicMessageQueue(this SpiderBuilder builder)
{
builder.Services.AddSingleton <IDynamicMessageQueue, InMemoryMessageQueue>();
return(builder);
}