public static DotnetSpiderBuilder AddLocalDownloadCenter(this DotnetSpiderBuilder builder)
{
Check.NotNull(builder, nameof(builder));
builder.Services.AddSingleton <IDownloadCenter, LocalDownloadCenter>();
builder.Services.AddSingleton <IDownloaderAgentStore, LocalDownloaderAgentStore>();
return(builder);
}
public static IServiceCollection AddDotnetSpider(this IServiceCollection services,
Action <DotnetSpiderBuilder> configureBuilder = null)
{
Check.NotNull(services, nameof(services));
DotnetSpiderBuilder builder = new DotnetSpiderBuilder(services);
configureBuilder?.Invoke(builder);
return(services);
}
public static DotnetSpiderBuilder AddDownloaderCenter(this DotnetSpiderBuilder builder,
Action <DownloadCenterBuilder> configure = null)
{
Check.NotNull(builder, nameof(builder));
builder.Services.AddSingleton <IDownloadCenter, LocalDownloadCenter>();
DownloadCenterBuilder downloadCenterBuilder = new DownloadCenterBuilder(builder.Services);
configure?.Invoke(downloadCenterBuilder);
return(builder);
}
public static DotnetSpiderBuilder AddSpiderStatisticsCenter(this DotnetSpiderBuilder builder,
Action <StatisticsBuilder> configure)
{
Check.NotNull(builder, nameof(builder));
builder.Services.AddSingleton <IStatisticsCenter, StatisticsCenter>();
var spiderStatisticsBuilder = new StatisticsBuilder(builder.Services);
configure?.Invoke(spiderStatisticsBuilder);
return(builder);
}
public static DotnetSpiderBuilder AddDownloaderAgent(this DotnetSpiderBuilder builder,
Action <AgentBuilder> configure = null)
{
Check.NotNull(builder, nameof(builder));
builder.Services.AddSingleton <IDownloaderAllocator, DownloaderAllocator>();
builder.Services.AddSingleton <IDownloaderAgent, LocalDownloaderAgent>();
builder.Services.AddSingleton <NetworkCenter>();
builder.Services.AddScoped <IDownloaderAgentOptions, DownloaderAgentOptions>();
AgentBuilder spiderAgentBuilder = new AgentBuilder(builder.Services);
configure?.Invoke(spiderAgentBuilder);
return(builder);
}
/// <summary>
/// 单机模式
/// 在单机模式下，使用内存型消息队列，因此只有在此作用域 SpiderBuilder 下构建的的爬虫才会共用一个消息队列。
/// </summary>
/// <param name="builder">爬虫构造器</param>
/// <returns>爬虫构造器</returns>
public static SpiderBuilder UseStandalone(this SpiderBuilder builder)
{
Check.NotNull(builder, nameof(builder));
var dotnetSpiderBuilder = new DotnetSpiderBuilder(builder.Services);
dotnetSpiderBuilder.UseLocalMessageQueue();
dotnetSpiderBuilder.AddLocalDownloaderAgent(x =>
{
x.UseFileLocker();
x.UseDefaultAdslRedialer();
x.UseDefaultInternetDetector();
});
dotnetSpiderBuilder.AddLocalDownloadCenter();
dotnetSpiderBuilder.AddSpiderStatisticsCenter(x => x.UseMemory());
return(builder);
}
public static DotnetSpiderBuilder UserKafka(this DotnetSpiderBuilder builder)
{
return(builder);
}
public static DotnetSpiderBuilder UseLocalMessageQueue(this DotnetSpiderBuilder builder)
{
builder.Services.AddSingleton <IMessageQueue, LocalMessageQueue>();
return(builder);
}
public static DotnetSpiderBuilder UseDynamicMessageQueue(this DotnetSpiderBuilder builder)
{
//builder.Services.AddSingleton<IMessageQueue, LocalMessageQueue>();
builder.Services.AddSingleton <IDynamicMessageQueue, InMemoryMessageQueue>();
return(builder);
}