public static Builder UseRequestHasher <TRequestHasher>(this Builder builder) where TRequestHasher : class, IRequestHasher
{
builder.ConfigureServices(x =>
{
x.AddSingleton <IRequestHasher, TRequestHasher>();
});
return(builder);
}
// ReSharper disable once InconsistentNaming
public static Builder UseMD5HashAlgorithmService(this Builder builder)
{
builder.ConfigureServices(x =>
{
x.AddSingleton <IHashAlgorithmService, MD5HashAlgorithmService>();
});
return(builder);
}
public static Builder UseRequestHasher(this Builder builder)
{
builder.ConfigureServices(x =>
{
x.AddSingleton <IRequestHasher, RequestHasher>();
});
return(builder);
}
public static Builder RegisterDownloader <TDownloader>(this Builder builder) where TDownloader : class, IDownloader
{
builder.ConfigureServices(x =>
{
x.AddSingleton <IDownloader, TDownloader>();
});
return(builder);
}
public static Builder UseProxy <TProxySupplier>(this Builder builder)
where TProxySupplier : class, IProxySupplier
{
builder.ConfigureServices(x =>
{
x.AddProxy <TProxySupplier>();
});
return(builder);
}
public static Builder UseQueueDistinctBfsScheduler <T>(this Builder builder)
where T : class, IDuplicateRemover
{
builder.ConfigureServices(x =>
{
x.TryAddSingleton <IDuplicateRemover, T>();
x.TryAddSingleton <IScheduler, QueueDistinctBfsScheduler>();
});
return(builder);
}
public static Builder CreateBuilder(Type type,
string[] args = null, Action <SpiderOptions> configureDelegate = null)
{
var builder = new Builder();
ConfigureBuilder(builder, args, configureDelegate);
builder.ConfigureServices(services =>
{
services.AddSingleton(typeof(IHostedService), type);
});
return(builder);
}
/// <summary>
/// 只有本地爬虫才能配置下载器，分布式爬虫的下载器注册是在下载器代理中
/// </summary>
/// <param name="builder"></param>
/// <typeparam name="TDownloader"></typeparam>
/// <returns></returns>
public static Builder UseDownloader <TDownloader>(this Builder builder)
where TDownloader : class, IDownloader
{
builder.ConfigureServices(x =>
{
x.AddAgent <TDownloader>(opts =>
{
opts.AgentId = ObjectId.NewId().ToString();
opts.AgentName = opts.AgentId;
});
});
return(builder);
}
/// <summary>
/// 只有本地爬虫才能配置下载器，分布式爬虫的下载器注册是在下载器代理中
/// </summary>
/// <param name="builder"></param>
/// <typeparam name="TDownloader"></typeparam>
/// <returns></returns>
public static Builder UseDownloader <TDownloader>(this Builder builder)
where TDownloader : class, IDownloader
{
builder.Properties["DefaultDownloader"] = $"DOTNET_SPIDER_{typeof(TDownloader).Name}";
builder.ConfigureServices(x =>
{
x.AddAgent <TDownloader>(opts =>
{
opts.AgentId = ObjectId.NewId().ToString();
opts.AgentName = opts.AgentId;
});
});
return(builder);
}
public static Builder UseQueueBfsScheduler(this Builder builder)
{
builder.ConfigureServices(x => { x.TryAddSingleton <IScheduler, QueueBfsScheduler>(); });
return(builder);
}
public static Builder UseDockerLifetime(this Builder builder)
{
builder.ConfigureServices(x => { x.AddSingleton <IHostLifetime, DockerLifeTime>(); });
return(builder);
}
public static Builder UseKuaidaili(this Builder builder)
{
builder.ConfigureServices(x => { x.AddSingleton <IProxySupplier, KuaidailiProxySupplier>(); });
return(builder);
}
