internal static StorageBase GetDefaultStorage(ISpiderOptions options)
{
var type = Type.GetType(options.Storage);
if (type == null)
{
throw new SpiderException("存储器类型配置不正确，或者未添加对应的库");
}
if (!typeof(StorageBase).IsAssignableFrom(type))
{
throw new SpiderException("存储器类型配置不正确");
}
var method = type.GetMethod("CreateFromOptions");
if (method == null)
{
throw new SpiderException("存储器未实现 CreateFromOptions 方法，无法自动创建");
}
var storage = method.Invoke(null, new object[] { options });
if (storage == null)
{
throw new SpiderException("创建默认存储器失败");
}
return((StorageBase)storage);
}
internal static StorageBase GetDefaultStorage(ISpiderOptions options)
{
var type = Type.GetType(options.Storage);
if (type == null)
{
throw new SpiderException("The Storage type is not configured correctly, or the corresponding library is not added.");
}
if (!typeof(StorageBase).IsAssignableFrom(type))
{
throw new SpiderException("Storage type configuration is incorrect");
}
var method = type.GetMethod("CreateFromOptions");
if (method == null)
{
throw new SpiderException("The Storage does not implement the CreateFromOptions method and cannot be created automatically");
}
var storage = method.Invoke(null, new object[] { options });
if (storage == null)
{
throw new SpiderException("Failed to create default storage");
}
return((StorageBase)storage);
}
/// <summary>
/// 根据配置返回存储器
/// </summary>
/// <param name="options">配置</param>
/// <returns></returns>
public new static PostgreSqlEntityStorage CreateFromOptions(ISpiderOptions options)
{
return(new PostgreSqlEntityStorage(options.StorageType, options.StorageConnectionString)
{
IgnoreCase = options.StorageIgnoreCase,
RetryTimes = options.StorageRetryTimes,
UseTransaction = options.StorageUseTransaction
});
}
/// <summary>
/// 构造方法
/// </summary>
/// <param name="mq">消息队列</param>
/// <param name="downloaderAgentStore">下载器代理存储</param>
/// <param name="options">系统选项</param>
/// <param name="logger">日志接口</param>
protected DownloadCenterBase(
IMessageQueue mq,
IDownloaderAgentStore downloaderAgentStore,
ISpiderOptions options,
ILogger logger)
{
Mq = mq;
DownloaderAgentStore = downloaderAgentStore;
Logger = logger;
Options = options;
}
/// <summary>
/// 根据配置返回存储器
/// </summary>
/// <param name="options">配置</param>
/// <returns></returns>
public static SqlServerEntityStorage CreateFromOptions(ISpiderOptions options)
{
var storage = new SqlServerEntityStorage(options.StorageType, options.StorageConnectionString)
{
IgnoreCase = options.StorageIgnoreCase,
RetryTimes = options.StorageRetryTimes,
UseTransaction = options.StorageUseTransaction
};
return(storage);
}
/// <summary>
/// 根据配置返回存储器
/// </summary>
/// <param name="options">配置</param>
/// <returns></returns>
public static MySqlFileEntityStorage CreateFromOptions(ISpiderOptions options)
{
var fileType = string.IsNullOrWhiteSpace(options.MySqlFileType)
? MySqlFileType.InsertSql
: (MySqlFileType)Enum.Parse(typeof(MySqlFileType),
options.MySqlFileType);
return(new MySqlFileEntityStorage(fileType)
{
IgnoreCase = options.StorageIgnoreCase
});
}
/// <summary>
/// 构造方法
/// </summary>
/// <param name="mq"></param>
/// <param name="options"></param>
/// <param name="logger"></param>
/// <param name="services">服务提供接口</param>
/// <param name="statisticsService"></param>
public Spider(
IMessageQueue mq,
IStatisticsService statisticsService,
ISpiderOptions options,
ILogger <Spider> logger,
IServiceProvider services)
{
_services = services;
_statisticsService = statisticsService;
_mq = mq;
_options = options;
_logger = logger;
Console.CancelKeyPress += ConsoleCancelKeyPress;
}
/// <summary>
/// 构造方法
/// </summary>
/// <param name="options">爬虫选项</param>
/// <param name="logger">日志接口</param>
public KafkaMessageQueue(ISpiderOptions options,
ILogger <KafkaMessageQueue> logger)
{
_logger = logger;
_config = new ConsumerConfig
{
GroupId = options.KafkaConsumerGroup,
BootstrapServers = options.KafkaBootstrapServers,
// Note: The AutoOffsetReset property determines the start offset in the event
// there are not yet any committed offsets for the consumer group for the
// topic/partitions of interest. By default, offsets are committed
// automatically, so in this example, consumption will only start from the
// earliest message in the topic 'my-topic' the first time you run the program.
AutoOffsetReset = AutoOffsetReset.Earliest
};
var productConfig = new ProducerConfig {
BootstrapServers = options.KafkaBootstrapServers
};
_producer = new ProducerBuilder <Null, string>(productConfig).Build();
}
/// <summary>
/// 根据配置返回存储器
/// </summary>
/// <param name="options"></param>
/// <returns></returns>
public static ConsoleEntityStorage CreateFromOptions(ISpiderOptions options)
{
return(new ConsoleEntityStorage());
}
public CnblogsSpider(IMessageQueue mq, IStatisticsService statisticsService, ISpiderOptions options, ILogger <Spider> logger, IServiceProvider services) : base(mq, statisticsService, options, logger, services)
{
}
public GithubSpider(IDynamicMessageQueue dmq, IMessageQueue mq, IStatisticsService statisticsService, ISpiderOptions options,
ILogger <Spider> logger, IServiceProvider services) : base(dmq, mq, statisticsService, options, logger, services)
{
}
public MySqlStatisticsStore(ISpiderOptions options)
{
_options = options;
}
public MySqlDownloaderAgentStore(ISpiderOptions options)
{
_options = options;
}
/// <summary>
/// 根据配置返回存储器
/// </summary>
/// <param name="options">配置</param>
/// <returns></returns>
public static JsonEntityFileStorage CreateFromOptions(ISpiderOptions options)
{
return(new JsonEntityFileStorage());
}
public static MongoEntityStorage CreateFromOptions(ISpiderOptions options)
{
return(new MongoEntityStorage(options.ConnectionString));
}
/// <summary>
/// 构造方法
/// </summary>
/// <param name="mq">消息队列</param>
/// <param name="downloaderAgentStore">下载器代理存储</param>
/// <param name="options">系统选项</param>
/// <param name="logger">日志接口</param>
public DownloadCenter(IMessageQueue mq, IDownloaderAgentStore downloaderAgentStore, ISpiderOptions options,
ILogger logger) : base(mq, downloaderAgentStore, options, logger)
{
}
public static FileStorage CreateFromOptions(ISpiderOptions options)
{
return(new FileStorage());
}
/// <summary>
/// 构造方法
/// </summary>
/// <param name="mq">消息队列</param>
/// <param name="downloaderAgentStore">下载器代理存储</param>
/// <param name="options">系统选项</param>
/// <param name="logger">日志接口</param>
public LocalDownloadCenter(IDynamicMessageQueue dmq, IMessageQueue mq,
IDownloaderAgentStore downloaderAgentStore, ISpiderOptions options,
ILogger <LocalDownloadCenter> logger) : base(dmq, mq, downloaderAgentStore, options, logger)
{
}
