internal static StorageBase GetDefaultStorage(ISpiderOptions options) { var type = Type.GetType(options.Storage); if (type == null) { throw new SpiderException("存储器类型配置不正确,或者未添加对应的库"); } if (!typeof(StorageBase).IsAssignableFrom(type)) { throw new SpiderException("存储器类型配置不正确"); } var method = type.GetMethod("CreateFromOptions"); if (method == null) { throw new SpiderException("存储器未实现 CreateFromOptions 方法,无法自动创建"); } var storage = method.Invoke(null, new object[] { options }); if (storage == null) { throw new SpiderException("创建默认存储器失败"); } return((StorageBase)storage); }
internal static StorageBase GetDefaultStorage(ISpiderOptions options) { var type = Type.GetType(options.Storage); if (type == null) { throw new SpiderException("The Storage type is not configured correctly, or the corresponding library is not added."); } if (!typeof(StorageBase).IsAssignableFrom(type)) { throw new SpiderException("Storage type configuration is incorrect"); } var method = type.GetMethod("CreateFromOptions"); if (method == null) { throw new SpiderException("The Storage does not implement the CreateFromOptions method and cannot be created automatically"); } var storage = method.Invoke(null, new object[] { options }); if (storage == null) { throw new SpiderException("Failed to create default storage"); } return((StorageBase)storage); }
/// <summary> /// 根据配置返回存储器 /// </summary> /// <param name="options">配置</param> /// <returns></returns> public new static PostgreSqlEntityStorage CreateFromOptions(ISpiderOptions options) { return(new PostgreSqlEntityStorage(options.StorageType, options.StorageConnectionString) { IgnoreCase = options.StorageIgnoreCase, RetryTimes = options.StorageRetryTimes, UseTransaction = options.StorageUseTransaction }); }
/// <summary> /// 构造方法 /// </summary> /// <param name="mq">消息队列</param> /// <param name="downloaderAgentStore">下载器代理存储</param> /// <param name="options">系统选项</param> /// <param name="logger">日志接口</param> protected DownloadCenterBase( IMessageQueue mq, IDownloaderAgentStore downloaderAgentStore, ISpiderOptions options, ILogger logger) { Mq = mq; DownloaderAgentStore = downloaderAgentStore; Logger = logger; Options = options; }
/// <summary> /// 根据配置返回存储器 /// </summary> /// <param name="options">配置</param> /// <returns></returns> public static SqlServerEntityStorage CreateFromOptions(ISpiderOptions options) { var storage = new SqlServerEntityStorage(options.StorageType, options.StorageConnectionString) { IgnoreCase = options.StorageIgnoreCase, RetryTimes = options.StorageRetryTimes, UseTransaction = options.StorageUseTransaction }; return(storage); }
/// <summary> /// 根据配置返回存储器 /// </summary> /// <param name="options">配置</param> /// <returns></returns> public static MySqlFileEntityStorage CreateFromOptions(ISpiderOptions options) { var fileType = string.IsNullOrWhiteSpace(options.MySqlFileType) ? MySqlFileType.InsertSql : (MySqlFileType)Enum.Parse(typeof(MySqlFileType), options.MySqlFileType); return(new MySqlFileEntityStorage(fileType) { IgnoreCase = options.StorageIgnoreCase }); }
/// <summary> /// 构造方法 /// </summary> /// <param name="mq"></param> /// <param name="options"></param> /// <param name="logger"></param> /// <param name="services">服务提供接口</param> /// <param name="statisticsService"></param> public Spider( IMessageQueue mq, IStatisticsService statisticsService, ISpiderOptions options, ILogger <Spider> logger, IServiceProvider services) { _services = services; _statisticsService = statisticsService; _mq = mq; _options = options; _logger = logger; Console.CancelKeyPress += ConsoleCancelKeyPress; }
/// <summary> /// 构造方法 /// </summary> /// <param name="options">爬虫选项</param> /// <param name="logger">日志接口</param> public KafkaMessageQueue(ISpiderOptions options, ILogger <KafkaMessageQueue> logger) { _logger = logger; _config = new ConsumerConfig { GroupId = options.KafkaConsumerGroup, BootstrapServers = options.KafkaBootstrapServers, // Note: The AutoOffsetReset property determines the start offset in the event // there are not yet any committed offsets for the consumer group for the // topic/partitions of interest. By default, offsets are committed // automatically, so in this example, consumption will only start from the // earliest message in the topic 'my-topic' the first time you run the program. AutoOffsetReset = AutoOffsetReset.Earliest }; var productConfig = new ProducerConfig { BootstrapServers = options.KafkaBootstrapServers }; _producer = new ProducerBuilder <Null, string>(productConfig).Build(); }
/// <summary> /// 根据配置返回存储器 /// </summary> /// <param name="options"></param> /// <returns></returns> public static ConsoleEntityStorage CreateFromOptions(ISpiderOptions options) { return(new ConsoleEntityStorage()); }
public CnblogsSpider(IMessageQueue mq, IStatisticsService statisticsService, ISpiderOptions options, ILogger <Spider> logger, IServiceProvider services) : base(mq, statisticsService, options, logger, services) { }
public GithubSpider(IDynamicMessageQueue dmq, IMessageQueue mq, IStatisticsService statisticsService, ISpiderOptions options, ILogger <Spider> logger, IServiceProvider services) : base(dmq, mq, statisticsService, options, logger, services) { }
public MySqlStatisticsStore(ISpiderOptions options) { _options = options; }
public MySqlDownloaderAgentStore(ISpiderOptions options) { _options = options; }
/// <summary> /// 根据配置返回存储器 /// </summary> /// <param name="options">配置</param> /// <returns></returns> public static JsonEntityFileStorage CreateFromOptions(ISpiderOptions options) { return(new JsonEntityFileStorage()); }
public static MongoEntityStorage CreateFromOptions(ISpiderOptions options) { return(new MongoEntityStorage(options.ConnectionString)); }
/// <summary> /// 构造方法 /// </summary> /// <param name="mq">消息队列</param> /// <param name="downloaderAgentStore">下载器代理存储</param> /// <param name="options">系统选项</param> /// <param name="logger">日志接口</param> public DownloadCenter(IMessageQueue mq, IDownloaderAgentStore downloaderAgentStore, ISpiderOptions options, ILogger logger) : base(mq, downloaderAgentStore, options, logger) { }
public static FileStorage CreateFromOptions(ISpiderOptions options) { return(new FileStorage()); }
/// <summary> /// 构造方法 /// </summary> /// <param name="mq">消息队列</param> /// <param name="downloaderAgentStore">下载器代理存储</param> /// <param name="options">系统选项</param> /// <param name="logger">日志接口</param> public LocalDownloadCenter(IDynamicMessageQueue dmq, IMessageQueue mq, IDownloaderAgentStore downloaderAgentStore, ISpiderOptions options, ILogger <LocalDownloadCenter> logger) : base(dmq, mq, downloaderAgentStore, options, logger) { }