protected override void PreInitComponent(params string[] arguments) { base.PreInitComponent(arguments); if (arguments.Contains("skip")) { return; } var entityProcessors = PageProcessors.Where(p => p is IEntityProcessor).ToList(); var entityPipelines = Pipelines.Where(p => p is BaseEntityPipeline).ToList(); if (entityProcessors.Count != 0 && entityPipelines.Count == 0) { throw new SpiderException("You may miss a entity pipeline."); } foreach (var processor in entityProcessors) { foreach (var pipeline in entityPipelines) { var entityProcessor = processor as IEntityProcessor; BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline; newPipeline.AddEntity(entityProcessor.EntityDefine); } } }
protected override void PreInitComponent(params string[] arguments) { base.PreInitComponent(arguments); if (Entities == null || Entities.Count == 0) { throw new SpiderException("Count of entity is zero."); } foreach (var entity in Entities) { foreach (var pipeline in Pipelines) { BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline; newPipeline?.AddEntity(entity); } } if (IfRequireInitStartRequests(arguments) && PrepareStartUrls != null) { for (int i = 0; i < PrepareStartUrls.Length; ++i) { var prepareStartUrl = PrepareStartUrls[i]; Logger.MyLog(Identity, $"[{i + 1}] Add extra start urls to scheduler.", LogLevel.Info); prepareStartUrl.Build(this, null); } } }
protected override void PreInitComponent(params string[] arguments) { base.PreInitComponent(arguments); if (arguments.Contains("skip")) { return; } if (Entities == null || Entities.Count == 0) { throw new SpiderException("Count of entity is zero."); } foreach (var entity in Entities) { foreach (var pipeline in Pipelines) { BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline; newPipeline?.AddEntity(entity); } } if (IfRequireInitStartRequests(arguments) && StartUrlBuilders != null && StartUrlBuilders.Count > 0) { for (int i = 0; i < StartUrlBuilders.Count; ++i) { var builder = StartUrlBuilders[i]; Logger.MyLog(Identity, $"[{i + 1}] Add extra start urls to scheduler.", LogLevel.Info); builder.Build(Site); } } }
protected override void PreInitComponent(params string[] arguments) { base.PreInitComponent(arguments); if (arguments.Contains("skip")) { return; } foreach (var processor in PageProcessors) { var entityProcessor = processor as IEntityProcessor; if (entityProcessor != null) { foreach (var pipeline in Pipelines) { BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline; newPipeline?.AddEntity(entityProcessor.EntityDefine); } } } if (IfRequireInitStartRequests(arguments) && StartUrlBuilders != null && StartUrlBuilders.Count > 0) { for (int i = 0; i < StartUrlBuilders.Count; ++i) { var builder = StartUrlBuilders[i]; Logger.MyLog(Identity, $"[{i + 1}] Add extra start urls to scheduler.", LogLevel.Info); builder.Build(Site); } } }
/// <summary> /// 构造方法 /// </summary> /// <param name="pipeline">数据管道</param> public DownloadCache(BaseEntityPipeline pipeline) { if (pipeline == null) { _pipeline = BaseEntityPipeline.GetPipelineFromAppConfig() as BaseEntityDbPipeline; } else { _pipeline = pipeline; } if (_pipeline == null) { throw new SpiderException("StorageCache's pipeline unfound"); } _pipeline.AddEntity(new EntityDefine <DownloadCacheData>()); _pipeline.Init(); }
public void GetPipelineFromAppConfig() { var configuration = ConfigurationManager.OpenMappedExeConfiguration(new ExeConfigurationFileMap { ExeConfigFilename = "app.config" }, ConfigurationUserLevel.None); var pipeline1 = BaseEntityPipeline.GetPipelineFromAppConfig(configuration.ConnectionStrings.ConnectionStrings["DataConnection"]); Assert.True(pipeline1 is MySqlEntityPipeline); var pipeline2 = BaseEntityPipeline.GetPipelineFromAppConfig(configuration.ConnectionStrings.ConnectionStrings["SqlServerDataConnection"]); Assert.True(pipeline2 is SqlServerEntityPipeline); var pipeline3 = BaseEntityPipeline.GetPipelineFromAppConfig(configuration.ConnectionStrings.ConnectionStrings["MongoDbDataConnection"]); Assert.True(pipeline3 is MongoDbEntityPipeline); }
protected override void InitPipelines(params string[] arguments) { if (Pipelines == null || Pipelines.Count == 0) { var defaultPipeline = GetDefaultPipeline(); if (defaultPipeline != null) { Pipelines.Add(defaultPipeline); } } if (!arguments.Contains("skip")) { var entityProcessors = PageProcessors.Where(p => p is IEntityProcessor).ToList(); var entityPipelines = Pipelines.Where(p => p is BaseEntityPipeline).ToList(); if (entityProcessors.Count != 0 && entityPipelines.Count == 0) { throw new SpiderException("You may miss a entity pipeline."); } foreach (var processor in entityProcessors) { foreach (var pipeline in entityPipelines) { var entityProcessor = processor as IEntityProcessor; BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline; newPipeline.AddEntity(entityProcessor.EntityDefine); } } } if (PageProcessors == null || PageProcessors.Count == 0) { throw new SpiderException("Count of PageProcessor is zero."); } foreach (var pipeline in Pipelines) { pipeline.InitPipeline(this); } }
protected override void PreInitComponent(params string[] arguments) { base.PreInitComponent(arguments); if (arguments.Contains("skip")) { return; } foreach (var processor in PageProcessors) { if (processor is IEntityProcessor entityProcessor) { foreach (var pipeline in Pipelines) { BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline; newPipeline?.AddEntity(entityProcessor.EntityDefine); } } } }
protected override IPipeline GetDefaultPipeline() { return(BaseEntityPipeline.GetPipelineFromAppConfig()); }
protected override void PreInitComponent(params string[] arguments) { if (Entities == null || Entities.Count == 0) { throw new SpiderException("Count of entity is zero."); } if (EntityPipelines == null || EntityPipelines.Count == 0) { throw new SpiderException("Need at least one entity pipeline."); } foreach (var entity in Entities) { var pipelines = new List <BaseEntityPipeline>(); foreach (var pipeline in EntityPipelines) { BaseEntityPipeline newPipeline = pipeline.Clone(); newPipeline.InitEntity(entity); if (newPipeline.IsEnabled) { pipelines.Add(newPipeline); } } if (pipelines.Count > 0) { AddPipeline(new EntityPipeline(entity.Entity.Name, pipelines)); } } bool needInitStartRequest = true; var redisConnectString = Configuration.GetValue("redis.connectString"); if (!string.IsNullOrEmpty(redisConnectString)) { RedisConnection = Cache.Instance.Get(redisConnectString); if (RedisConnection == null) { RedisConnection = new Infrastructure.RedisConnection(redisConnectString); Cache.Instance.Set(redisConnectString, RedisConnection); } } if (RedisConnection != null) { while (!RedisConnection.Database.LockTake(InitLockKey, "0", TimeSpan.FromMinutes(10))) { Thread.Sleep(1000); } var lockerValue = RedisConnection.Database.HashGet(InitStatusSetKey, Identity); needInitStartRequest = lockerValue != "init finished"; } if (arguments.Contains("rerun")) { Scheduler.Init(this); Scheduler.Dispose(); if (RedisConnection != null) { RedisConnection.Database.HashDelete(ValidateStatusKey, Identity); } needInitStartRequest = true; } if (needInitStartRequest && PrepareStartUrls != null) { for (int i = 0; i < PrepareStartUrls.Length; ++i) { var prepareStartUrl = PrepareStartUrls[i]; this.Log($"[步骤 {i + 2}] 添加链接到调度中心.", LogLevel.Info); prepareStartUrl.Build(this, null); } } RegisterControl(this); base.PreInitComponent(); }
public EntitySpider AddEntityPipeline(BaseEntityPipeline pipeline) { CheckIfRunning(); EntityPipelines.Add(pipeline); return(this); }
protected override void PreInitComponent(params string[] arguments) { base.PreInitComponent(); if (UseDbLog) { Monitor = new DbMonitor(Identity); } if (Site == null) { throw new SpiderException("Site should not be null."); } if (Entities == null || Entities.Count == 0) { throw new SpiderException("Count of entity is zero."); } foreach (var entity in Entities) { foreach (var pipeline in Pipelines) { BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline; newPipeline?.AddEntity(entity); } } bool needInitStartRequest = true; if (RedisConnection.Default != null) { if (arguments.Contains("rerun")) { RedisConnection.Default.Database.HashDelete(InitStatusSetKey, Identity); RedisConnection.Default.Database.LockRelease(InitLockKey, "0"); } while (!RedisConnection.Default.Database.LockTake(InitLockKey, "0", TimeSpan.FromMinutes(10))) { Thread.Sleep(1000); } var lockerValue = RedisConnection.Default.Database.HashGet(InitStatusSetKey, Identity); needInitStartRequest = lockerValue != "init finished"; } Scheduler.Init(this); if (arguments.Contains("rerun")) { Scheduler.Clean(); Scheduler.Dispose(); Verifier.RemoveVerifidationLock(Identity); needInitStartRequest = true; } if (needInitStartRequest && PrepareStartUrls != null) { for (int i = 0; i < PrepareStartUrls.Length; ++i) { var prepareStartUrl = PrepareStartUrls[i]; Logger.MyLog(Identity, $"[步骤 {i + 2}] 添加链接到调度中心.", LogLevel.Info); prepareStartUrl.Build(this, null); } } RegisterControl(this); }