Exemplo n.º 1
0
        protected override void PreInitComponent(params string[] arguments)
        {
            base.PreInitComponent(arguments);

            if (arguments.Contains("skip"))
            {
                return;
            }

            var entityProcessors = PageProcessors.Where(p => p is IEntityProcessor).ToList();
            var entityPipelines  = Pipelines.Where(p => p is BaseEntityPipeline).ToList();

            if (entityProcessors.Count != 0 && entityPipelines.Count == 0)
            {
                throw new SpiderException("You may miss a entity pipeline.");
            }
            foreach (var processor in entityProcessors)
            {
                foreach (var pipeline in entityPipelines)
                {
                    var entityProcessor            = processor as IEntityProcessor;
                    BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline;
                    newPipeline.AddEntity(entityProcessor.EntityDefine);
                }
            }
        }
Exemplo n.º 2
0
        protected override void PreInitComponent(params string[] arguments)
        {
            base.PreInitComponent(arguments);

            if (Entities == null || Entities.Count == 0)
            {
                throw new SpiderException("Count of entity is zero.");
            }

            foreach (var entity in Entities)
            {
                foreach (var pipeline in Pipelines)
                {
                    BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline;
                    newPipeline?.AddEntity(entity);
                }
            }

            if (IfRequireInitStartRequests(arguments) && PrepareStartUrls != null)
            {
                for (int i = 0; i < PrepareStartUrls.Length; ++i)
                {
                    var prepareStartUrl = PrepareStartUrls[i];
                    Logger.MyLog(Identity, $"[{i + 1}] Add extra start urls to scheduler.", LogLevel.Info);
                    prepareStartUrl.Build(this, null);
                }
            }
        }
Exemplo n.º 3
0
        protected override void PreInitComponent(params string[] arguments)
        {
            base.PreInitComponent(arguments);

            if (arguments.Contains("skip"))
            {
                return;
            }

            if (Entities == null || Entities.Count == 0)
            {
                throw new SpiderException("Count of entity is zero.");
            }

            foreach (var entity in Entities)
            {
                foreach (var pipeline in Pipelines)
                {
                    BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline;
                    newPipeline?.AddEntity(entity);
                }
            }

            if (IfRequireInitStartRequests(arguments) && StartUrlBuilders != null && StartUrlBuilders.Count > 0)
            {
                for (int i = 0; i < StartUrlBuilders.Count; ++i)
                {
                    var builder = StartUrlBuilders[i];
                    Logger.MyLog(Identity, $"[{i + 1}] Add extra start urls to scheduler.", LogLevel.Info);
                    builder.Build(Site);
                }
            }
        }
Exemplo n.º 4
0
        protected override void PreInitComponent(params string[] arguments)
        {
            base.PreInitComponent(arguments);

            if (arguments.Contains("skip"))
            {
                return;
            }

            foreach (var processor in PageProcessors)
            {
                var entityProcessor = processor as IEntityProcessor;
                if (entityProcessor != null)
                {
                    foreach (var pipeline in Pipelines)
                    {
                        BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline;
                        newPipeline?.AddEntity(entityProcessor.EntityDefine);
                    }
                }
            }

            if (IfRequireInitStartRequests(arguments) && StartUrlBuilders != null && StartUrlBuilders.Count > 0)
            {
                for (int i = 0; i < StartUrlBuilders.Count; ++i)
                {
                    var builder = StartUrlBuilders[i];
                    Logger.MyLog(Identity, $"[{i + 1}] Add extra start urls to scheduler.", LogLevel.Info);
                    builder.Build(Site);
                }
            }
        }
Exemplo n.º 5
0
        /// <summary>
        /// 构造方法
        /// </summary>
        /// <param name="pipeline">数据管道</param>
        public DownloadCache(BaseEntityPipeline pipeline)
        {
            if (pipeline == null)
            {
                _pipeline = BaseEntityPipeline.GetPipelineFromAppConfig() as BaseEntityDbPipeline;
            }
            else
            {
                _pipeline = pipeline;
            }

            if (_pipeline == null)
            {
                throw new SpiderException("StorageCache's pipeline unfound");
            }
            _pipeline.AddEntity(new EntityDefine <DownloadCacheData>());
            _pipeline.Init();
        }
Exemplo n.º 6
0
        public void GetPipelineFromAppConfig()
        {
            var configuration = ConfigurationManager.OpenMappedExeConfiguration(new ExeConfigurationFileMap
            {
                ExeConfigFilename = "app.config"
            }, ConfigurationUserLevel.None);
            var pipeline1 = BaseEntityPipeline.GetPipelineFromAppConfig(configuration.ConnectionStrings.ConnectionStrings["DataConnection"]);

            Assert.True(pipeline1 is MySqlEntityPipeline);

            var pipeline2 = BaseEntityPipeline.GetPipelineFromAppConfig(configuration.ConnectionStrings.ConnectionStrings["SqlServerDataConnection"]);

            Assert.True(pipeline2 is SqlServerEntityPipeline);

            var pipeline3 = BaseEntityPipeline.GetPipelineFromAppConfig(configuration.ConnectionStrings.ConnectionStrings["MongoDbDataConnection"]);

            Assert.True(pipeline3 is MongoDbEntityPipeline);
        }
Exemplo n.º 7
0
        protected override void InitPipelines(params string[] arguments)
        {
            if (Pipelines == null || Pipelines.Count == 0)
            {
                var defaultPipeline = GetDefaultPipeline();
                if (defaultPipeline != null)
                {
                    Pipelines.Add(defaultPipeline);
                }
            }

            if (!arguments.Contains("skip"))
            {
                var entityProcessors = PageProcessors.Where(p => p is IEntityProcessor).ToList();
                var entityPipelines  = Pipelines.Where(p => p is BaseEntityPipeline).ToList();

                if (entityProcessors.Count != 0 && entityPipelines.Count == 0)
                {
                    throw new SpiderException("You may miss a entity pipeline.");
                }
                foreach (var processor in entityProcessors)
                {
                    foreach (var pipeline in entityPipelines)
                    {
                        var entityProcessor            = processor as IEntityProcessor;
                        BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline;
                        newPipeline.AddEntity(entityProcessor.EntityDefine);
                    }
                }
            }

            if (PageProcessors == null || PageProcessors.Count == 0)
            {
                throw new SpiderException("Count of PageProcessor is zero.");
            }

            foreach (var pipeline in Pipelines)
            {
                pipeline.InitPipeline(this);
            }
        }
Exemplo n.º 8
0
        protected override void PreInitComponent(params string[] arguments)
        {
            base.PreInitComponent(arguments);

            if (arguments.Contains("skip"))
            {
                return;
            }

            foreach (var processor in PageProcessors)
            {
                if (processor is IEntityProcessor entityProcessor)
                {
                    foreach (var pipeline in Pipelines)
                    {
                        BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline;

                        newPipeline?.AddEntity(entityProcessor.EntityDefine);
                    }
                }
            }
        }
Exemplo n.º 9
0
 protected override IPipeline GetDefaultPipeline()
 {
     return(BaseEntityPipeline.GetPipelineFromAppConfig());
 }
Exemplo n.º 10
0
        protected override void PreInitComponent(params string[] arguments)
        {
            if (Entities == null || Entities.Count == 0)
            {
                throw new SpiderException("Count of entity is zero.");
            }

            if (EntityPipelines == null || EntityPipelines.Count == 0)
            {
                throw new SpiderException("Need at least one entity pipeline.");
            }

            foreach (var entity in Entities)
            {
                var pipelines = new List <BaseEntityPipeline>();
                foreach (var pipeline in EntityPipelines)
                {
                    BaseEntityPipeline newPipeline = pipeline.Clone();
                    newPipeline.InitEntity(entity);
                    if (newPipeline.IsEnabled)
                    {
                        pipelines.Add(newPipeline);
                    }
                }
                if (pipelines.Count > 0)
                {
                    AddPipeline(new EntityPipeline(entity.Entity.Name, pipelines));
                }
            }

            bool needInitStartRequest = true;
            var  redisConnectString   = Configuration.GetValue("redis.connectString");

            if (!string.IsNullOrEmpty(redisConnectString))
            {
                RedisConnection = Cache.Instance.Get(redisConnectString);
                if (RedisConnection == null)
                {
                    RedisConnection = new Infrastructure.RedisConnection(redisConnectString);
                    Cache.Instance.Set(redisConnectString, RedisConnection);
                }
            }


            if (RedisConnection != null)
            {
                while (!RedisConnection.Database.LockTake(InitLockKey, "0", TimeSpan.FromMinutes(10)))
                {
                    Thread.Sleep(1000);
                }
                var lockerValue = RedisConnection.Database.HashGet(InitStatusSetKey, Identity);
                needInitStartRequest = lockerValue != "init finished";
            }

            if (arguments.Contains("rerun"))
            {
                Scheduler.Init(this);
                Scheduler.Dispose();
                if (RedisConnection != null)
                {
                    RedisConnection.Database.HashDelete(ValidateStatusKey, Identity);
                }
                needInitStartRequest = true;
            }

            if (needInitStartRequest && PrepareStartUrls != null)
            {
                for (int i = 0; i < PrepareStartUrls.Length; ++i)
                {
                    var prepareStartUrl = PrepareStartUrls[i];
                    this.Log($"[步骤 {i + 2}] 添加链接到调度中心.", LogLevel.Info);
                    prepareStartUrl.Build(this, null);
                }
            }

            RegisterControl(this);

            base.PreInitComponent();
        }
Exemplo n.º 11
0
 public EntitySpider AddEntityPipeline(BaseEntityPipeline pipeline)
 {
     CheckIfRunning();
     EntityPipelines.Add(pipeline);
     return(this);
 }
Exemplo n.º 12
0
        protected override void PreInitComponent(params string[] arguments)
        {
            base.PreInitComponent();

            if (UseDbLog)
            {
                Monitor = new DbMonitor(Identity);
            }

            if (Site == null)
            {
                throw new SpiderException("Site should not be null.");
            }

            if (Entities == null || Entities.Count == 0)
            {
                throw new SpiderException("Count of entity is zero.");
            }

            foreach (var entity in Entities)
            {
                foreach (var pipeline in Pipelines)
                {
                    BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline;
                    newPipeline?.AddEntity(entity);
                }
            }

            bool needInitStartRequest = true;

            if (RedisConnection.Default != null)
            {
                if (arguments.Contains("rerun"))
                {
                    RedisConnection.Default.Database.HashDelete(InitStatusSetKey, Identity);
                    RedisConnection.Default.Database.LockRelease(InitLockKey, "0");
                }
                while (!RedisConnection.Default.Database.LockTake(InitLockKey, "0", TimeSpan.FromMinutes(10)))
                {
                    Thread.Sleep(1000);
                }
                var lockerValue = RedisConnection.Default.Database.HashGet(InitStatusSetKey, Identity);
                needInitStartRequest = lockerValue != "init finished";
            }

            Scheduler.Init(this);

            if (arguments.Contains("rerun"))
            {
                Scheduler.Clean();
                Scheduler.Dispose();
                Verifier.RemoveVerifidationLock(Identity);
                needInitStartRequest = true;
            }

            if (needInitStartRequest && PrepareStartUrls != null)
            {
                for (int i = 0; i < PrepareStartUrls.Length; ++i)
                {
                    var prepareStartUrl = PrepareStartUrls[i];
                    Logger.MyLog(Identity, $"[步骤 {i + 2}] 添加链接到调度中心.", LogLevel.Info);
                    prepareStartUrl.Build(this, null);
                }
            }

            RegisterControl(this);
        }