protected override void PreInitComponent(params string[] arguments)
{
base.PreInitComponent(arguments);
if (arguments.Contains("skip"))
{
return;
}
var entityProcessors = PageProcessors.Where(p => p is IEntityProcessor).ToList();
var entityPipelines = Pipelines.Where(p => p is BaseEntityPipeline).ToList();
if (entityProcessors.Count != 0 && entityPipelines.Count == 0)
{
throw new SpiderException("You may miss a entity pipeline.");
}
foreach (var processor in entityProcessors)
{
foreach (var pipeline in entityPipelines)
{
var entityProcessor = processor as IEntityProcessor;
BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline;
newPipeline.AddEntity(entityProcessor.EntityDefine);
}
}
}
protected override void PreInitComponent(params string[] arguments)
{
base.PreInitComponent(arguments);
if (Entities == null || Entities.Count == 0)
{
throw new SpiderException("Count of entity is zero.");
}
foreach (var entity in Entities)
{
foreach (var pipeline in Pipelines)
{
BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline;
newPipeline?.AddEntity(entity);
}
}
if (IfRequireInitStartRequests(arguments) && PrepareStartUrls != null)
{
for (int i = 0; i < PrepareStartUrls.Length; ++i)
{
var prepareStartUrl = PrepareStartUrls[i];
Logger.MyLog(Identity, $"[{i + 1}] Add extra start urls to scheduler.", LogLevel.Info);
prepareStartUrl.Build(this, null);
}
}
}
protected override void PreInitComponent(params string[] arguments)
{
base.PreInitComponent(arguments);
if (arguments.Contains("skip"))
{
return;
}
if (Entities == null || Entities.Count == 0)
{
throw new SpiderException("Count of entity is zero.");
}
foreach (var entity in Entities)
{
foreach (var pipeline in Pipelines)
{
BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline;
newPipeline?.AddEntity(entity);
}
}
if (IfRequireInitStartRequests(arguments) && StartUrlBuilders != null && StartUrlBuilders.Count > 0)
{
for (int i = 0; i < StartUrlBuilders.Count; ++i)
{
var builder = StartUrlBuilders[i];
Logger.MyLog(Identity, $"[{i + 1}] Add extra start urls to scheduler.", LogLevel.Info);
builder.Build(Site);
}
}
}
protected override void PreInitComponent(params string[] arguments)
{
base.PreInitComponent(arguments);
if (arguments.Contains("skip"))
{
return;
}
foreach (var processor in PageProcessors)
{
var entityProcessor = processor as IEntityProcessor;
if (entityProcessor != null)
{
foreach (var pipeline in Pipelines)
{
BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline;
newPipeline?.AddEntity(entityProcessor.EntityDefine);
}
}
}
if (IfRequireInitStartRequests(arguments) && StartUrlBuilders != null && StartUrlBuilders.Count > 0)
{
for (int i = 0; i < StartUrlBuilders.Count; ++i)
{
var builder = StartUrlBuilders[i];
Logger.MyLog(Identity, $"[{i + 1}] Add extra start urls to scheduler.", LogLevel.Info);
builder.Build(Site);
}
}
}
/// <summary>
/// 构造方法
/// </summary>
/// <param name="pipeline">数据管道</param>
public DownloadCache(BaseEntityPipeline pipeline)
{
if (pipeline == null)
{
_pipeline = BaseEntityPipeline.GetPipelineFromAppConfig() as BaseEntityDbPipeline;
}
else
{
_pipeline = pipeline;
}
if (_pipeline == null)
{
throw new SpiderException("StorageCache's pipeline unfound");
}
_pipeline.AddEntity(new EntityDefine <DownloadCacheData>());
_pipeline.Init();
}
public void GetPipelineFromAppConfig()
{
var configuration = ConfigurationManager.OpenMappedExeConfiguration(new ExeConfigurationFileMap
{
ExeConfigFilename = "app.config"
}, ConfigurationUserLevel.None);
var pipeline1 = BaseEntityPipeline.GetPipelineFromAppConfig(configuration.ConnectionStrings.ConnectionStrings["DataConnection"]);
Assert.True(pipeline1 is MySqlEntityPipeline);
var pipeline2 = BaseEntityPipeline.GetPipelineFromAppConfig(configuration.ConnectionStrings.ConnectionStrings["SqlServerDataConnection"]);
Assert.True(pipeline2 is SqlServerEntityPipeline);
var pipeline3 = BaseEntityPipeline.GetPipelineFromAppConfig(configuration.ConnectionStrings.ConnectionStrings["MongoDbDataConnection"]);
Assert.True(pipeline3 is MongoDbEntityPipeline);
}
protected override void InitPipelines(params string[] arguments)
{
if (Pipelines == null || Pipelines.Count == 0)
{
var defaultPipeline = GetDefaultPipeline();
if (defaultPipeline != null)
{
Pipelines.Add(defaultPipeline);
}
}
if (!arguments.Contains("skip"))
{
var entityProcessors = PageProcessors.Where(p => p is IEntityProcessor).ToList();
var entityPipelines = Pipelines.Where(p => p is BaseEntityPipeline).ToList();
if (entityProcessors.Count != 0 && entityPipelines.Count == 0)
{
throw new SpiderException("You may miss a entity pipeline.");
}
foreach (var processor in entityProcessors)
{
foreach (var pipeline in entityPipelines)
{
var entityProcessor = processor as IEntityProcessor;
BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline;
newPipeline.AddEntity(entityProcessor.EntityDefine);
}
}
}
if (PageProcessors == null || PageProcessors.Count == 0)
{
throw new SpiderException("Count of PageProcessor is zero.");
}
foreach (var pipeline in Pipelines)
{
pipeline.InitPipeline(this);
}
}
protected override void PreInitComponent(params string[] arguments)
{
base.PreInitComponent(arguments);
if (arguments.Contains("skip"))
{
return;
}
foreach (var processor in PageProcessors)
{
if (processor is IEntityProcessor entityProcessor)
{
foreach (var pipeline in Pipelines)
{
BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline;
newPipeline?.AddEntity(entityProcessor.EntityDefine);
}
}
}
}
protected override IPipeline GetDefaultPipeline()
{
return(BaseEntityPipeline.GetPipelineFromAppConfig());
}
protected override void PreInitComponent(params string[] arguments)
{
if (Entities == null || Entities.Count == 0)
{
throw new SpiderException("Count of entity is zero.");
}
if (EntityPipelines == null || EntityPipelines.Count == 0)
{
throw new SpiderException("Need at least one entity pipeline.");
}
foreach (var entity in Entities)
{
var pipelines = new List <BaseEntityPipeline>();
foreach (var pipeline in EntityPipelines)
{
BaseEntityPipeline newPipeline = pipeline.Clone();
newPipeline.InitEntity(entity);
if (newPipeline.IsEnabled)
{
pipelines.Add(newPipeline);
}
}
if (pipelines.Count > 0)
{
AddPipeline(new EntityPipeline(entity.Entity.Name, pipelines));
}
}
bool needInitStartRequest = true;
var redisConnectString = Configuration.GetValue("redis.connectString");
if (!string.IsNullOrEmpty(redisConnectString))
{
RedisConnection = Cache.Instance.Get(redisConnectString);
if (RedisConnection == null)
{
RedisConnection = new Infrastructure.RedisConnection(redisConnectString);
Cache.Instance.Set(redisConnectString, RedisConnection);
}
}
if (RedisConnection != null)
{
while (!RedisConnection.Database.LockTake(InitLockKey, "0", TimeSpan.FromMinutes(10)))
{
Thread.Sleep(1000);
}
var lockerValue = RedisConnection.Database.HashGet(InitStatusSetKey, Identity);
needInitStartRequest = lockerValue != "init finished";
}
if (arguments.Contains("rerun"))
{
Scheduler.Init(this);
Scheduler.Dispose();
if (RedisConnection != null)
{
RedisConnection.Database.HashDelete(ValidateStatusKey, Identity);
}
needInitStartRequest = true;
}
if (needInitStartRequest && PrepareStartUrls != null)
{
for (int i = 0; i < PrepareStartUrls.Length; ++i)
{
var prepareStartUrl = PrepareStartUrls[i];
this.Log($"[步骤 {i + 2}] 添加链接到调度中心.", LogLevel.Info);
prepareStartUrl.Build(this, null);
}
}
RegisterControl(this);
base.PreInitComponent();
}
public EntitySpider AddEntityPipeline(BaseEntityPipeline pipeline)
{
CheckIfRunning();
EntityPipelines.Add(pipeline);
return(this);
}
protected override void PreInitComponent(params string[] arguments)
{
base.PreInitComponent();
if (UseDbLog)
{
Monitor = new DbMonitor(Identity);
}
if (Site == null)
{
throw new SpiderException("Site should not be null.");
}
if (Entities == null || Entities.Count == 0)
{
throw new SpiderException("Count of entity is zero.");
}
foreach (var entity in Entities)
{
foreach (var pipeline in Pipelines)
{
BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline;
newPipeline?.AddEntity(entity);
}
}
bool needInitStartRequest = true;
if (RedisConnection.Default != null)
{
if (arguments.Contains("rerun"))
{
RedisConnection.Default.Database.HashDelete(InitStatusSetKey, Identity);
RedisConnection.Default.Database.LockRelease(InitLockKey, "0");
}
while (!RedisConnection.Default.Database.LockTake(InitLockKey, "0", TimeSpan.FromMinutes(10)))
{
Thread.Sleep(1000);
}
var lockerValue = RedisConnection.Default.Database.HashGet(InitStatusSetKey, Identity);
needInitStartRequest = lockerValue != "init finished";
}
Scheduler.Init(this);
if (arguments.Contains("rerun"))
{
Scheduler.Clean();
Scheduler.Dispose();
Verifier.RemoveVerifidationLock(Identity);
needInitStartRequest = true;
}
if (needInitStartRequest && PrepareStartUrls != null)
{
for (int i = 0; i < PrepareStartUrls.Length; ++i)
{
var prepareStartUrl = PrepareStartUrls[i];
Logger.MyLog(Identity, $"[步骤 {i + 2}] 添加链接到调度中心.", LogLevel.Info);
prepareStartUrl.Build(this, null);
}
}
RegisterControl(this);
}
