internal override void AddEntity(IEntityDefine entityDefine) { if (entityDefine == null) { throw new ArgumentException("Should not add a null entity to a entity dabase pipeline."); } if (!typeof(CassandraSpiderEntity).IsAssignableFrom(entityDefine.Type)) { throw new ArgumentException("Cassandra pipeline only support CassandraSpiderEntity."); } if (entityDefine.TableInfo == null) { Logger.MyLog(Spider?.Identity, $"Schema is necessary, Skip {GetType().Name} for {entityDefine.Name}.", LogLevel.Warn); return; } EntityAdapter entityAdapter = new EntityAdapter(entityDefine.TableInfo, entityDefine.Columns); if (entityAdapter.Table.UpdateColumns != null && entityAdapter.Table.UpdateColumns.Length > 0) { entityAdapter.PipelineMode = PipelineMode.Update; } else { entityAdapter.PipelineMode = DefaultPipelineModel; } InitAllCqlOfEntity(entityAdapter); EntityAdapters.TryAdd(entityDefine.Name, entityAdapter); }
internal override void AddEntity(IEntityDefine metadata) { if (metadata.TableInfo == null) { Logger.MyLog(Spider?.Identity, $"Schema is necessary, skip {GetType().Name} for {metadata.Name}.", LogLevel.Warn); return; } MongoClient client = new MongoClient(ConnectString); var db = client.GetDatabase(metadata.TableInfo.Database); Collections.TryAdd(metadata.Name, db.GetCollection <BsonDocument>(metadata.TableInfo.CalculateTableName())); }
/// <summary> /// 添加爬虫实体类的定义 /// </summary> /// <param name="entityDefine">爬虫实体类的定义</param> public override void AddEntity(IEntityDefine entityDefine) { if (entityDefine.TableInfo == null) { Logger.Log($"Schema is necessary, skip {GetType().Name} for {entityDefine.Name}.", Level.Warn); return; } MongoClient client = new MongoClient(_connectString); var db = client.GetDatabase(entityDefine.TableInfo.Database); _collections.TryAdd(entityDefine.Name, db.GetCollection <BsonDocument>(entityDefine.TableInfo.CalculateTableName())); }
/// <summary> /// 添加爬虫实体类的定义 /// </summary> /// <param name="entityDefine">爬虫实体类的定义</param> public virtual void AddEntity(IEntityDefine entityDefine) { if (entityDefine == null) { throw new ArgumentException("Should not add a null entity to a entity dabase pipeline."); } if (entityDefine.TableInfo == null) { Logger.Log($"Schema is necessary, Skip {GetType().Name} for {entityDefine.Name}.", Level.Warn); return; } EntityAdapter entityAdapter = new EntityAdapter(entityDefine.TableInfo, entityDefine.Columns); EntityAdapters.TryAdd(entityDefine.Name, entityAdapter); }
public abstract void AddEntity(IEntityDefine type);
public override void AddEntity(IEntityDefine metadata) { }
internal override void AddEntity(IEntityDefine type) { }
/// <summary> /// 添加爬虫实体类的定义 /// </summary> /// <param name="entityDefine">爬虫实体类的定义</param> public override void AddEntity(IEntityDefine entityDefine) { }
internal abstract void AddEntity(IEntityDefine type);
internal override void AddEntity(IEntityDefine metadata) { }