internal override void AddEntity(IEntityDefine entityDefine)
{
if (entityDefine == null)
{
throw new ArgumentException("Should not add a null entity to a entity dabase pipeline.");
}
if (!typeof(CassandraSpiderEntity).IsAssignableFrom(entityDefine.Type))
{
throw new ArgumentException("Cassandra pipeline only support CassandraSpiderEntity.");
}
if (entityDefine.TableInfo == null)
{
Logger.MyLog(Spider?.Identity, $"Schema is necessary, Skip {GetType().Name} for {entityDefine.Name}.", LogLevel.Warn);
return;
}
EntityAdapter entityAdapter = new EntityAdapter(entityDefine.TableInfo, entityDefine.Columns);
if (entityAdapter.Table.UpdateColumns != null && entityAdapter.Table.UpdateColumns.Length > 0)
{
entityAdapter.PipelineMode = PipelineMode.Update;
}
else
{
entityAdapter.PipelineMode = DefaultPipelineModel;
}
InitAllCqlOfEntity(entityAdapter);
EntityAdapters.TryAdd(entityDefine.Name, entityAdapter);
}
internal override void AddEntity(IEntityDefine metadata)
{
if (metadata.TableInfo == null)
{
Logger.MyLog(Spider?.Identity, $"Schema is necessary, skip {GetType().Name} for {metadata.Name}.", LogLevel.Warn);
return;
}
MongoClient client = new MongoClient(ConnectString);
var db = client.GetDatabase(metadata.TableInfo.Database);
Collections.TryAdd(metadata.Name, db.GetCollection <BsonDocument>(metadata.TableInfo.CalculateTableName()));
}
/// <summary>
/// 添加爬虫实体类的定义
/// </summary>
/// <param name="entityDefine">爬虫实体类的定义</param>
public override void AddEntity(IEntityDefine entityDefine)
{
if (entityDefine.TableInfo == null)
{
Logger.Log($"Schema is necessary, skip {GetType().Name} for {entityDefine.Name}.", Level.Warn);
return;
}
MongoClient client = new MongoClient(_connectString);
var db = client.GetDatabase(entityDefine.TableInfo.Database);
_collections.TryAdd(entityDefine.Name, db.GetCollection <BsonDocument>(entityDefine.TableInfo.CalculateTableName()));
}
/// <summary>
/// 添加爬虫实体类的定义
/// </summary>
/// <param name="entityDefine">爬虫实体类的定义</param>
public virtual void AddEntity(IEntityDefine entityDefine)
{
if (entityDefine == null)
{
throw new ArgumentException("Should not add a null entity to a entity dabase pipeline.");
}
if (entityDefine.TableInfo == null)
{
Logger.Log($"Schema is necessary, Skip {GetType().Name} for {entityDefine.Name}.", Level.Warn);
return;
}
EntityAdapter entityAdapter = new EntityAdapter(entityDefine.TableInfo, entityDefine.Columns);
EntityAdapters.TryAdd(entityDefine.Name, entityAdapter);
}
public abstract void AddEntity(IEntityDefine type);
public override void AddEntity(IEntityDefine metadata)
{
}
internal override void AddEntity(IEntityDefine type)
{
}
/// <summary>
/// 添加爬虫实体类的定义
/// </summary>
/// <param name="entityDefine">爬虫实体类的定义</param>
public override void AddEntity(IEntityDefine entityDefine)
{
}
internal abstract void AddEntity(IEntityDefine type);
internal override void AddEntity(IEntityDefine metadata)
{
}
