private EntitySpider GetEntitySpider()
{
EntitySpider context = new DefaultEntitySpider(Site)
{
CachedSize = CachedSize,
Deep = Deep,
Downloader = GetDownloader(Downloader),
EmptySleepTime = EmptySleepTime,
SkipWhenResultIsEmpty = SkipWhenResultIsEmpty,
Scheduler = GetScheduler(Scheduler),
ThreadNum = ThreadNum,
Entities = Entities
};
context.Name = Name;
context.Identity = Identity;
context.AddPipelines(GetPipepines(Pipelines));
context.RedisConnectString = RedisConnectString;
context.ConnectString = ConnectString;
foreach (var entity in Entities)
{
EntityProcessor processor = new EntityProcessor(Site, entity);
context.AddPageProcessor(processor);
}
return(context);
}
public void ColumnOfIndexesIsInt()
{
EntitySpider context = new DefaultEntitySpider();
context.Identity = (Guid.NewGuid().ToString("N"));
context.ThreadNum = 1;
context.AddPipeline(new MySqlEntityPipeline("Database='mysql';Data Source=localhost;User ID=root;Password=;Port=3306"));
context.AddEntityType <Entity16>();
}
public void SqlServerDataTypeTests()
{
#if !NET45
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
return;
}
#endif
using (var conn = new SqlConnection("Server=.\\SQLEXPRESS;Database=master;Trusted_Connection=True;MultipleActiveResultSets=true"))
{
try
{
conn.Execute("create database test;");
}
catch
{
}
EntitySpider context = new DefaultEntitySpider();
context.Identity = (Guid.NewGuid().ToString("N"));
context.ThreadNum = 1;
context.AddPipeline(new SqlServerEntityPipeline("Server=.\\SQLEXPRESS;Database=master;Trusted_Connection=True;MultipleActiveResultSets=true"));
context.AddStartUrl("http://baidu.com");
context.AddEntityType <Entity15>();
context.Run("running-test");
var columns = conn.Query <ColumnInfo>("USE [test];select b.name Name,c.name+'(' + cast(c.length as varchar)+')' [Type] from sysobjects a,syscolumns b,systypes c where a.id=b.id and a.name='table15' and a.xtype='U'and b.xtype=c.xtype").ToList();
Assert.Equal(11, columns.Count);
Assert.Equal("Int".ToLower(), columns[0].Name);
Assert.Equal("Time".ToLower(), columns[1].Name);
Assert.Equal("CDate".ToLower(), columns[2].Name);
Assert.Equal("Float".ToLower(), columns[3].Name);
Assert.Equal("Double".ToLower(), columns[4].Name);
Assert.Equal("BigInt".ToLower(), columns[5].Name);
Assert.Equal("__Id".ToLower(), columns[6].Name);
Assert.Equal("String".ToLower(), columns[7].Name);
Assert.Equal("String1".ToLower(), columns[8].Name);
Assert.Equal("int(4)", columns[0].Type);
Assert.Equal("datetime(8)", columns[1].Type);
Assert.Equal("datetime(8)", columns[2].Type);
Assert.Equal("float(8)", columns[3].Type);
Assert.Equal("float(8)", columns[4].Type);
Assert.Equal("bigint(8)", columns[5].Type);
Assert.Equal("bigint(8)", columns[6].Type);
Assert.Equal("nvarchar(8000)", columns[7].Type);
Assert.Equal("nvarchar(8000)", columns[8].Type);
conn.Execute("USE [test]; drop table [test].dbo.[table15]");
}
}
public void ColumnOfUniqueOverLength()
{
try
{
EntitySpider context = new DefaultEntitySpider();
context.Identity = (Guid.NewGuid().ToString("N"));
context.ThreadNum = 1;
var entity = new ModelDefine <Entity18>();
throw new Exception("Failed.");
}
catch (Exception e)
{
Assert.Equal("Column length of unique should not large than 256", e.Message);
}
}
public void MultiEntitiesInitPipelines()
{
EntitySpider context = new DefaultEntitySpider();
context.Identity = (Guid.NewGuid().ToString("N"));
context.ThreadNum = 1;
context.AddPipeline(new MySqlEntityPipeline("Database='mysql';Data Source=localhost;User ID=root;Password=;Port=3306;SslMode=None;"));
context.AddPipeline(new MySqlFileEntityPipeline());
context.AddPipeline(new ConsoleEntityPipeline());
context.AddPipeline(new JsonFileEntityPipeline());
context.AddStartUrl("http://baidu.com");
context.AddEntityType <Entity13>();
context.AddEntityType <Entity12>();
context.Run("running-test");
var entityPipelines = context.GetPipelines();
Assert.Equal(4, entityPipelines.Count);
var pipeline1 = (MySqlEntityPipeline)entityPipelines[0];
Assert.Equal("Database='mysql';Data Source=localhost;User ID=root;Password=;Port=3306;SslMode=None;", pipeline1.ConnectionStringSettings.ConnectionString);
Assert.Equal("MySqlFileEntityPipeline", entityPipelines[1].GetType().Name);
Assert.Equal("ConsoleEntityPipeline", entityPipelines[2].GetType().Name);
Assert.Equal("JsonFileEntityPipeline", entityPipelines[3].GetType().Name);
var pipelines = context.GetPipelines();
Assert.Equal(4, pipelines.Count);
IEntityPipeline pipeline = (IEntityPipeline)pipelines[0];
//entityPipelines = pipeline.GetEntityPipelines();
//Assert.Equal(4, entityPipelines.Count);
//pipeline1 = (MySqlEntityPipeline)entityPipelines[0];
//Assert.Equal("test", pipeline1.GetSchema().Database);
//Assert.Equal("table13", pipeline1.GetSchema().Name);
using (MySqlConnection conn = new MySqlConnection("Database='mysql';Data Source=localhost;User ID=root;Password=;Port=3306;SslMode=None;"))
{
conn.Execute($"DROP table test.table12");
conn.Execute($"DROP table test.table13");
}
}
public void ColumnOfUniqueOverLength()
{
try
{
EntitySpider context = new DefaultEntitySpider();
context.Identity = (Guid.NewGuid().ToString("N"));
context.ThreadNum = 1;
var entity = new EntityDefine <Entity18>();
var pipeline = new MySqlEntityPipeline("Database='mysql';Data Source=localhost;User ID=root;Password=;Port=3306");
pipeline.AddEntity(entity);
throw new Exception("Failed.");
}
catch (Exception e)
{
Assert.Equal("Column length of unique should not large than 256.", e.Message);
}
}
public void MySqlDataTypeTests()
{
using (MySqlConnection conn = new MySqlConnection("Database='mysql';Data Source=localhost;User ID=root;Password=;Port=3306;SslMode=None;"))
{
EntitySpider context = new DefaultEntitySpider();
context.Identity = (Guid.NewGuid().ToString("N"));
context.ThreadNum = 1;
context.AddPipeline(new MySqlEntityPipeline("Database='mysql';Data Source=localhost;User ID=root;Password=;Port=3306;SslMode=None;"));
context.AddStartUrl("http://baidu.com");
context.AddEntityType <Entity15>();
context.Run("running-test");
var columns = conn.Query <ColumnInfo>("SELECT COLUMN_NAME as `Name`, COLUMN_TYPE as `Type` FROM information_schema.columns WHERE table_name='table15' AND table_schema = 'test';").ToList();;
Assert.Equal(9, columns.Count);
Assert.Equal("Int", columns[0].Name);
Assert.Equal("BigInt", columns[1].Name);
Assert.Equal("String", columns[2].Name);
Assert.Equal("Time", columns[3].Name);
Assert.Equal("Float", columns[4].Name);
Assert.Equal("Double", columns[5].Name);
Assert.Equal("String1", columns[6].Name);
Assert.Equal("__Id", columns[7].Name);
Assert.Equal("CDate", columns[8].Name);
Assert.Equal("int(11)", columns[0].Type);
Assert.Equal("bigint(20)", columns[1].Type);
Assert.Equal("longtext", columns[2].Type);
Assert.Equal("timestamp", columns[3].Type);
Assert.Equal("float", columns[4].Type);
Assert.Equal("double", columns[5].Type);
Assert.Equal("varchar(100)", columns[6].Type);
Assert.Equal("timestamp", columns[8].Type);
Assert.Equal("bigint(20)", columns[7].Type);
conn.Execute("drop table `test`.`table15`");
}
}
public void SqlServerDataTypeTests()
{
using (var conn = new SqlConnection("Server=.\\SQLEXPRESS;Database=test;Trusted_Connection=True;MultipleActiveResultSets=true"))
{
EntitySpider context = new DefaultEntitySpider();
context.Identity = (Guid.NewGuid().ToString("N"));
context.ThreadNum = 1;
context.AddPipeline(new SqlServerEntityPipeline("Server=.\\SQLEXPRESS;Database=test;Trusted_Connection=True;MultipleActiveResultSets=true"));
context.AddStartUrl("http://baidu.com");
context.AddEntityType(typeof(Entity15));
context.Run("running-test");
var columns = conn.Query <ColumnInfo>("USE [test];select b.name Name,c.name+'(' + cast(c.length as varchar)+')' [Type] from sysobjects a,syscolumns b,systypes c where a.id=b.id and a.name='table15' and a.xtype='U'and b.xtype=c.xtype").ToList();
Assert.AreEqual(11, columns.Count);
Assert.AreEqual("Int", columns[0].Name);
Assert.AreEqual("Time", columns[1].Name);
Assert.AreEqual("CDate", columns[2].Name);
Assert.AreEqual("Float", columns[3].Name);
Assert.AreEqual("Double", columns[4].Name);
Assert.AreEqual("BigInt", columns[5].Name);
Assert.AreEqual(Core.Infrastructure.Environment.IdColumn, columns[6].Name);
Assert.AreEqual("String", columns[7].Name);
Assert.AreEqual("String1", columns[8].Name);
Assert.AreEqual("int(4)", columns[0].Type);
Assert.AreEqual("datetime(8)", columns[1].Type);
Assert.AreEqual("datetime(8)", columns[2].Type);
Assert.AreEqual("float(8)", columns[3].Type);
Assert.AreEqual("float(8)", columns[4].Type);
Assert.AreEqual("bigint(8)", columns[5].Type);
Assert.AreEqual("bigint(8)", columns[6].Type);
Assert.AreEqual("nvarchar(8000)", columns[7].Type);
Assert.AreEqual("nvarchar(8000)", columns[8].Type);
conn.Execute("USE [test]; drop table [test].dbo.[table15]");
}
}
