public void TestCorrectRedisSetting()
{
EntitySpider spider = new EntitySpider(new Site());
spider.AddEntityPipeline(new ConsoleEntityPipeline());
spider.AddEntityType(typeof(TestEntity));
spider.Run("running-test");
}
public void MultiEntitiesInitPipelines()
{
EntitySpider context = new EntitySpider(new Site());
context.SetThreadNum(1);
context.SetIdentity("test-MultiEntitiesInitPipelines");
context.AddEntityPipeline(new MySqlEntityPipeline("Database='test';Data Source=localhost;User ID=root;Password=1qazZAQ!;Port=3306"));
context.AddEntityPipeline(new MySqlFileEntityPipeline());
context.AddEntityPipeline(new ConsoleEntityPipeline());
context.AddEntityPipeline(new JsonFileEntityPipeline());
#if !NET_CORE
//context.AddEntityPipeline(new MongoDbEntityPipeline("mongo"));
#endif
context.AddStartUrl("http://a.com");
context.AddEntityType(typeof(Entity13));
context.AddEntityType(typeof(Entity12));
context.Run("running-test");
var entityPipelines = context.EntityPipelines;
#if NET_CORE
Assert.Equal(4, entityPipelines.Count);
#else
Assert.Equal(4, entityPipelines.Count);
//Assert.Equal(5, entityPipelines.Count);
#endif
var pipeline1 = (MySqlEntityPipeline)entityPipelines[0];
Assert.Equal("Database='test';Data Source=localhost;User ID=root;Password=1qazZAQ!;Port=3306", pipeline1.ConnectString);
Assert.Equal("MySqlFileEntityPipeline", entityPipelines[1].GetType().Name);
Assert.Equal("ConsoleEntityPipeline", entityPipelines[2].GetType().Name);
Assert.Equal("JsonFileEntityPipeline", entityPipelines[3].GetType().Name);
#if !NET_CORE
//Assert.Equal("MongoDbEntityPipeline", entityPipelines[4].GetType().Name);
//var pipeline2 = (MySqlEntityPipeline)entityPipelines[4];
//Assert.Equal("mongo", pipeline2.ConnectString);
#endif
var pipelines = context.GetPipelines();
Assert.Equal(1, pipelines.Count);
EntityPipeline pipeline = (EntityPipeline)pipelines[0];
entityPipelines = pipeline.GetEntityPipelines();
Assert.Equal(4, entityPipelines.Count);
pipeline1 = (MySqlEntityPipeline)entityPipelines[0];
Assert.Equal("db", pipeline1.GetSchema().Database);
Assert.Equal("table", pipeline1.GetSchema().TableName);
#if !NET_CORE
//var pipeline2 = (MongoDbEntityPipeline)entityPipelines[4];
//Assert.Equal("db", pipeline2.GetSchema().Database);
//Assert.Equal("table", pipeline2.GetSchema().TableName);
#endif
using (MySqlConnection conn = new MySqlConnection("Database='mysql';Data Source=localhost;User ID=root;Password=1qazZAQ!;Port=3306"))
{
conn.Execute($"DROP table db.table");
}
}
public void ThrowExceptionWhenNoEntity()
{
try
{
EntitySpider spider = new EntitySpider(new Site());
spider.Run("running-test");
}
catch (SpiderException exception)
{
Assert.Equal("Count of entity is 0.", exception.Message);
}
}
public void ThrowExceptionWhenNoEntityPipeline()
{
try
{
EntitySpider spider = new EntitySpider(new Site());
spider.AddEntityType(typeof(TestEntity));
spider.Run("running-test");
}
catch (SpiderException exception)
{
Assert.Equal("Need at least one entity pipeline.", exception.Message);
}
}
public void ClearScheduler()
{
EntitySpider spider = new EntitySpider(new Site());
spider.Identity = Guid.NewGuid().ToString("N");
spider.SetScheduler(new RedisScheduler
{
Host = "localhost",
Password = "******"
});
spider.AddStartUrl("https://baidu.com");
spider.AddEntityPipeline(new ConsoleEntityPipeline());
spider.AddEntityType(typeof(TestEntity));
spider.Run();
var confiruation = new ConfigurationOptions()
{
ServiceName = "DotnetSpider",
ConnectTimeout = 65530,
KeepAlive = 8,
ConnectRetry = 3,
ResponseTimeout = 3000,
Password = "******",
AllowAdmin = true
};
confiruation.EndPoints.Add(new DnsEndPoint("127.0.0.1", 6379));
var redis = ConnectionMultiplexer.Connect(confiruation);
var db = redis.GetDatabase(0);
var md5 = Encrypt.Md5Encrypt(spider.Identity);
var itemKey = "item-" + md5;
var setKey = "set-" + md5;
var queueKey = "queue-" + md5;
var errorCountKey = "error-record" + md5;
var successCountKey = "success-record" + md5;
//queue
Assert.Equal(0, db.ListLength(queueKey));
//set
Assert.Equal(0, db.SetLength(setKey));
//item
Assert.Equal(0, db.HashLength(itemKey));
//error-count
Assert.Equal(false, db.StringGet(errorCountKey).HasValue);
//success-count
Assert.Equal(false, db.StringGet(successCountKey).HasValue);
}
public void MultiEntitiesInitPipelines()
{
EntitySpider context = new EntitySpider(new Site());
context.SetIdentity(Guid.NewGuid().ToString("N"));
context.SetThreadNum(1);
context.AddPipeline(new MySqlEntityPipeline("Database='test';Data Source=localhost;User ID=root;Password=1qazZAQ!;Port=3306"));
context.AddPipeline(new MySqlFileEntityPipeline());
context.AddPipeline(new ConsoleEntityPipeline());
context.AddPipeline(new JsonFileEntityPipeline());
context.AddStartUrl("http://baidu.com");
context.AddEntityType(typeof(Entity13));
context.AddEntityType(typeof(Entity12));
context.Run("running-test");
var entityPipelines = context.Pipelines;
Assert.AreEqual(4, entityPipelines.Count);
var pipeline1 = (MySqlEntityPipeline)entityPipelines[0];
Assert.AreEqual("Database='test';Data Source=localhost;User ID=root;Password=1qazZAQ!;Port=3306", pipeline1.ConnectString);
Assert.AreEqual("MySqlFileEntityPipeline", entityPipelines[1].GetType().Name);
Assert.AreEqual("ConsoleEntityPipeline", entityPipelines[2].GetType().Name);
Assert.AreEqual("JsonFileEntityPipeline", entityPipelines[3].GetType().Name);
var pipelines = context.GetPipelines();
Assert.AreEqual(4, pipelines.Count);
IEntityPipeline pipeline = (IEntityPipeline)pipelines[0];
//entityPipelines = pipeline.GetEntityPipelines();
//Assert.AreEqual(4, entityPipelines.Count);
//pipeline1 = (MySqlEntityPipeline)entityPipelines[0];
//Assert.AreEqual("db", pipeline1.GetSchema().Database);
//Assert.AreEqual("table13", pipeline1.GetSchema().Name);
using (MySqlConnection conn = new MySqlConnection("Database='mysql';Data Source=localhost;User ID=root;Password=1qazZAQ!;Port=3306"))
{
conn.Execute($"DROP table db.table12");
conn.Execute($"DROP table db.table13");
}
}
static async Task Main(string[] args)
{
var configure = new LoggerConfiguration()
#if DEBUG
.MinimumLevel.Verbose()
#else
.MinimumLevel.Information()
#endif
.MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
.Enrich.FromLogContext()
.WriteTo.Console().WriteTo
.RollingFile("dotnet-spider.log");
Log.Logger = configure.CreateLogger();
await EntitySpider.Run();
// await DistributedSpider.Run();
Console.Read();
}
public void SqlServerDataTypeTests()
{
using (var conn = new SqlConnection("Server=.\\SQLEXPRESS;Database=test;Trusted_Connection=True;MultipleActiveResultSets=true"))
{
EntitySpider context = new EntitySpider(new Site());
context.SetIdentity(Guid.NewGuid().ToString("N"));
context.SetThreadNum(1);
context.AddPipeline(new SqlServerEntityPipeline("Server=.\\SQLEXPRESS;Database=test;Trusted_Connection=True;MultipleActiveResultSets=true"));
context.AddStartUrl("http://baidu.com");
context.AddEntityType(typeof(Entity15));
context.Run("running-test");
var columns = conn.Query <ColumnInfo>("USE [test];select b.name Name,c.name+'(' + cast(c.length as varchar)+')' [Type] from sysobjects a,syscolumns b,systypes c where a.id=b.id and a.name='table15' and a.xtype='U'and b.xtype=c.xtype").ToList();;
Assert.AreEqual(11, columns.Count);
Assert.AreEqual("Int", columns[0].Name);
Assert.AreEqual("Time", columns[1].Name);
Assert.AreEqual("CDate", columns[2].Name);
Assert.AreEqual("Float", columns[3].Name);
Assert.AreEqual("Double", columns[4].Name);
Assert.AreEqual("BigInt", columns[5].Name);
Assert.AreEqual("__Id", columns[6].Name);
Assert.AreEqual("String", columns[7].Name);
Assert.AreEqual("String1", columns[8].Name);
Assert.AreEqual("int(4)", columns[0].Type);
Assert.AreEqual("datetime(8)", columns[1].Type);
Assert.AreEqual("datetime(8)", columns[2].Type);
Assert.AreEqual("float(8)", columns[3].Type);
Assert.AreEqual("float(8)", columns[4].Type);
Assert.AreEqual("bigint(8)", columns[5].Type);
Assert.AreEqual("bigint(8)", columns[6].Type);
Assert.AreEqual("nvarchar(8000)", columns[7].Type);
Assert.AreEqual("nvarchar(8000)", columns[8].Type);
conn.Execute("USE [test]; drop table [test].dbo.[table15]");
}
}
public void MySqlDataTypeTests()
{
using (MySqlConnection conn = new MySqlConnection("Database='mysql';Data Source=localhost;User ID=root;Password=1qazZAQ!;Port=3306"))
{
EntitySpider context = new EntitySpider(new Site());
context.SetIdentity(Guid.NewGuid().ToString("N"));
context.SetThreadNum(1);
context.AddPipeline(new MySqlEntityPipeline("Database='test';Data Source=localhost;User ID=root;Password=1qazZAQ!;Port=3306"));
context.AddStartUrl("http://baidu.com");
context.AddEntityType(typeof(Entity15));
context.Run("running-test");
var columns = conn.Query <ColumnInfo>("SELECT COLUMN_NAME as `Name`, COLUMN_TYPE as `Type` FROM information_schema.columns WHERE table_name='table15' AND table_schema = 'test';").ToList();;
Assert.AreEqual(9, columns.Count);
Assert.AreEqual("Int", columns[0].Name);
Assert.AreEqual("BigInt", columns[1].Name);
Assert.AreEqual("String", columns[2].Name);
Assert.AreEqual("Time", columns[3].Name);
Assert.AreEqual("Float", columns[4].Name);
Assert.AreEqual("Double", columns[5].Name);
Assert.AreEqual("String1", columns[6].Name);
Assert.AreEqual("cdate", columns[7].Name);
Assert.AreEqual("__id", columns[8].Name);
Assert.AreEqual("int(11)", columns[0].Type);
Assert.AreEqual("bigint(20)", columns[1].Type);
Assert.AreEqual("text", columns[2].Type);
Assert.AreEqual("timestamp", columns[3].Type);
Assert.AreEqual("float", columns[4].Type);
Assert.AreEqual("double", columns[5].Type);
Assert.AreEqual("varchar(100)", columns[6].Type);
Assert.AreEqual("timestamp", columns[7].Type);
Assert.AreEqual("bigint(20)", columns[8].Type);
conn.Execute("drop table `test`.`table15`");
}
}