public void MySqlFileEntityPipeline_InsertSql()
{
var id = Guid.NewGuid().ToString("N");
var folder = Path.Combine(Env.BaseDirectory, id);
var path = Path.Combine(folder, "mysql", "baidu.baidu_search_mysql_file.sql");
try
{
BaiduSearchSpider spider = new BaiduSearchSpider();
spider.Identity = id;
spider.Run();
var lines = File.ReadAllLines(path);
Assert.Equal(20, lines.Length);
using (var conn = new MySqlConnection(Env.DataConnectionStringSettings.ConnectionString))
{
conn.Execute("DELETE FROM baidu.baidu_search_mysql_file");
foreach (var sql in lines)
{
conn.Execute(sql);
}
var count = conn.QueryFirst <int>("SELECT COUNT(*) FROM baidu.baidu_search_mysql_file");
Assert.Equal(20, count);
conn.Execute("DROP TABLE baidu.baidu_search_mysql_file");
}
}
finally
{
if (Directory.Exists(folder))
{
Directory.Delete(folder, true);
}
}
}
public void DestoryDownloader()
{
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
return;
}
BaiduSearchSpider spider = new BaiduSearchSpider();
spider.Run();
}
public void MixProcessorAndMissEntityPipeline()
{
var exceptoin = Assert.Throws <SpiderException>(() =>
{
var id = Guid.NewGuid().ToString("N");
BaiduSearchSpider spider = new BaiduSearchSpider();
spider.Identity = id;
spider.Run();
});
Assert.Equal("You may miss a entity pipeline", exceptoin.Message);
}
public void AutoIncrementTargetUrlsExtractor_Test()
{
var id = Guid.NewGuid().ToString("N");
BaiduSearchSpider spider = new BaiduSearchSpider(id);
spider.Run();
using (var conn = Env.DataConnectionStringSettings.CreateDbConnection())
{
var count = conn.QueryFirst <int>($"SELECT COUNT(*) FROM test.baidu_search WHERE guid='{id}'");
Assert.Equal(60, count);
}
}
public void EntitySpiderWithDefaultPipeline()
{
var guid = Guid.NewGuid().ToString();
BaiduSearchSpider spider = new BaiduSearchSpider(guid);
spider.Run();
using (var conn = Env.DataConnectionStringSettings.GetDbConnection())
{
var count = conn.QueryFirst <int>($"SELECT COUNT(*) FROM test.baidu_search WHERE Guid='{guid}'");
Assert.Equal(20, count);
}
}
public void EntitySpiderWithDefaultPipeline()
{
var datetime = DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss");
var guid = Guid.NewGuid().ToString();
BaiduSearchSpider spider = new BaiduSearchSpider(guid);
spider.Run();
using (var conn = new MySqlConnection(Config.ConnectString))
{
var count = conn.QueryFirst <int>($"SELECT COUNT(*) FROM test.baidu_search WHERE Guid='{guid}'");
Assert.AreEqual(20, count);
}
}
public void EntitySpiderWithDefaultPipeline()
{
var datetime = DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss");
var guid = Guid.NewGuid().ToString();
BaiduSearchSpider spider = new BaiduSearchSpider(guid);
spider.Run();
using (var conn = Core.Environment.DataConnectionStringSettings.GetDbConnection())
{
var count = conn.QueryFirst <int>($"SELECT COUNT(*) FROM test.baidu_search WHERE Guid='{guid}'");
Assert.Equal(20, count);
}
}
public void MixProcessor()
{
var id = Guid.NewGuid().ToString("N");
BaiduSearchSpider spider = new BaiduSearchSpider();
spider.AddPipeline(new MySqlEntityPipeline());
spider.Identity = id;
spider.Run();
using (var conn = new MySqlConnection(Env.DataConnectionString))
{
var count = conn.QueryFirst <int>("SELECT COUNT(*) FROM baidu.baidu_search_mixprocessor");
Assert.Equal(20, count);
conn.Execute("DROP TABLE baidu.baidu_search_mixprocessor");
}
}
public void DestoryDownloader()
{
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
return;
}
var chromedriverCount1 = Process.GetProcessesByName("chromedriver").Length;
BaiduSearchSpider spider = new BaiduSearchSpider();
spider.Run();
var chromedriverCount2 = Process.GetProcessesByName("chromedriver").Length;
Assert.Equal(chromedriverCount1, chromedriverCount2);
}
public static void Run()
{
BaiduSearchSpider spider = new BaiduSearchSpider();
spider.Run();
}
public void DestoryDownloader()
{
BaiduSearchSpider spider = new BaiduSearchSpider();
spider.Run();
}
