public void MySqlFileEntityPipeline_InsertSql() { var id = Guid.NewGuid().ToString("N"); var folder = Path.Combine(Env.BaseDirectory, id); var path = Path.Combine(folder, "mysql", "baidu.baidu_search_mysql_file.sql"); try { BaiduSearchSpider spider = new BaiduSearchSpider(); spider.Identity = id; spider.Run(); var lines = File.ReadAllLines(path); Assert.Equal(20, lines.Length); using (var conn = new MySqlConnection(Env.DataConnectionStringSettings.ConnectionString)) { conn.Execute("DELETE FROM baidu.baidu_search_mysql_file"); foreach (var sql in lines) { conn.Execute(sql); } var count = conn.QueryFirst <int>("SELECT COUNT(*) FROM baidu.baidu_search_mysql_file"); Assert.Equal(20, count); conn.Execute("DROP TABLE baidu.baidu_search_mysql_file"); } } finally { if (Directory.Exists(folder)) { Directory.Delete(folder, true); } } }
public void DestoryDownloader() { if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { return; } BaiduSearchSpider spider = new BaiduSearchSpider(); spider.Run(); }
public void MixProcessorAndMissEntityPipeline() { var exceptoin = Assert.Throws <SpiderException>(() => { var id = Guid.NewGuid().ToString("N"); BaiduSearchSpider spider = new BaiduSearchSpider(); spider.Identity = id; spider.Run(); }); Assert.Equal("You may miss a entity pipeline", exceptoin.Message); }
public void AutoIncrementTargetUrlsExtractor_Test() { var id = Guid.NewGuid().ToString("N"); BaiduSearchSpider spider = new BaiduSearchSpider(id); spider.Run(); using (var conn = Env.DataConnectionStringSettings.CreateDbConnection()) { var count = conn.QueryFirst <int>($"SELECT COUNT(*) FROM test.baidu_search WHERE guid='{id}'"); Assert.Equal(60, count); } }
public void EntitySpiderWithDefaultPipeline() { var guid = Guid.NewGuid().ToString(); BaiduSearchSpider spider = new BaiduSearchSpider(guid); spider.Run(); using (var conn = Env.DataConnectionStringSettings.GetDbConnection()) { var count = conn.QueryFirst <int>($"SELECT COUNT(*) FROM test.baidu_search WHERE Guid='{guid}'"); Assert.Equal(20, count); } }
public void EntitySpiderWithDefaultPipeline() { var datetime = DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss"); var guid = Guid.NewGuid().ToString(); BaiduSearchSpider spider = new BaiduSearchSpider(guid); spider.Run(); using (var conn = new MySqlConnection(Config.ConnectString)) { var count = conn.QueryFirst <int>($"SELECT COUNT(*) FROM test.baidu_search WHERE Guid='{guid}'"); Assert.AreEqual(20, count); } }
public void EntitySpiderWithDefaultPipeline() { var datetime = DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss"); var guid = Guid.NewGuid().ToString(); BaiduSearchSpider spider = new BaiduSearchSpider(guid); spider.Run(); using (var conn = Core.Environment.DataConnectionStringSettings.GetDbConnection()) { var count = conn.QueryFirst <int>($"SELECT COUNT(*) FROM test.baidu_search WHERE Guid='{guid}'"); Assert.Equal(20, count); } }
public void MixProcessor() { var id = Guid.NewGuid().ToString("N"); BaiduSearchSpider spider = new BaiduSearchSpider(); spider.AddPipeline(new MySqlEntityPipeline()); spider.Identity = id; spider.Run(); using (var conn = new MySqlConnection(Env.DataConnectionString)) { var count = conn.QueryFirst <int>("SELECT COUNT(*) FROM baidu.baidu_search_mixprocessor"); Assert.Equal(20, count); conn.Execute("DROP TABLE baidu.baidu_search_mixprocessor"); } }
public void DestoryDownloader() { if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { return; } var chromedriverCount1 = Process.GetProcessesByName("chromedriver").Length; BaiduSearchSpider spider = new BaiduSearchSpider(); spider.Run(); var chromedriverCount2 = Process.GetProcessesByName("chromedriver").Length; Assert.Equal(chromedriverCount1, chromedriverCount2); }
public static void Run() { BaiduSearchSpider spider = new BaiduSearchSpider(); spider.Run(); }
public void DestoryDownloader() { BaiduSearchSpider spider = new BaiduSearchSpider(); spider.Run(); }