public void DatebaseLogAndStatus()
{
Env.Reload();
string id = Guid.NewGuid().ToString("N");
string taskGroup = Guid.NewGuid().ToString("N");
string userId = Guid.NewGuid().ToString("N");
using (Spider spider = Spider.Create(new Site {
EncodingName = "UTF-8", SleepTime = 1000
},
id,
new QueueDuplicateRemovedScheduler(),
new TestPageProcessor()))
{
spider.Monitor = new DbMonitor(spider.TaskId, spider.Identity);
spider.AddPipeline(new TestPipeline());
spider.ThreadNum = 1;
for (int i = 0; i < 5; i++)
{
spider.AddStartUrl("http://www.baidu.com/" + i);
}
spider.Run();
}
Thread.Sleep(3000);
using (var conn = (Env.SystemConnectionStringSettings.GetDbConnection()))
{
Assert.StartsWith("Crawl complete, cost", conn.Query <Log>($"SELECT * FROM DotnetSpider.Log where Identity='{id}'").Last().message);
Assert.Equal(1, conn.Query <CountResult>($"SELECT COUNT(*) as Count FROM DotnetSpider.Status where Identity='{id}'").First().Count);
Assert.Equal("Finished", conn.Query <statusObj>($"SELECT * FROM DotnetSpider.Status where Identity='{id}'").First().status);
}
}
public void DatebaseLogAndStatus()
{
LogUtil.Init();
string id = Guid.NewGuid().ToString("N");
Env.NodeId = "DEFAULT";
using (Spider spider = Spider.Create(new Site {
EncodingName = "UTF-8"
},
id,
new QueueDuplicateRemovedScheduler(),
new TestPageProcessor()))
{
spider.Downloader = new TestDownloader();
spider.TaskId = "1";
spider.Monitor = new MySqlMonitor(spider.TaskId, spider.Identity, false, "Database='mysql';Data Source=localhost;User ID=root;Port=3306;SslMode=None;");
spider.AddPipeline(new TestPipeline());
for (int i = 0; i < 5; i++)
{
Serilog.Log.Logger.Information("add start url" + i, id);
spider.AddStartUrl("http://www.baidu.com/" + i);
}
spider.EmptySleepTime = 1000;
spider.Run();
}
using (var conn = new MySqlConnection("Database='mysql';Data Source=localhost;User ID=root;Port=3306;SslMode=None;"))
{
var logs = conn.Query <Log>($"SELECT * FROM dotnetspider.log where identity='{id}'").ToList();
Assert.StartsWith("Crawl complete, cost", logs[logs.Count - 1].message);
Assert.Equal(1, conn.Query <CountResult>($"SELECT COUNT(*) as Count FROM dotnetspider.status where identity='{id}'").First().Count);
Assert.Equal("Finished", conn.Query <statusObj>($"SELECT * FROM dotnetspider.status where identity='{id}'").First().status);
}
}
public void DatebaseLogAndStatus()
{
string id = Guid.NewGuid().ToString("N");
string taskGroup = Guid.NewGuid().ToString("N");
string userId = Guid.NewGuid().ToString("N");
string connectString = "Database='test';Data Source=localhost;User ID=root;Password=1qazZAQ!;Port=3306";
Configuration.SetValue("logAndStatusConnectString", connectString);
Assert.Equal("Database='test';Data Source=localhost;User ID=root;Password=1qazZAQ!;Port=3306", Configuration.GetValue("logAndStatusConnectString"));
using (Spider spider = Spider.Create(new Site {
EncodingName = "UTF-8", MinSleepTime = 1000
},
id,
userId,
taskGroup,
new TestPageProcessor(), new QueueDuplicateRemovedScheduler()))
{
spider.AddPipeline(new TestPipeline()).SetThreadNum(1);
for (int i = 0; i < 5; i++)
{
spider.AddStartUrl("http://www.baidu.com/" + i);
}
MonitorCenter.Register(spider);
spider.Run();
}
using (MySqlConnection conn = new MySqlConnection(connectString))
{
Assert.Equal(1, conn.Query <CountResult>($"SELECT COUNT(*) as Count FROM dotnetspider.status where userid='{userId}' and taskgroup='{taskGroup}' and identity='{id}'").First().Count);
Assert.Equal(7, conn.Query <CountResult>($"SELECT COUNT(*) as Count FROM dotnetspider.log where userid='{userId}' and taskgroup='{taskGroup}' and identity='{id}'").First().Count);
}
}
public void DatebaseLogAndStatus()
{
string id = Guid.NewGuid().ToString("N");
string taskGroup = Guid.NewGuid().ToString("N");
string userId = Guid.NewGuid().ToString("N");
using (Spider spider = Spider.Create(new Site {
EncodingName = "UTF-8", SleepTime = 1000
},
id,
new QueueDuplicateRemovedScheduler(),
new TestPageProcessor()))
{
spider.Monitor = new MySqlMonitor(spider.TaskId, spider.Identity, false, "Database='mysql';Data Source=localhost;User ID=root;Port=3306;SslMode=None;");
spider.AddPipeline(new TestPipeline());
spider.ThreadNum = 1;
for (int i = 0; i < 5; i++)
{
spider.AddStartUrl("http://www.baidu.com/" + i);
}
spider.Run();
}
Thread.Sleep(3000);
using (var conn = new MySqlConnection("Database='mysql';Data Source=localhost;User ID=root;Port=3306;SslMode=None;"))
{
Assert.StartsWith("Crawl complete, cost", conn.Query <Log>($"SELECT * FROM DotnetSpider.Log where Identity='{id}'").Last().message);
Assert.Equal(1, conn.Query <CountResult>($"SELECT COUNT(*) as Count FROM DotnetSpider.Status where Identity='{id}'").First().Count);
Assert.Equal("Finished", conn.Query <statusObj>($"SELECT * FROM DotnetSpider.Status where Identity='{id}'").First().status);
}
}
public void DatebaseLogAndStatus()
{
string id = Guid.NewGuid().ToString("N");
string taskGroup = Guid.NewGuid().ToString("N");
string userId = Guid.NewGuid().ToString("N");
string connectString = "Database='mysql';Data Source=localhost;User ID=root;Password=1qazZAQ!;Port=3306";
Core.Infrastructure.Config.SetValue("connectString", connectString);
Assert.AreEqual("Database='mysql';Data Source=localhost;User ID=root;Password=1qazZAQ!;Port=3306", Core.Infrastructure.Config.GetValue("connectString"));
using (Spider spider = Spider.Create(new Site {
EncodingName = "UTF-8", SleepTime = 1000
},
id,
new QueueDuplicateRemovedScheduler(),
new TestPageProcessor()))
{
spider.AddPipeline(new TestPipeline());
spider.ThreadNum = 1;
for (int i = 0; i < 5; i++)
{
spider.AddStartUrl("http://www.baidu.com/" + i);
}
spider.Monitor = new DbMonitor(id);
spider.Run();
}
using (MySqlConnection conn = new MySqlConnection(connectString))
{
Assert.AreEqual(15, conn.Query <CountResult>($"SELECT COUNT(*) as Count FROM dotnetspider.log where identity='{id}'").First().Count);
Assert.AreEqual(1, conn.Query <CountResult>($"SELECT COUNT(*) as Count FROM dotnetspider.status where identity='{id}'").First().Count);
}
}
public static Spider AddDataBasePipeline(this Spider spider, IDatabaseStore databaseStore)
{
return(spider.AddPipeline(new DatabasePipeline(databaseStore)));
}