/// <summary>
/// 更新美食模块的URL
/// </summary>
public void UpdateMeiShi()
{
HtmlAgilityPack.HtmlDocument doc = null;
IDbConnection conn = null;
try
{
conn = DapperManager.GetConnection();
conn.Open();
var list_source = conn.Query <City>("SELECT ID,Name,URL,MeiShiURL FROM T_City WHERE State = @State", new { State = 1 });
//一个一个访问URL，并获取美食模块的URL
for (int i = 0; i < list_source.Count(); i++)
{
if (list_source.ElementAt(i).MeiShiURL != null)
{
continue;
}
try
{
doc = webClient.Load(list_source.ElementAt(i).URL);
}
catch (System.Net.WebException webEx)
{
listBox1.Invoke(new Action(() =>
{
listBox1.Items.Insert(0, webEx.Message);
}));
continue;
}
var hrefList = doc.DocumentNode.SelectNodes("//div[@class='category-nav-content-wrapper']//a[@href]");
foreach (var item in hrefList)
{
if (item.InnerText.Equals("美食"))
{
var MeiShiURL = item.Attributes["href"].Value;
list_source.ElementAt(i).MeiShiURL = MeiShiURL;
var count = conn.Execute("UPDATE T_City SET MeiShiURL = @MeiShiURL WHERE ID = @ID", list_source.ElementAt(i));
listBox1.Invoke(new Action(() =>
{
listBox1.Items.Insert(0, $"{DateTime.Now } 更新了{list_source.ElementAt(i).Name}的美食地址");
}));
break;
}
}
}
}
catch (Exception ex)
{
MyLog.logger.Error(ex.Message);
}
finally
{
conn.Close();
}
//hrefList = doc.DocumentNode.SelectNodes("//div[@id='app']//a[@href]");
}
static void ListRole()
{
using (var conn = DapperManager.GetConnection())
{
foreach (var item in conn.GetAll <RoleEntity>())
{
Console.WriteLine($"{item.Id,-10}{item.Name,-10}");
}
}
}
static void CreateRole(string roleName)
{
using (var conn = DapperManager.GetConnection())
{
var maxId = 1;
var role = new RoleEntity()
{
Id = maxId,
Name = roleName
};
var roles = conn.GetAll <RoleEntity>().ToList();
if (roles.Any())
{
maxId = roles.Max(r => r.Id);
role.Id = maxId << 1;
}
conn.Insert(role);
Console.WriteLine($"insert role: {roleName} => succeed!");
ListRole();
}
}
private void 更新城市ToolStripMenuItem_Click(object sender, EventArgs e)
{
listBox1.Items.Insert(0, $"{DateTime.Now.ToShortTimeString()} 请求数据中...");
try
{
LoadCity();
IDbConnection conn = null;
try
{
conn = DapperManager.GetConnection();
conn.Open();
_cancel_token = new CancellationTokenSource();
var list_source = conn.Query <City>("SELECT ID,Name,URL,MeiShiURL FROM T_City WHERE State = @State", new { State = 1 });
var offset = list_source.Count() / 3;
Task task = new Task(() => {
UpdateMeiShi(list_source.Take(offset).ToList(), list_source.Count());
});
task.Start();
Task task2 = new Task(() => {
UpdateMeiShi(list_source.Skip(offset).Take(offset).ToList(), list_source.Count());
});
task2.Start();
Task task3 = new Task(() => {
UpdateMeiShi(list_source.Skip(offset * 2).ToList(), list_source.Count());
});
task3.Start();
}
catch (Exception ex)
{
MyLog.logger.Error(ex.Message);
}
}
catch (Exception ex)
{
MyLog.logger.Error(ex.Message);
}
}
/// <summary>
/// 更新美食模块的URL
/// </summary>
public void UpdateMeiShi(List <City> list_source, int total)
{
HtmlAgilityPack.HtmlDocument doc = null;
IDbConnection conn = null;
try
{
conn = DapperManager.GetConnection();
//一个一个访问URL，并获取美食模块的URL
for (int i = 0; i < list_source.Count(); i++)
{
if (_cancel_token.IsCancellationRequested)
{
numberOfMeishi = 0;
return;
}
try
{
doc = _webClient.Load(list_source.ElementAt(i).URL);
}
catch (System.Net.WebException ex)
{
listBox1.Invoke(new Action(() =>
{
listBox1.Items.Insert(0, $"{DateTime.Now} 更新{list_source.ElementAt(i).Name}的美食地址失败,请查看错误日志,{numberOfMeishi}/{total}");
}));
MyLog.logger.Error(ex.Message);
continue;
}
finally
{
listBox1.Invoke(new Action(() =>
{
listBox1.Items.Insert(0, $"{DateTime.Now} 更新了{list_source.ElementAt(i).Name}的美食地址,{numberOfMeishi}/{total}");
}));
Interlocked.Increment(ref numberOfMeishi);
}
var hrefList = doc.DocumentNode.SelectNodes("//div[@class='category-nav-content-wrapper']//a[@href]");
foreach (var item in hrefList)
{
//可能会有很多模块，比如美食，外卖，酒店，电影，机票等，这里只取美食模块的数据
if (item.InnerText.Equals("美食"))
{
var MeiShiURL = item.Attributes["href"].Value;
list_source.ElementAt(i).MeiShiURL = MeiShiURL;
var count = conn.Execute("UPDATE T_City SET MeiShiURL = @MeiShiURL WHERE ID = @ID", list_source.ElementAt(i));
break;
}
}
}
}
catch (Exception ex)
{
MyLog.logger.Error(ex.Message);
}
finally
{
conn.Close();
}
}
/// <summary>
/// 获取所有城市列表
/// </summary>
public void LoadCity()
{
HtmlAgilityPack.HtmlDocument doc = null;
try
{
//加载城市列表
doc = _webClient.Load(Global.uri_city);
}
catch (Exception ex)
{
MyLog.logger.Error(ex.Message);
}
//Dom节点获取列表
HtmlNodeCollection hrefList = doc.DocumentNode.SelectNodes("//div[@id='app']//a[@href]");
List <City> list_target = new List <City>();
for (int i = 0; i < hrefList.Count; i++)
{
var url = hrefList[i].Attributes["href"];
if (i == 0)
{
listBox1.Invoke(new Action(() =>
{
listBox1.Items.Insert(0, $"导入{i + 1}/{hrefList.Count}条数据");
}));
}
else
{
listBox1.Items[0] = $"导入{i + 1}/{hrefList.Count}条数据";
}
list_target.Add(new City()
{
URL = url.Value.Insert(0, "http:"),
Name = hrefList[i].InnerText
});
}
int count = 0;
listBox1.Invoke(new Action(() =>
{
listBox1.Items.Insert(0, $"{DateTime.Now.ToShortTimeString()} 导入数据中...");
}));
IDbConnection conn = null;
try
{
conn = DapperManager.GetConnection();
conn.Open();
var list_source = conn.Query <City>("SELECT Name,URL FROM T_City");
var list_except = list_target.Except(list_source);
count = conn.Execute("INSERT INTO T_City(Name,URL) VALUES(@Name,@URL)", list_except);
}
catch (Exception ex)
{
MyLog.logger.Error(ex.Message);
}
finally
{
conn.Close();
}
listBox1.Invoke(new Action(() =>
{
listBox1.Items.Insert(0, $"{DateTime.Now.ToShortTimeString()} 解析完毕，导入{count}条数据...");
}));
}
