private void OnEnable()
{
progress.OnProgressComplete += HandleRunningOutOfTime;
for (int i = 0; i < responses.Length; i++)
{
responses[i].OnResponse += HandleResponse;
}
currentJoke = jokes[Random.Range(0, jokes.Count)];
ShowCall();
}
/// <summary>
/// 处理队列数据
/// </summary>
/// <param name="count">数量</param>
private void RunConcurrentQueue(int count)
{
if (currendQueue.Any() && currendQueue.Count >= count)
{
for (int i = 0; i < currendQueue.Count; i++)
{
JokeData info = null;
currendQueue.TryDequeue(out info);
var update = Db.Updateable(info).UpdateColumns(s => new { s.Title, s.Contents, s.PointsCount, s.ZanCount, s.UpdateBy, s.UpdateDate }).WhereColumns(it => new { it.Title });
if (update.ExecuteCommand() <= 0)
{
Db.Insertable(info).ExecuteReturnBigIdentity();
}
}
}
}
public void TestMethod1()
{
// Arrange
var data = new JokeData
{
categories = new List <string> {
"teste1", "teste2"
},
Id = 1,
Joke = "My Joke"
};
// Assert
data.categories.Should().NotBeEmpty();
data.Id.Should().Be(1);
data.Joke.Should().Be("My Joke");
}
/// <summary>
/// 获取笑话数据
/// </summary>
private void GetJokeData()
{
try
{
Output(DateTime.Now + " → " + " 程序开始执行！", Color.Blue);
HtmlWeb web = new HtmlWeb();
web.OverrideEncoding = Encoding.GetEncoding("utf-8");
HtmlAgilityPack.HtmlDocument htmlDoc = web.Load(linkLabel1.Text);
HtmlNode strTemp = htmlDoc.DocumentNode.SelectSingleNode(@"/html/body/div[4]/div[1]/div[2]/a[6]");
if (strTemp == null)
{
Output(DateTime.Now + " → " + " 获取总页数失败！", Color.Red);
return;
}
string str = CommonHelper.GetTitleContent(strTemp.OuterHtml, "a", "href");
if (string.IsNullOrEmpty(str))
{
Output(DateTime.Now + " → " + " 获取总页数失败！", Color.Red);
return;
}
string result = Regex.Replace(str, @"[^0-9]+", "");
long pageCount = long.Parse(result);
Output(DateTime.Now + " → " + " 程序获取到总页数为：" + pageCount + "", Color.Blue);
for (int i = 1; i <= pageCount; i++)
{
try
{
rt_txt.AppendText(DateTime.Now + " → " + " 程序开始请求第" + i + "页数据！" + "\r\n");
string url = string.Format("https://www.biedoul.com/index/{0}", i);
web.OverrideEncoding = Encoding.GetEncoding("utf-8");
HtmlAgilityPack.HtmlDocument html = web.Load(url);
HtmlNodeCollection rtitle = html.DocumentNode.SelectNodes(@"//*[@class='xhlist']");//返回标题数据集合
if (rtitle == null)
{
Output(DateTime.Now + " → " + " 未请求到数据，在第" + i.ToString() + "页,程序将睡眠5秒钟跳过本页请求数据！", Color.Red);
Thread.Sleep(5000);
continue;
}
else
{
foreach (HtmlNode node in rtitle)
{
try
{
string title = node.SelectSingleNode(@"span/dd/a/strong").InnerText; //请求获取标题
string content = node.SelectSingleNode(@"dd").InnerHtml; //请求获取内容
string zanCount = node.SelectSingleNode(@"div/div[2]/a[1]/p").InnerText; //请求获取点赞数
string pointsCount = node.SelectSingleNode(@"div/div[2]/a[2]/p").InnerText; //请求获取减分数
var model = new JokeData()
{
Contents = content,
CreateBy = 1,
CreateDate = DateTime.Now,
PointsCount = long.Parse(pointsCount == "" ? "0" : pointsCount),
Title = title,
UpdateBy = 1,
UpdateDate = DateTime.Now,
ZanCount = long.Parse(zanCount == "" ? "0" : zanCount)
};
currendQueue.Enqueue(model);
Output(DateTime.Now + " → " + " 成功获取标题为：【" + title + "】已保存队列", Color.BlueViolet);
}
catch (Exception ex)
{
rt_txt.AppendText(DateTime.Now + " → " + ex.Message + "\r\n");
continue;
}
}
}
RunConcurrentQueue(100);
Output(DateTime.Now + " → " + " 第" + i + "页数据请求保存完毕！", Color.Black);
Output(DateTime.Now + " → " + " 程序睡眠5秒钟，稍后开始请求下一页数据！", Color.Black);
Thread.Sleep(5000);
GC.Collect();
}
catch (Exception ex)
{
rt_txt.AppendText(DateTime.Now + " → " + ex.Message + "\r\n");
continue;
}
}
RunConcurrentQueue(1);
Output(DateTime.Now + " → " + " 所有页数据请求完毕，开始执行队列将数据保存至数据库...请稍后！", Color.Red);
Output(DateTime.Now + " → " + " 数据已保存完毕！", Color.Red);
GC.Collect();
}
catch (Exception ex)
{
rt_txt.AppendText(DateTime.Now + " → " + ex.Message + "\r\n");
return;
}
}
