public void NewsUrlGatheringTest()
{
//http://www.toutiao.com/api/article/recent/?source=2&category=news_car&as=A18538618270765&cp=581280A796A56E1&_=1477576549309
//var url = "http://www.toutiao.com/api/article/feed/?category=__all__&utm_source=toutiao&widen=0&max_behot_time=0&max_behot_time_tmp=0&as=A1D5B8000CC7350&cp=580C772325900E1";
var url = "http://www.toutiao.com/api/article/feed/?category=__all__&utm_source=toutiao&widen=0&max_behot_time=0&max_behot_time_tmp=0&as=A1B5E83210C6E63&cp=5820D6CE66539E1";
bll.GatheringAuthorUrlFromChannel(url, 100, 0);
}
/// <summary>
/// 头条频道新闻抓取处理
/// </summary>
public static void AuthorUrlGathering()
{
try
{
if (Global.IsEnableGatherChannel != "1")
{
return;
}
////时段控制 0-8点不抓取
//if (DateTime.Now.Hour < 8)
//{
// return;
//}
while (true)
{
Log.Info("频道新闻抓取开始:" + DateTime.Now);
var siteList = XmlDeal.GetSitesInfo();
if (siteList != null && siteList.Count > 0)
{
foreach (var site in siteList)
{
if (site.SiteName.ToLower() == "toutiao")
{
var bll = new ToutiaoGather();
bll.GatheringAuthorUrlFromChannel(site.Url, site.NewsType, 0);
}
Thread.Sleep(60 * 1000);
}
}
else
{
Log.Error("抓取错误-检查site.xml" + DateTime.Now);
}
Log.Info("频道新闻抓取结束:" + DateTime.Now);
Thread.Sleep(60 * 1000);
}
}
catch (Exception ex)
{
Log.Error(ex.Message + ex.StackTrace);
}
}
/// <summary>
/// 头条频道新闻抓取处理
/// </summary>
public static void GatheringAuthorUrlFromChannel()
{
try
{
if (Global.IsEnableGatherChannel != "1")
{
return;
}
////时段控制 0-8点不抓取
//if (DateTime.Now.Hour < 8)
//{
// return;
//}
int i = 0;
while (true && ProxyDeal.IsProxyReady)
{
i++;
Log.Info("频道新闻抓取开始 i=" + i + " time=" + DateTime.Now);
var siteList = XmlDeal.GetSitesInfo();
if (siteList != null && siteList.Count > 0)
{
//foreach (var site in siteList)
//{
// if (site.SiteName.ToLower() == "toutiao")
// {
// var bll = new ToutiaoGather();
// bll.AuthorUrlGathering(site.Url, site.NewsType);
// }
// Thread.Sleep(60 * 1000);
//}
#region === 改成随机，不固定顺序，避免多开时从同一个顺序启动抓取 ===
Random rnd = new Random();
var iStart = rnd.Next(0, siteList.Count);
//增加从下面索引开始的机率
if (iStart % 3 == 0)
{
iStart = 0;
}
if (iStart % 4 == 0)
{
iStart = 1;
}
if (iStart % 5 == 0)
{
iStart = 5;
}
if (iStart % 6 == 0)
{
iStart = 9;
}
for (var start = iStart; start < siteList.Count; start++)
{
if (start > siteList.Count || start < 0)
{
start = 0;
}
if (siteList[start].SiteName.ToLower() == "toutiao")
{
var bll = new ToutiaoGather();
bll.GatheringAuthorUrlFromChannel(siteList[start].Url, siteList[start].NewsType, 0);
}
Thread.Sleep(5 * 1000);
}
#endregion
}
else
{
Log.Error("抓取错误-检查site.xml" + DateTime.Now);
}
Log.Info("频道新闻抓取结束 i=" + i + " time=" + DateTime.Now);
Thread.Sleep(60 * 1000);
}
if (!ProxyDeal.IsProxyReady)
{
Log.Info("代理未准备好" + DateTime.Now);
Thread.Sleep(60 * 1000);
}
}
catch (Exception ex)
{
Log.Error(ex.Message + ex.StackTrace);
}
}