public void NewsUrlGatheringTest() { //http://www.toutiao.com/api/article/recent/?source=2&category=news_car&as=A18538618270765&cp=581280A796A56E1&_=1477576549309 //var url = "http://www.toutiao.com/api/article/feed/?category=__all__&utm_source=toutiao&widen=0&max_behot_time=0&max_behot_time_tmp=0&as=A1D5B8000CC7350&cp=580C772325900E1"; var url = "http://www.toutiao.com/api/article/feed/?category=__all__&utm_source=toutiao&widen=0&max_behot_time=0&max_behot_time_tmp=0&as=A1B5E83210C6E63&cp=5820D6CE66539E1"; bll.GatheringAuthorUrlFromChannel(url, 100, 0); }
/// <summary> /// 头条频道新闻抓取处理 /// </summary> public static void AuthorUrlGathering() { try { if (Global.IsEnableGatherChannel != "1") { return; } ////时段控制 0-8点不抓取 //if (DateTime.Now.Hour < 8) //{ // return; //} while (true) { Log.Info("频道新闻抓取开始:" + DateTime.Now); var siteList = XmlDeal.GetSitesInfo(); if (siteList != null && siteList.Count > 0) { foreach (var site in siteList) { if (site.SiteName.ToLower() == "toutiao") { var bll = new ToutiaoGather(); bll.GatheringAuthorUrlFromChannel(site.Url, site.NewsType, 0); } Thread.Sleep(60 * 1000); } } else { Log.Error("抓取错误-检查site.xml" + DateTime.Now); } Log.Info("频道新闻抓取结束:" + DateTime.Now); Thread.Sleep(60 * 1000); } } catch (Exception ex) { Log.Error(ex.Message + ex.StackTrace); } }
/// <summary> /// 头条频道新闻抓取处理 /// </summary> public static void GatheringAuthorUrlFromChannel() { try { if (Global.IsEnableGatherChannel != "1") { return; } ////时段控制 0-8点不抓取 //if (DateTime.Now.Hour < 8) //{ // return; //} int i = 0; while (true && ProxyDeal.IsProxyReady) { i++; Log.Info("频道新闻抓取开始 i=" + i + " time=" + DateTime.Now); var siteList = XmlDeal.GetSitesInfo(); if (siteList != null && siteList.Count > 0) { //foreach (var site in siteList) //{ // if (site.SiteName.ToLower() == "toutiao") // { // var bll = new ToutiaoGather(); // bll.AuthorUrlGathering(site.Url, site.NewsType); // } // Thread.Sleep(60 * 1000); //} #region === 改成随机,不固定顺序,避免多开时从同一个顺序启动抓取 === Random rnd = new Random(); var iStart = rnd.Next(0, siteList.Count); //增加从下面索引开始的机率 if (iStart % 3 == 0) { iStart = 0; } if (iStart % 4 == 0) { iStart = 1; } if (iStart % 5 == 0) { iStart = 5; } if (iStart % 6 == 0) { iStart = 9; } for (var start = iStart; start < siteList.Count; start++) { if (start > siteList.Count || start < 0) { start = 0; } if (siteList[start].SiteName.ToLower() == "toutiao") { var bll = new ToutiaoGather(); bll.GatheringAuthorUrlFromChannel(siteList[start].Url, siteList[start].NewsType, 0); } Thread.Sleep(5 * 1000); } #endregion } else { Log.Error("抓取错误-检查site.xml" + DateTime.Now); } Log.Info("频道新闻抓取结束 i=" + i + " time=" + DateTime.Now); Thread.Sleep(60 * 1000); } if (!ProxyDeal.IsProxyReady) { Log.Info("代理未准备好" + DateTime.Now); Thread.Sleep(60 * 1000); } } catch (Exception ex) { Log.Error(ex.Message + ex.StackTrace); } }