internal void DownLoopAsync() { //DownNewDayAsync(); Setting.LastPageIndex = 12766; DownAsync(); var PageHandler = new HandlerHtml(); Task.Factory.StartNew(() => { foreach (var Item in DoF.DownloadCollect.GetConsumingEnumerable()) { Setting.LastPageIndex = Item.Item1; DataBaseCommand.SavePage(Item.Item2); if (PageHandler.HandlerToHtml(Item.Item2) != 75) { Loger.Instance.LocalInfo("当前获得条目小于75条,检查是否完成获取"); DoF.DownloadCollect.CompleteAdding(); DoF.CancelSign.Cancel(); } } }, TaskCreationOptions.LongRunning); Task.Factory.StartNew(() => { foreach (var Item in PageHandler.DataCollect.GetConsumingEnumerable()) { var Status = PageInDateStatus(Item.Item1.Item1); if (Status == -1) { DataBaseCommand.SaveToDataBaseRange(Item.Item2, Item.Item1.Item2, true); } else if (Status == 0) { DataBaseCommand.SaveToDataBaseOneByOne(Item.Item2, Item.Item1.Item2, true); } else { Loger.Instance.LocalInfo($"发现{Item.Item1.Item1}已经存在"); DataBaseCommand.SaveToDataBaseRange(Item.Item2, Item.Item1.Item2, true); //return true; } } int PageInDateStatus(string Date) { var Status = DataBaseCommand.GetDateInfo(Date); if (Status == null) { return(-1); } if (Status.Status) { return(1); } return(0); } }, TaskCreationOptions.LongRunning); }
public int HandlerToHtml(string result) { if (AnalysisData == null) { AnalysisData = new ConcurrentDictionary <int, TorrentInfo>(); } var LoopConut = 0; if (result != "") { Loger.Instance.DateInfo(DateOfNow); Interlocked.Increment(ref pageCount); var HtmlDoc = new HtmlDocument(); HtmlDoc.LoadHtml(result); foreach (var item in HtmlDoc.DocumentNode.SelectNodes(@"/html/body/div[1]/div[2]/table/tbody/tr")) { Interlocked.Increment(ref LoopConut); var TempData = new TorrentInfo(); var temp = HtmlNode.CreateNode(item.OuterHtml); TempData.Class = item.Attributes["class"].Value; TempData.Catagory = temp.SelectSingleNode("td[1]/a").Attributes["title"] .Value; TempData.Title = temp.SelectSingleNode("td[2]/a").Attributes["title"] .Value; TempData.Url = temp.SelectSingleNode("td[2]/a").Attributes["href"] .Value; TempData.Torrent = temp.SelectSingleNode("td[3]/a[1]").Attributes["href"].Value; if (TempData.Torrent.StartsWith("magnet")) { TempData.Magnet = TempData.Torrent; TempData.Torrent = ""; } else { TempData.Magnet = temp.SelectSingleNode("td[3]/a[2]").Attributes["href"].Value; } TempData.Size = temp.SelectSingleNode("td[4]").InnerText; TempData.Timestamp = int.Parse(temp.SelectSingleNode("td[5]").Attributes["data-timestamp"].Value); TempData.Date = temp.SelectSingleNode("td[5]").InnerText; TempData.Up = temp.SelectSingleNode("td[6]").InnerText; TempData.Leeches = temp.SelectSingleNode("td[7]").InnerText; TempData.Complete = temp.SelectSingleNode("td[8]").InnerText; if (string.IsNullOrEmpty(DateOfNow) || AddFin) { if (!AddFin) { Loger.Instance.WithTimeStart($"开始获取{TempData.Day}数据", Time); } DateOfNow = TempData.Day; AddFin = false; } if (DateOfNow != TempData.Day) { DataCollect.TryAdd( new Tuple <Tuple <string, int>, System.Collections.Generic.ICollection <TorrentInfo> >( new Tuple <string, int>(DateOfNow, pageCount), AnalysisData.Values)); pageCount = 0; AnalysisData.Clear(); AddFin = true; Loger.Instance.WithTimeRestart($"结束获取{DateOfNow}数据", Time); } AnalysisData.AddOrUpdate(TempData.id, TempData, (key, Value) => TempData); } if (DateOfNow == "2000-01-01") { DataBaseCommand.SaveToDataBaseRange(AnalysisData.Values, pageCount, false); AnalysisData.Clear(); } } return(LoopConut); }