internal void DownLoopAsync() { //DownNewDayAsync(); Setting.LastPageIndex = 12766; DownAsync(); var PageHandler = new HandlerHtml(); Task.Factory.StartNew(() => { foreach (var Item in DoF.DownloadCollect.GetConsumingEnumerable()) { Setting.LastPageIndex = Item.Item1; DataBaseCommand.SavePage(Item.Item2); if (PageHandler.HandlerToHtml(Item.Item2) != 75) { Loger.Instance.LocalInfo("当前获得条目小于75条,检查是否完成获取"); DoF.DownloadCollect.CompleteAdding(); DoF.CancelSign.Cancel(); } } }, TaskCreationOptions.LongRunning); Task.Factory.StartNew(() => { foreach (var Item in PageHandler.DataCollect.GetConsumingEnumerable()) { var Status = PageInDateStatus(Item.Item1.Item1); if (Status == -1) { DataBaseCommand.SaveToDataBaseRange(Item.Item2, Item.Item1.Item2, true); } else if (Status == 0) { DataBaseCommand.SaveToDataBaseOneByOne(Item.Item2, Item.Item1.Item2, true); } else { Loger.Instance.LocalInfo($"发现{Item.Item1.Item1}已经存在"); DataBaseCommand.SaveToDataBaseRange(Item.Item2, Item.Item1.Item2, true); //return true; } } int PageInDateStatus(string Date) { var Status = DataBaseCommand.GetDateInfo(Date); if (Status == null) { return(-1); } if (Status.Status) { return(1); } return(0); } }, TaskCreationOptions.LongRunning); }
public int HandlerToHtml(string result) { if (AnalysisData == null) { AnalysisData = new ConcurrentDictionary <int, TorrentInfo>(); } var LoopConut = 0; if (result != "") { Loger.Instance.DateInfo(DateOfNow); Interlocked.Increment(ref pageCount); var HtmlDoc = new HtmlDocument(); HtmlDoc.LoadHtml(result); foreach (var item in HtmlDoc.DocumentNode.SelectNodes(@"/html/body/div[1]/div[2]/table/tbody/tr")) { Interlocked.Increment(ref LoopConut); var TempData = new TorrentInfo(); var temp = HtmlNode.CreateNode(item.OuterHtml); TempData.Class = item.Attributes["class"].Value; TempData.Catagory = temp.SelectSingleNode("td[1]/a").Attributes["title"] .Value; TempData.Title = temp.SelectSingleNode("td[2]/a").Attributes["title"] .Value; TempData.Url = temp.SelectSingleNode("td[2]/a").Attributes["href"] .Value; TempData.Torrent = temp.SelectSingleNode("td[3]/a[1]").Attributes["href"].Value; if (TempData.Torrent.StartsWith("magnet")) { TempData.Magnet = TempData.Torrent; TempData.Torrent = ""; } else { TempData.Magnet = temp.SelectSingleNode("td[3]/a[2]").Attributes["href"].Value; } TempData.Size = temp.SelectSingleNode("td[4]").InnerText; TempData.Timestamp = int.Parse(temp.SelectSingleNode("td[5]").Attributes["data-timestamp"].Value); TempData.Date = temp.SelectSingleNode("td[5]").InnerText; TempData.Up = temp.SelectSingleNode("td[6]").InnerText; TempData.Leeches = temp.SelectSingleNode("td[7]").InnerText; TempData.Complete = temp.SelectSingleNode("td[8]").InnerText; if (string.IsNullOrEmpty(DateOfNow) || AddFin) { if (!AddFin) { Loger.Instance.WithTimeStart($"开始获取{TempData.Day}数据", Time); } DateOfNow = TempData.Day; AddFin = false; } if (DateOfNow != TempData.Day) { DataCollect.TryAdd( new Tuple <Tuple <string, int>, System.Collections.Generic.ICollection <TorrentInfo> >( new Tuple <string, int>(DateOfNow, pageCount), AnalysisData.Values)); pageCount = 0; AnalysisData.Clear(); AddFin = true; Loger.Instance.WithTimeRestart($"结束获取{DateOfNow}数据", Time); } AnalysisData.AddOrUpdate(TempData.id, TempData, (key, Value) => TempData); } if (DateOfNow == "2000-01-01") { DataBaseCommand.SaveToDataBaseRange(AnalysisData.Values, pageCount, false); AnalysisData.Clear(); } } return(LoopConut); }
private static void Main(string[] args) { /* var DownloadCollect = new BlockingCollection<TorrentInfo2>(); * var DateRecordC = new BlockingCollection<DateRecord>(); * var CounT = 0; * Task.Factory.StartNew(() => * { * using (var db = new LiteDatabase(@"Nyaa.db")) * { * var DateRecord = db.GetCollection<TorrentInfo>("NyaaDB"); * CounT = DateRecord.Count(); * Parallel.ForEach(DateRecord.FindAll(), VARIABLE => * { * if (!string.IsNullOrEmpty(VARIABLE.Torrent)) * { * DownloadCollect.TryAdd(new TorrentInfo2() * { * Catagory = VARIABLE.Catagory, * Timestamp = VARIABLE.id, * Class = VARIABLE.Class, * Title = VARIABLE.Title, * Torrent = VARIABLE.Torrent, * Magnet = VARIABLE.Magnet, * Size = VARIABLE.Size, * Date = VARIABLE.Date, * Up = VARIABLE.Up, * Leeches = VARIABLE.Leeches, * Complete = VARIABLE.Complete * }); * } * }); * DownloadCollect.CompleteAdding(); * var Record = db.GetCollection<DateRecord>("DateRecord"); * * Parallel.ForEach(Record.FindAll(), VARIABLE => { DateRecordC.TryAdd(VARIABLE); }); * DateRecordC.CompleteAdding(); * CounT = Record.Count(); * } * }); * var NowC = 0; * Task.Factory.StartNew(() => * { * using (var db = new LiteDatabase(@"Nyaa2.db")) * { * var SettingData = db.GetCollection<GlobalSet>("Setting"); * SettingData.Upsert(new GlobalSet {_id = "Address", Value = "https://sukebei.nyaa.si/"}); * SettingData.Upsert(new GlobalSet {_id = "LastCount", Value = "2790"}); * var DateRecord = db.GetCollection<DateRecord>("DateRecord"); * DateRecord.EnsureIndex(X => X._id); * var NyaaDB = db.GetCollection<TorrentInfo2>("NyaaDB"); * NyaaDB.EnsureIndex(x => x.Catagory); * NyaaDB.EnsureIndex(x => x.Date); * NyaaDB.EnsureIndex(x => x.id); * * Parallel.ForEach(DownloadCollect.GetConsumingEnumerable(), * VARIABLE => * { * NyaaDB.Insert(VARIABLE); * Interlocked.Increment(ref NowC); * }); * NowC = 0; * Parallel.ForEach(DateRecordC.GetConsumingEnumerable(), * VARIABLE => * { * DateRecord.Insert(VARIABLE); * Interlocked.Increment(ref NowC); * }); * } * }); * * * while (true) * { * Console.SetCursorPosition(0, 0); * Console.Write(" "); * Console.SetCursorPosition(0, 0); * Console.Write($"{NowC}/{CounT}"); * Thread.Sleep(500); * } * */ /* do * { * * var ST = string.Empty; * * for (int i = 0; i < new Random().Next(10, 30); i++) * { * ST += $"启动{i}"; * } * Loger.Instance.LocalInfo(ST); * Thread.Sleep(1); * } while (true);*/ Loger.Instance.LocalInfo($"启动"); DataBaseCommand.Init(); Loger.Instance.LocalInfo("数据库初始化完毕"); Task.Factory.StartNew(() => { Loger.Instance.LocalInfo("主线程启动"); new DownLoop().DownLoopAsync(); }, TaskCreationOptions.LongRunning); Console.CancelKeyPress += delegate { DataBaseCommand.SaveLastCountStatus(); }; //new DownWork(); /* var Web= new WebClient(); * try * { * Web.DownloadString("https://sukebei.nyaa.si/?p=1"); * } * catch (Exception ex) * { * while (ex != null) * { * Console.WriteLine(ex.Message); * ex = ex.InnerException; * * } * }*/ // Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd")); // new DownWork(); /* Task.Factory.StartNew(() => * { * Loger.Instance.Info("开始获取网页数据"); * new DownloadHelp(Setting.LastPageIndex); * * }, Setting.CancelSign.Token); * Task.Factory.StartNew(() => * { * Loger.Instance.Info("启动网页数据分析和保存"); * //管道剩余待处理数据 * //剩余处理时间 * //处理完毕且管道已经清空 * foreach (var item in DownloadHelp.DownloadCollect.GetConsumingEnumerable()) * { * * } * * }, Setting.CancelSign.Token);*/ //test //new WebPageGet(@"https://sukebei.nyaa.si/?p=500000"); //GetDataFromDataBase(); // var ret = new HandlerHtml(File.ReadAllText("save.txt")); // SaveToDataBaseFormList(ret.AnalysisData.Values); //SaveToDataBaseOneByOne(ret.AnalysisData.Values); var TCPCmd = TCPCommand.Init(1000); TCPCmd.StartListener(); }