Exemplo n.º 1
0
        internal void DownLoopAsync()
        {
            //DownNewDayAsync();
            Setting.LastPageIndex = 12766;
            DownAsync();
            var PageHandler = new HandlerHtml();

            Task.Factory.StartNew(() =>
            {
                foreach (var Item in DoF.DownloadCollect.GetConsumingEnumerable())
                {
                    Setting.LastPageIndex = Item.Item1;
                    DataBaseCommand.SavePage(Item.Item2);
                    if (PageHandler.HandlerToHtml(Item.Item2) != 75)
                    {
                        Loger.Instance.LocalInfo("当前获得条目小于75条,检查是否完成获取");
                        DoF.DownloadCollect.CompleteAdding();
                        DoF.CancelSign.Cancel();
                    }
                }
            }, TaskCreationOptions.LongRunning);
            Task.Factory.StartNew(() =>
            {
                foreach (var Item in PageHandler.DataCollect.GetConsumingEnumerable())
                {
                    var Status = PageInDateStatus(Item.Item1.Item1);
                    if (Status == -1)
                    {
                        DataBaseCommand.SaveToDataBaseRange(Item.Item2, Item.Item1.Item2, true);
                    }
                    else if (Status == 0)
                    {
                        DataBaseCommand.SaveToDataBaseOneByOne(Item.Item2, Item.Item1.Item2, true);
                    }
                    else
                    {
                        Loger.Instance.LocalInfo($"发现{Item.Item1.Item1}已经存在");
                        DataBaseCommand.SaveToDataBaseRange(Item.Item2, Item.Item1.Item2, true);
                        //return true;
                    }
                }

                int PageInDateStatus(string Date)
                {
                    var Status = DataBaseCommand.GetDateInfo(Date);
                    if (Status == null)
                    {
                        return(-1);
                    }

                    if (Status.Status)
                    {
                        return(1);
                    }

                    return(0);
                }
            }, TaskCreationOptions.LongRunning);
        }
Exemplo n.º 2
0
        public int HandlerToHtml(string result)
        {
            if (AnalysisData == null)
            {
                AnalysisData = new ConcurrentDictionary <int, TorrentInfo>();
            }
            var LoopConut = 0;

            if (result != "")
            {
                Loger.Instance.DateInfo(DateOfNow);
                Interlocked.Increment(ref pageCount);
                var HtmlDoc = new HtmlDocument();
                HtmlDoc.LoadHtml(result);
                foreach (var item in HtmlDoc.DocumentNode.SelectNodes(@"/html/body/div[1]/div[2]/table/tbody/tr"))
                {
                    Interlocked.Increment(ref LoopConut);
                    var TempData = new TorrentInfo();
                    var temp     = HtmlNode.CreateNode(item.OuterHtml);
                    TempData.Class    = item.Attributes["class"].Value;
                    TempData.Catagory = temp.SelectSingleNode("td[1]/a").Attributes["title"]
                                        .Value;
                    TempData.Title = temp.SelectSingleNode("td[2]/a").Attributes["title"]
                                     .Value;
                    TempData.Url = temp.SelectSingleNode("td[2]/a").Attributes["href"]
                                   .Value;
                    TempData.Torrent = temp.SelectSingleNode("td[3]/a[1]").Attributes["href"].Value;
                    if (TempData.Torrent.StartsWith("magnet"))
                    {
                        TempData.Magnet  = TempData.Torrent;
                        TempData.Torrent = "";
                    }
                    else
                    {
                        TempData.Magnet = temp.SelectSingleNode("td[3]/a[2]").Attributes["href"].Value;
                    }

                    TempData.Size = temp.SelectSingleNode("td[4]").InnerText;

                    TempData.Timestamp = int.Parse(temp.SelectSingleNode("td[5]").Attributes["data-timestamp"].Value);
                    TempData.Date      = temp.SelectSingleNode("td[5]").InnerText;
                    TempData.Up        = temp.SelectSingleNode("td[6]").InnerText;
                    TempData.Leeches   = temp.SelectSingleNode("td[7]").InnerText;
                    TempData.Complete  = temp.SelectSingleNode("td[8]").InnerText;
                    if (string.IsNullOrEmpty(DateOfNow) || AddFin)
                    {
                        if (!AddFin)
                        {
                            Loger.Instance.WithTimeStart($"开始获取{TempData.Day}数据", Time);
                        }
                        DateOfNow = TempData.Day;
                        AddFin    = false;
                    }

                    if (DateOfNow != TempData.Day)
                    {
                        DataCollect.TryAdd(
                            new Tuple <Tuple <string, int>, System.Collections.Generic.ICollection <TorrentInfo> >(
                                new Tuple <string, int>(DateOfNow, pageCount), AnalysisData.Values));
                        pageCount = 0;
                        AnalysisData.Clear();
                        AddFin = true;
                        Loger.Instance.WithTimeRestart($"结束获取{DateOfNow}数据", Time);
                    }
                    AnalysisData.AddOrUpdate(TempData.id, TempData, (key, Value) => TempData);
                }

                if (DateOfNow == "2000-01-01")
                {
                    DataBaseCommand.SaveToDataBaseRange(AnalysisData.Values, pageCount, false);
                    AnalysisData.Clear();
                }
            }

            return(LoopConut);
        }
Exemplo n.º 3
0
        private static void Main(string[] args)
        {
            /*  var DownloadCollect = new BlockingCollection<TorrentInfo2>();
             * var DateRecordC = new BlockingCollection<DateRecord>();
             * var CounT = 0;
             * Task.Factory.StartNew(() =>
             * {
             *    using (var db = new LiteDatabase(@"Nyaa.db"))
             *    {
             *        var DateRecord = db.GetCollection<TorrentInfo>("NyaaDB");
             *        CounT = DateRecord.Count();
             *        Parallel.ForEach(DateRecord.FindAll(), VARIABLE =>
             *        {
             *            if (!string.IsNullOrEmpty(VARIABLE.Torrent))
             *            {
             *                DownloadCollect.TryAdd(new TorrentInfo2()
             *                {
             *                    Catagory = VARIABLE.Catagory,
             *                    Timestamp = VARIABLE.id,
             *                    Class = VARIABLE.Class,
             *                    Title = VARIABLE.Title,
             *                    Torrent = VARIABLE.Torrent,
             *                    Magnet = VARIABLE.Magnet,
             *                    Size = VARIABLE.Size,
             *                    Date = VARIABLE.Date,
             *                    Up = VARIABLE.Up,
             *                    Leeches = VARIABLE.Leeches,
             *                    Complete = VARIABLE.Complete
             *                });
             *            }
             *        });
             *        DownloadCollect.CompleteAdding();
             *        var Record = db.GetCollection<DateRecord>("DateRecord");
             *
             *        Parallel.ForEach(Record.FindAll(), VARIABLE => { DateRecordC.TryAdd(VARIABLE); });
             *        DateRecordC.CompleteAdding();
             *        CounT = Record.Count();
             *    }
             * });
             * var NowC = 0;
             * Task.Factory.StartNew(() =>
             * {
             *    using (var db = new LiteDatabase(@"Nyaa2.db"))
             *    {
             *        var SettingData = db.GetCollection<GlobalSet>("Setting");
             *        SettingData.Upsert(new GlobalSet {_id = "Address", Value = "https://sukebei.nyaa.si/"});
             *        SettingData.Upsert(new GlobalSet {_id = "LastCount", Value = "2790"});
             *        var DateRecord = db.GetCollection<DateRecord>("DateRecord");
             *        DateRecord.EnsureIndex(X => X._id);
             *        var NyaaDB = db.GetCollection<TorrentInfo2>("NyaaDB");
             *        NyaaDB.EnsureIndex(x => x.Catagory);
             *        NyaaDB.EnsureIndex(x => x.Date);
             *        NyaaDB.EnsureIndex(x => x.id);
             *
             *        Parallel.ForEach(DownloadCollect.GetConsumingEnumerable(),
             *            VARIABLE =>
             *            {
             *                NyaaDB.Insert(VARIABLE);
             *                Interlocked.Increment(ref NowC);
             *            });
             *        NowC = 0;
             *        Parallel.ForEach(DateRecordC.GetConsumingEnumerable(),
             *            VARIABLE =>
             *            {
             *                DateRecord.Insert(VARIABLE);
             *                Interlocked.Increment(ref NowC);
             *            });
             *    }
             * });
             *
             *
             * while (true)
             * {
             *    Console.SetCursorPosition(0, 0);
             *    Console.Write("                                  ");
             *    Console.SetCursorPosition(0, 0);
             *    Console.Write($"{NowC}/{CounT}");
             *    Thread.Sleep(500);
             * }
             *
             */

            /*   do
             * {
             *
             *        var ST = string.Empty;
             *
             *     for (int i = 0; i < new Random().Next(10, 30); i++)
             *     {
             *         ST += $"启动{i}";
             *     }
             *     Loger.Instance.LocalInfo(ST);
             *     Thread.Sleep(1);
             * } while (true);*/

            Loger.Instance.LocalInfo($"启动");
            DataBaseCommand.Init();
            Loger.Instance.LocalInfo("数据库初始化完毕");
            Task.Factory.StartNew(() =>
            {
                Loger.Instance.LocalInfo("主线程启动");
                new DownLoop().DownLoopAsync();
            }, TaskCreationOptions.LongRunning);
            Console.CancelKeyPress += delegate { DataBaseCommand.SaveLastCountStatus(); };


            //new DownWork();

            /*   var Web= new  WebClient();
             *    try
             *    {
             *        Web.DownloadString("https://sukebei.nyaa.si/?p=1");
             *    }
             *    catch (Exception ex)
             *    {
             *        while (ex != null)
             *        {
             *            Console.WriteLine(ex.Message);
             *            ex = ex.InnerException;
             *
             *        }
             *    }*/

            // Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd"));


            // new DownWork();

            /*  Task.Factory.StartNew(() =>
            *  {
            *     Loger.Instance.Info("开始获取网页数据");
            *    new DownloadHelp(Setting.LastPageIndex);
            *
            *  }, Setting.CancelSign.Token);
            *  Task.Factory.StartNew(() =>
            *  {
            *     Loger.Instance.Info("启动网页数据分析和保存");
            *     //管道剩余待处理数据
            *     //剩余处理时间
            *     //处理完毕且管道已经清空
            *     foreach (var item in DownloadHelp.DownloadCollect.GetConsumingEnumerable())
            *     {
            *
            *     }
            *
            *  }, Setting.CancelSign.Token);*/

            //test
            //new WebPageGet(@"https://sukebei.nyaa.si/?p=500000");
            //GetDataFromDataBase();
            // var ret = new HandlerHtml(File.ReadAllText("save.txt"));
            // SaveToDataBaseFormList(ret.AnalysisData.Values);
            //SaveToDataBaseOneByOne(ret.AnalysisData.Values);
            var TCPCmd = TCPCommand.Init(1000);

            TCPCmd.StartListener();
        }