internal void DownLoopAsync()
{
//DownNewDayAsync();
Setting.LastPageIndex = 12766;
DownAsync();
var PageHandler = new HandlerHtml();
Task.Factory.StartNew(() =>
{
foreach (var Item in DoF.DownloadCollect.GetConsumingEnumerable())
{
Setting.LastPageIndex = Item.Item1;
DataBaseCommand.SavePage(Item.Item2);
if (PageHandler.HandlerToHtml(Item.Item2) != 75)
{
Loger.Instance.LocalInfo("当前获得条目小于75条,检查是否完成获取");
DoF.DownloadCollect.CompleteAdding();
DoF.CancelSign.Cancel();
}
}
}, TaskCreationOptions.LongRunning);
Task.Factory.StartNew(() =>
{
foreach (var Item in PageHandler.DataCollect.GetConsumingEnumerable())
{
var Status = PageInDateStatus(Item.Item1.Item1);
if (Status == -1)
{
DataBaseCommand.SaveToDataBaseRange(Item.Item2, Item.Item1.Item2, true);
}
else if (Status == 0)
{
DataBaseCommand.SaveToDataBaseOneByOne(Item.Item2, Item.Item1.Item2, true);
}
else
{
Loger.Instance.LocalInfo($"发现{Item.Item1.Item1}已经存在");
DataBaseCommand.SaveToDataBaseRange(Item.Item2, Item.Item1.Item2, true);
//return true;
}
}
int PageInDateStatus(string Date)
{
var Status = DataBaseCommand.GetDateInfo(Date);
if (Status == null)
{
return(-1);
}
if (Status.Status)
{
return(1);
}
return(0);
}
}, TaskCreationOptions.LongRunning);
}
public int HandlerToHtml(string result)
{
if (AnalysisData == null)
{
AnalysisData = new ConcurrentDictionary <int, TorrentInfo>();
}
var LoopConut = 0;
if (result != "")
{
Loger.Instance.DateInfo(DateOfNow);
Interlocked.Increment(ref pageCount);
var HtmlDoc = new HtmlDocument();
HtmlDoc.LoadHtml(result);
foreach (var item in HtmlDoc.DocumentNode.SelectNodes(@"/html/body/div[1]/div[2]/table/tbody/tr"))
{
Interlocked.Increment(ref LoopConut);
var TempData = new TorrentInfo();
var temp = HtmlNode.CreateNode(item.OuterHtml);
TempData.Class = item.Attributes["class"].Value;
TempData.Catagory = temp.SelectSingleNode("td[1]/a").Attributes["title"]
.Value;
TempData.Title = temp.SelectSingleNode("td[2]/a").Attributes["title"]
.Value;
TempData.Url = temp.SelectSingleNode("td[2]/a").Attributes["href"]
.Value;
TempData.Torrent = temp.SelectSingleNode("td[3]/a[1]").Attributes["href"].Value;
if (TempData.Torrent.StartsWith("magnet"))
{
TempData.Magnet = TempData.Torrent;
TempData.Torrent = "";
}
else
{
TempData.Magnet = temp.SelectSingleNode("td[3]/a[2]").Attributes["href"].Value;
}
TempData.Size = temp.SelectSingleNode("td[4]").InnerText;
TempData.Timestamp = int.Parse(temp.SelectSingleNode("td[5]").Attributes["data-timestamp"].Value);
TempData.Date = temp.SelectSingleNode("td[5]").InnerText;
TempData.Up = temp.SelectSingleNode("td[6]").InnerText;
TempData.Leeches = temp.SelectSingleNode("td[7]").InnerText;
TempData.Complete = temp.SelectSingleNode("td[8]").InnerText;
if (string.IsNullOrEmpty(DateOfNow) || AddFin)
{
if (!AddFin)
{
Loger.Instance.WithTimeStart($"开始获取{TempData.Day}数据", Time);
}
DateOfNow = TempData.Day;
AddFin = false;
}
if (DateOfNow != TempData.Day)
{
DataCollect.TryAdd(
new Tuple <Tuple <string, int>, System.Collections.Generic.ICollection <TorrentInfo> >(
new Tuple <string, int>(DateOfNow, pageCount), AnalysisData.Values));
pageCount = 0;
AnalysisData.Clear();
AddFin = true;
Loger.Instance.WithTimeRestart($"结束获取{DateOfNow}数据", Time);
}
AnalysisData.AddOrUpdate(TempData.id, TempData, (key, Value) => TempData);
}
if (DateOfNow == "2000-01-01")
{
DataBaseCommand.SaveToDataBaseRange(AnalysisData.Values, pageCount, false);
AnalysisData.Clear();
}
}
return(LoopConut);
}
private static void Main(string[] args)
{
/* var DownloadCollect = new BlockingCollection<TorrentInfo2>();
* var DateRecordC = new BlockingCollection<DateRecord>();
* var CounT = 0;
* Task.Factory.StartNew(() =>
* {
* using (var db = new LiteDatabase(@"Nyaa.db"))
* {
* var DateRecord = db.GetCollection<TorrentInfo>("NyaaDB");
* CounT = DateRecord.Count();
* Parallel.ForEach(DateRecord.FindAll(), VARIABLE =>
* {
* if (!string.IsNullOrEmpty(VARIABLE.Torrent))
* {
* DownloadCollect.TryAdd(new TorrentInfo2()
* {
* Catagory = VARIABLE.Catagory,
* Timestamp = VARIABLE.id,
* Class = VARIABLE.Class,
* Title = VARIABLE.Title,
* Torrent = VARIABLE.Torrent,
* Magnet = VARIABLE.Magnet,
* Size = VARIABLE.Size,
* Date = VARIABLE.Date,
* Up = VARIABLE.Up,
* Leeches = VARIABLE.Leeches,
* Complete = VARIABLE.Complete
* });
* }
* });
* DownloadCollect.CompleteAdding();
* var Record = db.GetCollection<DateRecord>("DateRecord");
*
* Parallel.ForEach(Record.FindAll(), VARIABLE => { DateRecordC.TryAdd(VARIABLE); });
* DateRecordC.CompleteAdding();
* CounT = Record.Count();
* }
* });
* var NowC = 0;
* Task.Factory.StartNew(() =>
* {
* using (var db = new LiteDatabase(@"Nyaa2.db"))
* {
* var SettingData = db.GetCollection<GlobalSet>("Setting");
* SettingData.Upsert(new GlobalSet {_id = "Address", Value = "https://sukebei.nyaa.si/"});
* SettingData.Upsert(new GlobalSet {_id = "LastCount", Value = "2790"});
* var DateRecord = db.GetCollection<DateRecord>("DateRecord");
* DateRecord.EnsureIndex(X => X._id);
* var NyaaDB = db.GetCollection<TorrentInfo2>("NyaaDB");
* NyaaDB.EnsureIndex(x => x.Catagory);
* NyaaDB.EnsureIndex(x => x.Date);
* NyaaDB.EnsureIndex(x => x.id);
*
* Parallel.ForEach(DownloadCollect.GetConsumingEnumerable(),
* VARIABLE =>
* {
* NyaaDB.Insert(VARIABLE);
* Interlocked.Increment(ref NowC);
* });
* NowC = 0;
* Parallel.ForEach(DateRecordC.GetConsumingEnumerable(),
* VARIABLE =>
* {
* DateRecord.Insert(VARIABLE);
* Interlocked.Increment(ref NowC);
* });
* }
* });
*
*
* while (true)
* {
* Console.SetCursorPosition(0, 0);
* Console.Write(" ");
* Console.SetCursorPosition(0, 0);
* Console.Write($"{NowC}/{CounT}");
* Thread.Sleep(500);
* }
*
*/
/* do
* {
*
* var ST = string.Empty;
*
* for (int i = 0; i < new Random().Next(10, 30); i++)
* {
* ST += $"启动{i}";
* }
* Loger.Instance.LocalInfo(ST);
* Thread.Sleep(1);
* } while (true);*/
Loger.Instance.LocalInfo($"启动");
DataBaseCommand.Init();
Loger.Instance.LocalInfo("数据库初始化完毕");
Task.Factory.StartNew(() =>
{
Loger.Instance.LocalInfo("主线程启动");
new DownLoop().DownLoopAsync();
}, TaskCreationOptions.LongRunning);
Console.CancelKeyPress += delegate { DataBaseCommand.SaveLastCountStatus(); };
//new DownWork();
/* var Web= new WebClient();
* try
* {
* Web.DownloadString("https://sukebei.nyaa.si/?p=1");
* }
* catch (Exception ex)
* {
* while (ex != null)
* {
* Console.WriteLine(ex.Message);
* ex = ex.InnerException;
*
* }
* }*/
// Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd"));
// new DownWork();
/* Task.Factory.StartNew(() =>
* {
* Loger.Instance.Info("开始获取网页数据");
* new DownloadHelp(Setting.LastPageIndex);
*
* }, Setting.CancelSign.Token);
* Task.Factory.StartNew(() =>
* {
* Loger.Instance.Info("启动网页数据分析和保存");
* //管道剩余待处理数据
* //剩余处理时间
* //处理完毕且管道已经清空
* foreach (var item in DownloadHelp.DownloadCollect.GetConsumingEnumerable())
* {
*
* }
*
* }, Setting.CancelSign.Token);*/
//test
//new WebPageGet(@"https://sukebei.nyaa.si/?p=500000");
//GetDataFromDataBase();
// var ret = new HandlerHtml(File.ReadAllText("save.txt"));
// SaveToDataBaseFormList(ret.AnalysisData.Values);
//SaveToDataBaseOneByOne(ret.AnalysisData.Values);
var TCPCmd = TCPCommand.Init(1000);
TCPCmd.StartListener();
}
