internal void DownLoopAsync()
{
//DownNewDayAsync();
Setting.LastPageIndex = 12766;
DownAsync();
var PageHandler = new HandlerHtml();
Task.Factory.StartNew(() =>
{
foreach (var Item in DoF.DownloadCollect.GetConsumingEnumerable())
{
Setting.LastPageIndex = Item.Item1;
DataBaseCommand.SavePage(Item.Item2);
if (PageHandler.HandlerToHtml(Item.Item2) != 75)
{
Loger.Instance.LocalInfo("当前获得条目小于75条,检查是否完成获取");
DoF.DownloadCollect.CompleteAdding();
DoF.CancelSign.Cancel();
}
}
}, TaskCreationOptions.LongRunning);
Task.Factory.StartNew(() =>
{
foreach (var Item in PageHandler.DataCollect.GetConsumingEnumerable())
{
var Status = PageInDateStatus(Item.Item1.Item1);
if (Status == -1)
{
DataBaseCommand.SaveToDataBaseRange(Item.Item2, Item.Item1.Item2, true);
}
else if (Status == 0)
{
DataBaseCommand.SaveToDataBaseOneByOne(Item.Item2, Item.Item1.Item2, true);
}
else
{
Loger.Instance.LocalInfo($"发现{Item.Item1.Item1}已经存在");
DataBaseCommand.SaveToDataBaseRange(Item.Item2, Item.Item1.Item2, true);
//return true;
}
}
int PageInDateStatus(string Date)
{
var Status = DataBaseCommand.GetDateInfo(Date);
if (Status == null)
{
return(-1);
}
if (Status.Status)
{
return(1);
}
return(0);
}
}, TaskCreationOptions.LongRunning);
}
public int HandlerToHtml(string result)
{
if (AnalysisData == null)
{
AnalysisData = new ConcurrentDictionary <int, TorrentInfo>();
}
var LoopConut = 0;
if (result != "")
{
Loger.Instance.DateInfo(DateOfNow);
Interlocked.Increment(ref pageCount);
var HtmlDoc = new HtmlDocument();
HtmlDoc.LoadHtml(result);
foreach (var item in HtmlDoc.DocumentNode.SelectNodes(@"/html/body/div[1]/div[2]/table/tbody/tr"))
{
Interlocked.Increment(ref LoopConut);
var TempData = new TorrentInfo();
var temp = HtmlNode.CreateNode(item.OuterHtml);
TempData.Class = item.Attributes["class"].Value;
TempData.Catagory = temp.SelectSingleNode("td[1]/a").Attributes["title"]
.Value;
TempData.Title = temp.SelectSingleNode("td[2]/a").Attributes["title"]
.Value;
TempData.Url = temp.SelectSingleNode("td[2]/a").Attributes["href"]
.Value;
TempData.Torrent = temp.SelectSingleNode("td[3]/a[1]").Attributes["href"].Value;
if (TempData.Torrent.StartsWith("magnet"))
{
TempData.Magnet = TempData.Torrent;
TempData.Torrent = "";
}
else
{
TempData.Magnet = temp.SelectSingleNode("td[3]/a[2]").Attributes["href"].Value;
}
TempData.Size = temp.SelectSingleNode("td[4]").InnerText;
TempData.Timestamp = int.Parse(temp.SelectSingleNode("td[5]").Attributes["data-timestamp"].Value);
TempData.Date = temp.SelectSingleNode("td[5]").InnerText;
TempData.Up = temp.SelectSingleNode("td[6]").InnerText;
TempData.Leeches = temp.SelectSingleNode("td[7]").InnerText;
TempData.Complete = temp.SelectSingleNode("td[8]").InnerText;
if (string.IsNullOrEmpty(DateOfNow) || AddFin)
{
if (!AddFin)
{
Loger.Instance.WithTimeStart($"开始获取{TempData.Day}数据", Time);
}
DateOfNow = TempData.Day;
AddFin = false;
}
if (DateOfNow != TempData.Day)
{
DataCollect.TryAdd(
new Tuple <Tuple <string, int>, System.Collections.Generic.ICollection <TorrentInfo> >(
new Tuple <string, int>(DateOfNow, pageCount), AnalysisData.Values));
pageCount = 0;
AnalysisData.Clear();
AddFin = true;
Loger.Instance.WithTimeRestart($"结束获取{DateOfNow}数据", Time);
}
AnalysisData.AddOrUpdate(TempData.id, TempData, (key, Value) => TempData);
}
if (DateOfNow == "2000-01-01")
{
DataBaseCommand.SaveToDataBaseRange(AnalysisData.Values, pageCount, false);
AnalysisData.Clear();
}
}
return(LoopConut);
}