private void Test()
{
CodeSite.EnterMethod(this, "Test");
ScrapingBrowser scrapingBrowser = new ScrapingBrowser();
scrapingBrowser.IgnoreCookies = true;
re:
try
{
var s = scrapingBrowser.NavigateTo(new Uri("https://www.yidaiyilu.gov.cn/info/iList.jsp?cat_id=10009"), HttpVerb.Get, "");
CodeSite.Send("s", s);
}
catch (Exception ex)
{
if (ex is WebException we && we.Response is HttpWebResponse hwr)
{
if ((int)hwr.StatusCode == 521)
{
CodeSite.SendReminder("破解爬虫防护");
scrapingBrowser.SetCookies(new Uri("https://www.yidaiyilu.gov.cn/"), Jsl.GetCookies(hwr));
goto re;
}
}
}
finally
{
CodeSite.ExitMethod(this, "Test");
}
}
public Task GlobalAsync()
{
CodeSite.EnterMethod(this, "GlobalAsync");
try
{
CheckTask();
cts = new CancellationTokenSource();
Task = Task.Run(() =>
{
try
{
var save = SaveAsync();
Thread.CurrentThread.Name = "GlobalAsync";
IPAddress ip = IPAddress.Parse(LastProgress.Ping);
CodeSite.Send("LastIP", ip.ToString());
IPNetwork net = IPNetwork.Parse(ip.ToString(), 10);
uint count = (uint)net.Total / 256;
for (uint i = net.Network.ToUInt32() >> 8; i <= uint.MaxValue >> 8; i++)
{
if (cts.IsCancellationRequested)
{
break;
}
uint uip = i << 8;
if (!IPNetworkSet.IPv4Reserved.Contains(uip) && !IPNetworkSet.IPv4Assigned.Contains(uip) ^ Properties.Settings.Default.IPv4Assigned)
{
for (uint j = 0; j < 256; j++)
{
listIP.Add(uip + j);
}
}
if (i % count == count - 1)//每组
{
if (listIP.Count > 0)
{
//CodeSite.Send("listIP.Count", listIP.Count);
LastProgress.Ping = listIP.First().ToIPAddress().ToString();
CodeSite.Send("StartIP", LastProgress.Ping);
progressFormat = string.Format("{0}-{1},{{0,8}}/{2},新增{{1,8}}", listIP.First().ToIPAddress(), listIP.Last().ToIPAddress(), listIP.Count);
ExceptDB();
CheckList();
listIP.Clear();
}
}
}
bcIPv4SSL.CompleteAdding();
save.Wait();
}
catch (Exception ex)
{
ex.SendCodeSite("GlobalAsync");
}
}, cts.Token);
return(Task);
}
finally
{
CodeSite.ExitMethod(this, "GlobalAsync");
}
}
void SaveThreadStop()
{
CodeSite.EnterMethod(this, "SaveThreadStop");
try
{
ctsSave.Cancel();
taskSave.Wait();
}
finally
{
CodeSite.ExitMethod(this, "SaveThreadStop");
}
}
public bool Cancel()
{
if (cts != null && !cts.IsCancellationRequested)
{
CodeSite.EnterMethod(this, "Cancel");
try
{
cts.Cancel();
CodeSite.SendNote("cts.Cancel();");
task.Wait();
return(true);
}
finally
{
CodeSite.ExitMethod(this, "Cancel");
}
}
return(false);
}
void SaveThreadStart()
{
CodeSite.EnterMethod(this, "SaveThreadStart");
try
{
ctsSave = new CancellationTokenSource();
taskSave = Task.Factory.StartNew(() =>
{
IPv4SSL[] ipa = new IPv4SSL[100];
int c;
while (!ctsSave.IsCancellationRequested || !IPStack.IsEmpty)
{
if ((c = IPStack.TryPopRange(ipa)) > 0)
{
var ip = ipa.Take(c);
if (ProgressIP != null)
{
foreach (var item in ip)
{
if (item.Isgws)
{
ProgressIP.Report(item);
}
}
}
while (!SaveDB(ip) && !ctsSave.IsCancellationRequested)
{
Thread.Sleep(1000);
}
}
if (IPStack.IsEmpty)
{
Thread.Sleep(1000);
}
}
}, ctsSave.Token);
}
finally
{
CodeSite.ExitMethod(this, "SaveThreadStart");
}
}
public Task GlobalAsync()
{
CodeSite.EnterMethod(this, "GlobalAsync");
try
{
CheckTask();
cts = new CancellationTokenSource();
Task = Task.Run(() =>
{
try
{
var save = SaveAsync();
Thread.CurrentThread.Name = "GlobalAsync";
IPAddress ip = IPAddress.Parse(LastProgress.Ssl);
CodeSite.Send("LastIP", ip.ToString());
while (SelectDB(ip) && !cts.IsCancellationRequested)
{
listIP = listIPv4SSL.Select(f => (uint)f.Address).ToList();
LastProgress.Ssl = ip.ToString();
CodeSite.Send("StartIP", string.Format("{0} - {1}", LastProgress.Ssl, listIP.Count));
progressFormat = string.Format("{0}-{1},{{0,8}}/{2},新增{{1,8}}", listIP.First().ToIPAddress(), listIP.Last().ToIPAddress(), listIP.Count);
ip = (listIP.Last() + 1).ToIPAddress();
Shuffle();
CheckList();
listIP.Clear();
}
bcIPv4SSL.CompleteAdding();
save.Wait();
}
catch (Exception ex)
{
ex.SendCodeSite("GlobalAsync");
}
}, cts.Token);
return(Task);
}
finally
{
CodeSite.ExitMethod(this, "GlobalAsync");
}
}
