private void Test() { CodeSite.EnterMethod(this, "Test"); ScrapingBrowser scrapingBrowser = new ScrapingBrowser(); scrapingBrowser.IgnoreCookies = true; re: try { var s = scrapingBrowser.NavigateTo(new Uri("https://www.yidaiyilu.gov.cn/info/iList.jsp?cat_id=10009"), HttpVerb.Get, ""); CodeSite.Send("s", s); } catch (Exception ex) { if (ex is WebException we && we.Response is HttpWebResponse hwr) { if ((int)hwr.StatusCode == 521) { CodeSite.SendReminder("破解爬虫防护"); scrapingBrowser.SetCookies(new Uri("https://www.yidaiyilu.gov.cn/"), Jsl.GetCookies(hwr)); goto re; } } } finally { CodeSite.ExitMethod(this, "Test"); } }
public Task GlobalAsync() { CodeSite.EnterMethod(this, "GlobalAsync"); try { CheckTask(); cts = new CancellationTokenSource(); Task = Task.Run(() => { try { var save = SaveAsync(); Thread.CurrentThread.Name = "GlobalAsync"; IPAddress ip = IPAddress.Parse(LastProgress.Ping); CodeSite.Send("LastIP", ip.ToString()); IPNetwork net = IPNetwork.Parse(ip.ToString(), 10); uint count = (uint)net.Total / 256; for (uint i = net.Network.ToUInt32() >> 8; i <= uint.MaxValue >> 8; i++) { if (cts.IsCancellationRequested) { break; } uint uip = i << 8; if (!IPNetworkSet.IPv4Reserved.Contains(uip) && !IPNetworkSet.IPv4Assigned.Contains(uip) ^ Properties.Settings.Default.IPv4Assigned) { for (uint j = 0; j < 256; j++) { listIP.Add(uip + j); } } if (i % count == count - 1)//每组 { if (listIP.Count > 0) { //CodeSite.Send("listIP.Count", listIP.Count); LastProgress.Ping = listIP.First().ToIPAddress().ToString(); CodeSite.Send("StartIP", LastProgress.Ping); progressFormat = string.Format("{0}-{1},{{0,8}}/{2},新增{{1,8}}", listIP.First().ToIPAddress(), listIP.Last().ToIPAddress(), listIP.Count); ExceptDB(); CheckList(); listIP.Clear(); } } } bcIPv4SSL.CompleteAdding(); save.Wait(); } catch (Exception ex) { ex.SendCodeSite("GlobalAsync"); } }, cts.Token); return(Task); } finally { CodeSite.ExitMethod(this, "GlobalAsync"); } }
void SaveThreadStop() { CodeSite.EnterMethod(this, "SaveThreadStop"); try { ctsSave.Cancel(); taskSave.Wait(); } finally { CodeSite.ExitMethod(this, "SaveThreadStop"); } }
public bool Cancel() { if (cts != null && !cts.IsCancellationRequested) { CodeSite.EnterMethod(this, "Cancel"); try { cts.Cancel(); CodeSite.SendNote("cts.Cancel();"); task.Wait(); return(true); } finally { CodeSite.ExitMethod(this, "Cancel"); } } return(false); }
void SaveThreadStart() { CodeSite.EnterMethod(this, "SaveThreadStart"); try { ctsSave = new CancellationTokenSource(); taskSave = Task.Factory.StartNew(() => { IPv4SSL[] ipa = new IPv4SSL[100]; int c; while (!ctsSave.IsCancellationRequested || !IPStack.IsEmpty) { if ((c = IPStack.TryPopRange(ipa)) > 0) { var ip = ipa.Take(c); if (ProgressIP != null) { foreach (var item in ip) { if (item.Isgws) { ProgressIP.Report(item); } } } while (!SaveDB(ip) && !ctsSave.IsCancellationRequested) { Thread.Sleep(1000); } } if (IPStack.IsEmpty) { Thread.Sleep(1000); } } }, ctsSave.Token); } finally { CodeSite.ExitMethod(this, "SaveThreadStart"); } }
public Task GlobalAsync() { CodeSite.EnterMethod(this, "GlobalAsync"); try { CheckTask(); cts = new CancellationTokenSource(); Task = Task.Run(() => { try { var save = SaveAsync(); Thread.CurrentThread.Name = "GlobalAsync"; IPAddress ip = IPAddress.Parse(LastProgress.Ssl); CodeSite.Send("LastIP", ip.ToString()); while (SelectDB(ip) && !cts.IsCancellationRequested) { listIP = listIPv4SSL.Select(f => (uint)f.Address).ToList(); LastProgress.Ssl = ip.ToString(); CodeSite.Send("StartIP", string.Format("{0} - {1}", LastProgress.Ssl, listIP.Count)); progressFormat = string.Format("{0}-{1},{{0,8}}/{2},新增{{1,8}}", listIP.First().ToIPAddress(), listIP.Last().ToIPAddress(), listIP.Count); ip = (listIP.Last() + 1).ToIPAddress(); Shuffle(); CheckList(); listIP.Clear(); } bcIPv4SSL.CompleteAdding(); save.Wait(); } catch (Exception ex) { ex.SendCodeSite("GlobalAsync"); } }, cts.Token); return(Task); } finally { CodeSite.ExitMethod(this, "GlobalAsync"); } }