private void DoAutoAddFreeProxy(FreeProxySource src)
{
foreach (var path in src.Paths)
{
RestClient client = new RestClient(src.Host);
var req = client.TakeRequest(path);
req.Node.Method = Method.GET;
req.Node.AddHeader("User-Agent", "Mozilla / 5.0(Windows NT 10.0; WOW64; rv: 63.0) Gecko / 20100101 Firefox / 63.0");
req.Node.AddHeader("Accept", "text / html,application / xhtml + xml,application / xml; q = 0.9,*/*;q=0.8");
req.Node.AddHeader("Accept-Language", "zh-TW,zh;q=0.8,en-US;q=0.5,en;q=0.3");
req.Node.AddHeader("Accept-Encoding", "gzip, deflate");
var dReq = ((dynamic)req);
//dReq.Parameter.url = HttpUtility.UrlPathEncode(longUrl);
var rlt = client.Execute(req);
var htmlText = encoding.GetString(rlt.RawBytes);
var matchs = src.ContentRegex.Matches(htmlText);
foreach (Match match in matchs)
{
if (match.Success)
{
var ip = match.Groups[1].Value;
if (string.IsNullOrEmpty(ip))
{
continue;
}
if (!string.IsNullOrEmpty(src.ReplaceDotString))
{
ip = ip.Replace(src.ReplaceDotString, "");
}
if (!string.IsNullOrEmpty(src.ReplaceDotString))
{
ip = ip.Replace(src.ReplaceDotString, ".");
}
var port = Convert.ToInt32(match.Groups[2].Value, src.IsHexPortType ? 16 : 10);
var newProxy = new WebProxyServerEntity()
{
FailedNum = 0, IP = ip, Port = port
};
if (this.ProxyServers.Find(pxy => pxy.IP == newProxy.IP) == null)
{
if (CheckProxy(newProxy))
{
ProxyServers.Add(newProxy);
}
}
}
}
}
}
private WebProxyServerEntity OnRestClientExecuteErrorEvent(WebProxyServerEntity oldProxy, Exception e)
{
oldProxy.FailedNum = oldProxy.FailedNum + 1;
SaveProxyServers();
for (int i = 0; i < ProxyServers.Count; i++)
{
WebProxyServerEntity proxyServer = NextProxyServer(failedNumForErrorClient, 0);
if (proxyServer != null)
{
return(proxyServer);
}
}
return(null);
}
public bool CheckProxy(WebProxyServerEntity proxyEntity, int timeout = 1000)
{
var proxyEntitys = ProxyServers;
var url = "http://humanstxt.org";
try
{
var client = new DynamicRestClient(url);
var req = client.TakeRequest("/humans.txt");
var now = DateTime.Now;
client.Timeout = 5000; //本機
if (!string.IsNullOrEmpty(proxyEntity.IP))
{
client.Proxy = new WebProxy(proxyEntity.IP, proxyEntity.Port);
client.Timeout = timeout;
client.ReadWriteTimeout = timeout * 3;//避免某些會卡住 124.219.176.139:47801
}
client.ProxyServerEntity = proxyEntity;
int ct = proxyEntity.FailedNum;
var rlt = client.Execute2((RestRequest)req, false);
var info = encoding.GetString(rlt.RawBytes);
TimeSpan sp = DateTime.Now - now;
if (!info.Contains("* TEAM *"))
{
proxyEntity.FailedNum = ct + 1;
}
else if (proxyEntity.FailedNum >= 0)
{
proxyEntity.FailedNum = -1;
}
return(true);
}
catch//(Exception e)
{
proxyEntity.FailedNum = proxyEntity.FailedNum + 1;
return(false);
}
}
