public override HashSet <ProxyInfo> ProvideProxy()
{
Dictionary <string, ProxyInfo> proxyInfos = new Dictionary <string, ProxyInfo>();
foreach (WebPageProxySource src in Sources)
{
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(src.URL);
HtmlHelper.HttpWebRequestNormalSetup(request);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
string html;
if (HtmlHelper.GetHtml(response, out html))
{
MatchCollection mc = src.Regex.Matches(html);
foreach (Match m in mc)
{
string ip = m.Groups["ip"].Value;
string port = m.Groups["port"].Value;
int porti;
if (Int32.TryParse(port, out porti))
{
if (src.PortWhiteList.Count == 0 ||
src.PortWhiteList.Contains(porti))
{
proxyInfos[ip] = new ProxyInfo
{
HttpProxy = new WebProxy(ip, Int32.Parse(port)),
Location = IPLocationSearch.GetIPLocation(ip).country
};
}
}
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
//return proxyInfos.GetRange(0,30);
List <ProxyInfo> proxies = new List <ProxyInfo>(proxyInfos.Values);
int len = proxies.Count;
Random r = new Random();
while (len > MAX_PROVIDE)
{
int random = r.Next(len);
proxies.RemoveAt(random);
len--;
}
return(new HashSet <ProxyInfo>(proxies));
}
public static IEnumerable <ProxyInfo> ReadConfig(string path)
{
List <ProxyInfo> proxies = new List <ProxyInfo>();
XmlDocument doc = new XmlDocument();
doc.Load(path);
XmlNodeList nodelist = doc.SelectNodes("//ProxyInfo");
if (nodelist == null)
{
return(proxies);
}
foreach (XmlNode node in nodelist)
{
ProxyInfo pi = new ProxyInfo();
if (node.Attributes == null)
{
continue;
}
XmlAttribute addressX = node.Attributes["Address"];
XmlAttribute latencyX = node.Attributes["Latency"];
if (addressX == null || latencyX == null)
{
continue;
}
pi.HttpProxy = new WebProxy(addressX.Value);
int rtt;
if (!Int32.TryParse(latencyX.Value, out rtt))
{
continue;
}
pi.RTT = rtt;
pi.Location = IPLocationSearch.GetIPLocation(pi.HttpProxy.Address.Host).country;
proxies.Add(pi);
}
return(proxies);
}
public override HashSet <ProxyInfo> ProvideProxy()
{
Dictionary <string, ProxyInfo> proxyInfos = new Dictionary <string, ProxyInfo>();
foreach (string src in Sources)
{
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(src);
HtmlHelper.HttpWebRequestNormalSetup(request);
request.CookieContainer = cc;
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
string html;
if (HtmlHelper.GetHtml(response, out html))
{
MatchCollection mc = matchRegex.Matches(html);
foreach (Match m in mc)
{
string ip = m.Groups["ip"].Value;
ip = System.Web.HttpUtility.UrlDecode(ip);
Match mm = ipRegex.Match(ip);
if (mm.Success)
{
ip = mm.Captures[0].Value;
}
//<a href="http://www.freeproxylists.net/zh/118.123.248.103.html">118.123.248.103</a>
string port = m.Groups["port"].Value;
int porti;
if (Int32.TryParse(port, out porti))
{
if (portWhiteList.Count == 0 ||
portWhiteList.Contains(porti))
{
proxyInfos[ip] = new ProxyInfo
{
HttpProxy = new WebProxy(ip, Int32.Parse(port)),
Location = IPLocationSearch.GetIPLocation(ip).country
};
}
}
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
//return proxyInfos.GetRange(0,30);
List <ProxyInfo> proxies = new List <ProxyInfo>(proxyInfos.Values);
int len = proxies.Count;
Random r = new Random();
while (len > MAX_PROVIDE)
{
int random = r.Next(len);
proxies.RemoveAt(random);
len--;
}
return(new HashSet <ProxyInfo>(proxies));
}
