public void MultiThreadProxy()
{
IHttpClientPool pool = new HttpClientPool();
var spider = new DefaultSpider();
var downloader = new HttpClientDownloader();
ConcurrentDictionary <HttpClientEntry, int> tonggi = new ConcurrentDictionary <HttpClientEntry, int>();
Parallel.For(0, 1000, new ParallelOptions {
MaxDegreeOfParallelism = 1
}, (i) =>
{
var port = i % 10;
var proxy = new UseSpecifiedUriWebProxy(new Uri($"http://192.168.10.1:{port}"));
var item = pool.GetHttpClient(proxy.Hash);
if (tonggi.ContainsKey(item))
{
tonggi[item] = tonggi[item] + 1;
}
else
{
tonggi.TryAdd(item, 1);
}
});
Assert.Equal(10, tonggi.Count);
foreach (var pair in tonggi)
{
Assert.Equal(100, pair.Value);
}
}
public HttpClient GetHttpClient(UseSpecifiedUriWebProxy proxy)
{
if (proxy == null)
{
return(_noProxyHttpClient);
}
var key = proxy.GetHashCode();
if (key == -1)
{
return(_noProxyHttpClient);
}
if (_pool.Count % 100 == 0)
{
ClearHttpClient();
}
if (_pool.ContainsKey(key))
{
return(_pool[key].Client);
}
else
{
var client = new HttpClient(new GlobalRedirectHandler(new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip,
UseProxy = true,
UseCookies = false,
Proxy = proxy
}));
_pool.Add(key, new HttpClientObj
{
Client = client,
Start = DateTime.Now
});
return(client);
}
}
public void ReturnHttpProxy(UseSpecifiedUriWebProxy proxy, HttpStatusCode statusCode)
{
HttpProxyPool?.ReturnProxy(proxy, statusCode);
}