protected async Task <List <SpiderProxyUriEntity> > GetProxyEntitiesAsync(SpiderHttpClient webClient, HttpMethod httpMethod, IList <string> urls, int estimatedCount = 0)
{
var entities = estimatedCount > 0 ? new List <SpiderProxyUriEntity>(estimatedCount) : new List <SpiderProxyUriEntity>();
int urlCount = urls.Count;
Task[] tasks = new Task[urlCount];
for (int i = 0; i < urlCount; i++)
{
tasks[i] = Task.Run(async() =>
{
var tempList = await GetProxyEntitiesAsync(webClient, httpMethod, urls[i]);
if (!tempList.IsNullOrEmpty())
{
lock (entities)
{
entities.AddRange(tempList);
}
}
});
await Task.Delay(RandomTool.NextIntSafely(4000, 6000));
}
await Task.WhenAll(tasks);
return(entities);
}
private static async Task <HttpResponseMessage> GetJobListAsync(SpiderHttpClient client, string encodedCityName, string encodedKeyword, string type = "new")
{
Uri uri = PcWebApiProvider.GetJobListUri(encodedCityName, encodedKeyword, type);
Uri referer = PcWebApiProvider.GetJobListReferer(encodedKeyword);
using var reqMsg = new HttpRequestMessage(HttpMethod.Get, uri)
{
Version = HttpVersion.Version11
};
var headers = reqMsg.Headers;
headers.Referrer = referer;
headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3");
headers.Add("Pragma", "no-cache");
headers.Add("Cache-Control", "no-cache");
headers.Add("Upgrade-Insecure-Requests", "1");
try
{
var rspMsg = await client.SendAsync(reqMsg, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
if (rspMsg.IsSuccessStatusCode)
{
FixCookies(client.CookieContainer, uri);
}
return(rspMsg);
}
catch
{
return(null);
}
}
protected async Task <List <SpiderProxyUriEntity> > GetProxyEntitiesAsync(SpiderHttpClient webClient, HttpMethod httpMethod, string url)
{
List <SpiderProxyUriEntity> entities = null;
using (var reqMsg = new HttpRequestMessage(httpMethod, url))
{
HttpResponseMessage rspMsg;
try
{
rspMsg = await webClient.SendAsync(reqMsg);
}
catch
{
return(new List <SpiderProxyUriEntity>(1));
}
if (rspMsg != null)
{
using (rspMsg)
{
if (rspMsg.IsSuccessStatusCode)
{
var reader = await rspMsg.Content.ToHtmlReaderAsync();
entities = ApiProvider.GetProxyEntities(reader);
}
};
}
}
return(entities ?? new List <SpiderProxyUriEntity>(1));
}
private async Task <bool> TryInitCookiesAsync(SpiderHttpClient client, string encodedCityName, string encodedKeyword, string type = "new")
{
var jobListRspMsg = await GetJobListAsync(client, encodedCityName, encodedKeyword, type).ConfigureAwait(false);
if (jobListRspMsg != null)
{
jobListRspMsg.Dispose();
return(true);
}
return(false);
}
public override SpiderHttpClient CreateWebClient()
{
SpiderHttpClient client = new SpiderHttpClient();
client.DefaultRequestHeaders.Host = HomePageHost;
client.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
client.DefaultRequestHeaders.Add("Accept-Encoding", "br");
client.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.9");
client.DefaultRequestHeaders.Add("Upgrade-Insecure-Requests", "1");
client.DefaultRequestHeaders.Add("User-Agent", HttpConsole.DefaultPcUserAgent);
return(client);
}
public static SpiderHttpClient CreateWebClient(IWebProxy proxy)
{
SpiderHttpClient client = new SpiderHttpClient(proxy);
client.DefaultRequestHeaders.Host = HomePageHost;
client.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
client.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.9");
client.DefaultRequestHeaders.Add("Cache-Control", "no-cache");
client.DefaultRequestHeaders.Add("Pragma", "no-cache");
client.DefaultRequestHeaders.Add("Upgrade-Insecure-Requests", "1");
client.DefaultRequestHeaders.Add("User-Agent", HttpConsole.DefaultPcUserAgent);
return(client);
}
public const string WgUrl = "http://www.iphai.com/free/wg"; //国外高匿
public override SpiderHttpClient CreateWebClient()
{
SpiderHttpClient client = new SpiderHttpClient();
client.DefaultRequestHeaders.Host = HomePageHost;
client.DefaultRequestHeaders.Referrer = new Uri(NgUrl);
client.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
client.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.9");
client.DefaultRequestHeaders.Add("Accept-Charset", "utf-8");
client.DefaultRequestHeaders.Add("DNT", "1");
client.DefaultRequestHeaders.Add("User-Agent", HttpConsole.DefaultPcUserAgent);
return(client);
}
private static SpiderHttpClient CreateWebClient(IWebProxy proxy)
{
var client = new SpiderHttpClient(proxy);
client.DefaultRequestHeaders.Add("Accept", "application/json, text/javascript, */*; q=0.01");
client.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.9");
client.DefaultRequestHeaders.Add("User-Agent", HttpConsole.DefaultPcUserAgent);
client.DefaultRequestHeaders.Add("Origin", "http://deal.ggzy.gov.cn");
client.DefaultRequestHeaders.Referrer = PcWebApiProvider.ReferUri;
client.DefaultRequestHeaders.Host = "deal.ggzy.gov.cn";
client.DefaultRequestHeaders.Add("Cache-Control", "no-cache");
client.DefaultRequestHeaders.Add("Pragma", "no-cache");
client.DefaultRequestHeaders.Add("X-Requested-With", "XMLHttpRequest");
return(client);
}
private static SpiderHttpClient CreateWebClient(IWebProxy proxy)
{
var client = new SpiderHttpClient(proxy);
client.DefaultRequestHeaders.Add("Accept", "application/json, text/plain, */*");
client.DefaultRequestHeaders.Add("Accept-Encoding", "br");
client.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.9");
client.DefaultRequestHeaders.Add("User-Agent", HttpConsole.DefaultPcUserAgent);
client.DefaultRequestHeaders.Add("Origin", "https://live.bilibili.com");
client.DefaultRequestHeaders.Referrer = PcWebApiProvider.RefererUri;
client.DefaultRequestHeaders.Host = "api.live.bilibili.com";
client.DefaultRequestHeaders.Add("Cache-Control", "no-cache");
client.DefaultRequestHeaders.Add("Pragma", "no-cache");
return(client);
}
public override async Task RunAsync()
{
string caseName = ClassName;
var pa = ProxyAgent <SqlServerProxyDbContext> .CreateInstance("SqlServerTest", true, c => new SqlServerProxyDbContext(c));
var urls = ApiProvider.GetRequestUrls();
using (SpiderHttpClient webClient = ApiProvider.CreateWebClient())
{
var entities = await GetProxyEntitiesAsync(webClient, HttpMethod.Get, urls, urls.Count * 10);
if (entities.Count < 1)
{
return;
}
entities.ForEach(e => e.Source = caseName);
ShowLogInfo("CollectCount: " + entities.Count.ToString());
int insertCount = pa.InsertProxyEntities(entities);
ShowLogInfo("InsertCount: " + insertCount.ToString());
}
}
private SpiderHttpClient CreateHttpClient()
{
var handler = new SocketsHttpHandler()
{
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
UseCookies = true,
UseProxy = UseProxy
};
if (UseProxy)
{
handler.Proxy = HttpConsole.LocalWebProxy;
}
SpiderHttpClient client = new SpiderHttpClient(handler);
var headers = client.DefaultRequestHeaders;
headers.Host = PcWebApiProvider.HomePageUri.Host;
headers.Add("Accept-Encoding", "br");
headers.Add("Accept-Language", "zh-CN,zh;q=0.9");
headers.Add("User-Agent", HttpConsole.DefaultPcUserAgent);
return(client);
}
