protected async Task <List <SpiderProxyUriEntity> > GetProxyEntitiesAsync(SpiderHttpClient webClient, HttpMethod httpMethod, IList <string> urls, int estimatedCount = 0) { var entities = estimatedCount > 0 ? new List <SpiderProxyUriEntity>(estimatedCount) : new List <SpiderProxyUriEntity>(); int urlCount = urls.Count; Task[] tasks = new Task[urlCount]; for (int i = 0; i < urlCount; i++) { tasks[i] = Task.Run(async() => { var tempList = await GetProxyEntitiesAsync(webClient, httpMethod, urls[i]); if (!tempList.IsNullOrEmpty()) { lock (entities) { entities.AddRange(tempList); } } }); await Task.Delay(RandomTool.NextIntSafely(4000, 6000)); } await Task.WhenAll(tasks); return(entities); }
private static async Task <HttpResponseMessage> GetJobListAsync(SpiderHttpClient client, string encodedCityName, string encodedKeyword, string type = "new") { Uri uri = PcWebApiProvider.GetJobListUri(encodedCityName, encodedKeyword, type); Uri referer = PcWebApiProvider.GetJobListReferer(encodedKeyword); using var reqMsg = new HttpRequestMessage(HttpMethod.Get, uri) { Version = HttpVersion.Version11 }; var headers = reqMsg.Headers; headers.Referrer = referer; headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"); headers.Add("Pragma", "no-cache"); headers.Add("Cache-Control", "no-cache"); headers.Add("Upgrade-Insecure-Requests", "1"); try { var rspMsg = await client.SendAsync(reqMsg, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); if (rspMsg.IsSuccessStatusCode) { FixCookies(client.CookieContainer, uri); } return(rspMsg); } catch { return(null); } }
protected async Task <List <SpiderProxyUriEntity> > GetProxyEntitiesAsync(SpiderHttpClient webClient, HttpMethod httpMethod, string url) { List <SpiderProxyUriEntity> entities = null; using (var reqMsg = new HttpRequestMessage(httpMethod, url)) { HttpResponseMessage rspMsg; try { rspMsg = await webClient.SendAsync(reqMsg); } catch { return(new List <SpiderProxyUriEntity>(1)); } if (rspMsg != null) { using (rspMsg) { if (rspMsg.IsSuccessStatusCode) { var reader = await rspMsg.Content.ToHtmlReaderAsync(); entities = ApiProvider.GetProxyEntities(reader); } }; } } return(entities ?? new List <SpiderProxyUriEntity>(1)); }
private async Task <bool> TryInitCookiesAsync(SpiderHttpClient client, string encodedCityName, string encodedKeyword, string type = "new") { var jobListRspMsg = await GetJobListAsync(client, encodedCityName, encodedKeyword, type).ConfigureAwait(false); if (jobListRspMsg != null) { jobListRspMsg.Dispose(); return(true); } return(false); }
public override SpiderHttpClient CreateWebClient() { SpiderHttpClient client = new SpiderHttpClient(); client.DefaultRequestHeaders.Host = HomePageHost; client.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"); client.DefaultRequestHeaders.Add("Accept-Encoding", "br"); client.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.9"); client.DefaultRequestHeaders.Add("Upgrade-Insecure-Requests", "1"); client.DefaultRequestHeaders.Add("User-Agent", HttpConsole.DefaultPcUserAgent); return(client); }
public static SpiderHttpClient CreateWebClient(IWebProxy proxy) { SpiderHttpClient client = new SpiderHttpClient(proxy); client.DefaultRequestHeaders.Host = HomePageHost; client.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"); client.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.9"); client.DefaultRequestHeaders.Add("Cache-Control", "no-cache"); client.DefaultRequestHeaders.Add("Pragma", "no-cache"); client.DefaultRequestHeaders.Add("Upgrade-Insecure-Requests", "1"); client.DefaultRequestHeaders.Add("User-Agent", HttpConsole.DefaultPcUserAgent); return(client); }
public const string WgUrl = "http://www.iphai.com/free/wg"; //国外高匿 public override SpiderHttpClient CreateWebClient() { SpiderHttpClient client = new SpiderHttpClient(); client.DefaultRequestHeaders.Host = HomePageHost; client.DefaultRequestHeaders.Referrer = new Uri(NgUrl); client.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"); client.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.9"); client.DefaultRequestHeaders.Add("Accept-Charset", "utf-8"); client.DefaultRequestHeaders.Add("DNT", "1"); client.DefaultRequestHeaders.Add("User-Agent", HttpConsole.DefaultPcUserAgent); return(client); }
private static SpiderHttpClient CreateWebClient(IWebProxy proxy) { var client = new SpiderHttpClient(proxy); client.DefaultRequestHeaders.Add("Accept", "application/json, text/javascript, */*; q=0.01"); client.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.9"); client.DefaultRequestHeaders.Add("User-Agent", HttpConsole.DefaultPcUserAgent); client.DefaultRequestHeaders.Add("Origin", "http://deal.ggzy.gov.cn"); client.DefaultRequestHeaders.Referrer = PcWebApiProvider.ReferUri; client.DefaultRequestHeaders.Host = "deal.ggzy.gov.cn"; client.DefaultRequestHeaders.Add("Cache-Control", "no-cache"); client.DefaultRequestHeaders.Add("Pragma", "no-cache"); client.DefaultRequestHeaders.Add("X-Requested-With", "XMLHttpRequest"); return(client); }
private static SpiderHttpClient CreateWebClient(IWebProxy proxy) { var client = new SpiderHttpClient(proxy); client.DefaultRequestHeaders.Add("Accept", "application/json, text/plain, */*"); client.DefaultRequestHeaders.Add("Accept-Encoding", "br"); client.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.9"); client.DefaultRequestHeaders.Add("User-Agent", HttpConsole.DefaultPcUserAgent); client.DefaultRequestHeaders.Add("Origin", "https://live.bilibili.com"); client.DefaultRequestHeaders.Referrer = PcWebApiProvider.RefererUri; client.DefaultRequestHeaders.Host = "api.live.bilibili.com"; client.DefaultRequestHeaders.Add("Cache-Control", "no-cache"); client.DefaultRequestHeaders.Add("Pragma", "no-cache"); return(client); }
public override async Task RunAsync() { string caseName = ClassName; var pa = ProxyAgent <SqlServerProxyDbContext> .CreateInstance("SqlServerTest", true, c => new SqlServerProxyDbContext(c)); var urls = ApiProvider.GetRequestUrls(); using (SpiderHttpClient webClient = ApiProvider.CreateWebClient()) { var entities = await GetProxyEntitiesAsync(webClient, HttpMethod.Get, urls, urls.Count * 10); if (entities.Count < 1) { return; } entities.ForEach(e => e.Source = caseName); ShowLogInfo("CollectCount: " + entities.Count.ToString()); int insertCount = pa.InsertProxyEntities(entities); ShowLogInfo("InsertCount: " + insertCount.ToString()); } }
private SpiderHttpClient CreateHttpClient() { var handler = new SocketsHttpHandler() { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, UseCookies = true, UseProxy = UseProxy }; if (UseProxy) { handler.Proxy = HttpConsole.LocalWebProxy; } SpiderHttpClient client = new SpiderHttpClient(handler); var headers = client.DefaultRequestHeaders; headers.Host = PcWebApiProvider.HomePageUri.Host; headers.Add("Accept-Encoding", "br"); headers.Add("Accept-Language", "zh-CN,zh;q=0.9"); headers.Add("User-Agent", HttpConsole.DefaultPcUserAgent); return(client); }