Пример #1
0
            private static async Task <HttpResponseMessage> GetJobListAsync(SpiderHttpClient client, string encodedCityName, string encodedKeyword, string type = "new")
            {
                Uri uri     = PcWebApiProvider.GetJobListUri(encodedCityName, encodedKeyword, type);
                Uri referer = PcWebApiProvider.GetJobListReferer(encodedKeyword);

                using var reqMsg = new HttpRequestMessage(HttpMethod.Get, uri)
                      {
                          Version = HttpVersion.Version11
                      };
                var headers = reqMsg.Headers;

                headers.Referrer = referer;
                headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3");
                headers.Add("Pragma", "no-cache");
                headers.Add("Cache-Control", "no-cache");
                headers.Add("Upgrade-Insecure-Requests", "1");
                try
                {
                    var rspMsg = await client.SendAsync(reqMsg, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);

                    if (rspMsg.IsSuccessStatusCode)
                    {
                        FixCookies(client.CookieContainer, uri);
                    }
                    return(rspMsg);
                }
                catch
                {
                    return(null);
                }
            }
Пример #2
0
        protected async Task <List <SpiderProxyUriEntity> > GetProxyEntitiesAsync(SpiderHttpClient webClient, HttpMethod httpMethod, string url)
        {
            List <SpiderProxyUriEntity> entities = null;

            using (var reqMsg = new HttpRequestMessage(httpMethod, url))
            {
                HttpResponseMessage rspMsg;
                try
                {
                    rspMsg = await webClient.SendAsync(reqMsg);
                }
                catch
                {
                    return(new List <SpiderProxyUriEntity>(1));
                }
                if (rspMsg != null)
                {
                    using (rspMsg)
                    {
                        if (rspMsg.IsSuccessStatusCode)
                        {
                            var reader = await rspMsg.Content.ToHtmlReaderAsync();

                            entities = ApiProvider.GetProxyEntities(reader);
                        }
                    };
                }
            }
            return(entities ?? new List <SpiderProxyUriEntity>(1));
        }