public Feed TryScrapeFeed(ScraperConfig config)
{
CookieContainer cookieJar = new CookieContainer();
var cookies = WebDriver.Manage().Cookies.AllCookies;
foreach (var cookie in cookies)
{
cookieJar.Add(new Cookie(cookie.Name, cookie.Value, cookie.Path, cookie.Domain));
}
int maxRetries = 5;
int retry = 0;
Logger.Trace("Downloading {0}", config.ApiUrl);
Feed feed = null;
while (true)
{
if (retry >= maxRetries)
{
throw new ScraperException("Can't download project feed. Retries exceeded", config);
}
try
{
using (var client = new CookieWebClient(cookieJar))
{
if (!ProxyAddress.IsNullOrEmpty())
{
client.Proxy = new WebProxy(ProxyAddress);
}
var mozilaAgent =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36";
client.Headers.Add("User-Agent", mozilaAgent);
// Call authenticated resources
string result = client.DownloadString(config.ApiUrl);
feed = JsonConvert.DeserializeObject <Feed>(result);
}
break;
}
catch (Exception e)
{
Logger.Trace("Failed to download {0} page. Retry {1} of {2}", config.ApiUrl, retry, maxRetries);
Logger.Error(e);
retry++;
Task.Delay(TimeSpan.FromSeconds(10)).Wait();
}
}
Logger.Info("Successfully downloaded {0}", config.ApiUrl);
return(feed);
}
/// <summary>创建客户端会话</summary>
/// <returns></returns>
public virtual HttpClientX EnsureCreate()
{
var http = _client;
if (http == null)
{
var p = Proxy;
if (p == null && !ProxyAddress.IsNullOrEmpty())
{
Proxy = p = new WebProxy(ProxyAddress);
}
if (p == null)
{
http = new HttpClientX();
}
else
{
http = new HttpClientX(new HttpClientHandler {
Proxy = p
});
}
_client = http;
Request = http.DefaultRequestHeaders;
http.Timeout = new TimeSpan(0, 0, 0, 0, Timeout);
}
var req = http.DefaultRequestHeaders;
if (!UserAgent.IsNullOrEmpty())
{
req.UserAgent.ParseAdd(UserAgent);
}
if (!Accept.IsNullOrEmpty())
{
req.Accept.ParseAdd(Accept);
}
if (!AcceptLanguage.IsNullOrEmpty())
{
req.AcceptLanguage.ParseAdd(AcceptLanguage);
}
if (AutomaticDecompression != DecompressionMethods.None)
{
req.AcceptEncoding.ParseAdd("gzip, deflate");
}
if (!Referer.IsNullOrEmpty())
{
req.Referrer = new Uri(Referer);
}
GetCookie();
return(http);
}
/// <summary>创建客户端会话</summary>
/// <returns></returns>
public virtual HttpClient EnsureCreate()
{
var http = _client;
if (http == null)
{
var p = Proxy;
if (p == null && !ProxyAddress.IsNullOrEmpty())
{
Proxy = p = new WebProxy(ProxyAddress);
}
var handler = new HttpClientHandler();
if (p != null)
{
handler.UseProxy = true;
handler.Proxy = p;
}
else
{
handler.UseProxy = false;
handler.Proxy = null;
}
if (AutomaticDecompression != DecompressionMethods.None)
{
handler.AutomaticDecompression = AutomaticDecompression;
}
http = new HttpClient(handler);
_client = http;
Request = http.DefaultRequestHeaders;
http.Timeout = new TimeSpan(0, 0, 0, 0, Timeout);
}
var req = http.DefaultRequestHeaders;
if (!UserAgent.IsNullOrEmpty())
{
req.UserAgent.ParseAdd(UserAgent);
}
if (!Accept.IsNullOrEmpty())
{
req.Accept.ParseAdd(Accept);
}
if (!AcceptLanguage.IsNullOrEmpty())
{
req.AcceptLanguage.ParseAdd(AcceptLanguage);
}
if (AutomaticDecompression != DecompressionMethods.None)
{
req.AcceptEncoding.ParseAdd("gzip, deflate");
}
if (!Referer.IsNullOrEmpty())
{
req.Referrer = new Uri(Referer);
}
if (KeepAlive)
{
req.Connection.ParseAdd("keep-alive");
}
GetCookie(http);
return(http);
}