public WebResult Get(string url, Dictionary <string, string> headers, string userAgent = null, CookieContainer cc = null)
{
int attempt = 0;
while (true)
{
UpdateCallStats();
attempt++;
trace("Call " + numCalls + " Attempt: " + attempt + " GET: " + url);
WebResult wr = WebUtils.Get(url, headers: headers, cookies: cc, userAgent: userAgent).Result;
LogResult(wr);
if (wr.Code == System.Net.HttpStatusCode.OK)
{
return(wr);
}
if (attempt >= retries)
{
return(wr);
}
if (!IsRateBroken(wr))
{
if (!IsRetryable(wr))
{
return(wr);
}
SleepForRetry();
}
}
}
private static bool IsRetryable(WebResult wr)
{
if (wr.Exc != null)
{
return(true);
}
return(IsRetryable(wr.Code));
}
private void LogError(WebResult wr)
{
if (wr.Exc != null)
{
error(wr.Exc.ToString());
}
else
{
error(wr.Code.ToString());
}
}
private void LogResult(WebResult wr)
{
if (wr.Code == System.Net.HttpStatusCode.OK)
{
trace("OK");
}
else if (wr.Code == System.Net.HttpStatusCode.Redirect)
{
trace("Redirect");
}
else
{
LogError(wr);
}
}
private bool IsRateBroken(WebResult wr)
{
if (brokenRateCodes == null)
{
return(false);
}
int c = (int)wr.Code;
if (Array.IndexOf <int>(brokenRateCodes, c) < 0)
{
return(false);
}
int sec = sleepRateAfterRateBrokenSeconds;
if (sec == 0)
{
sec = throttleSleepSeconds;
}
error("Broken limit response detected going to sleep for seconds:" + sec);
System.Threading.Thread.Sleep(sec * 1000);
return(true);
}