public static RobotsTextResponse GetRobotsResponse(Uri uri, int retries = 0, string cacheKey = null)
{
if (cacheKey == null)
{
//in case of redirects, keep same key
cacheKey = uri.Authority;
}
if (retries == RunSettings.MaxRetries)
{
ApplicationCache.Robots.Add(cacheKey, null);
return(null);
}
retries++;
var robots = new Uri($"{uri.Scheme}://{uri.Authority}/robots.txt");
var page = Download(robots);
if (page == null)
{
ApplicationCache.Robots.Add(cacheKey, null);
return(null);
}
if (page.StatusCode != HttpStatusCode.OK && page.StatusCode != HttpStatusCode.NotModified)
{
if ((page.StatusCode == HttpStatusCode.Moved || page.StatusCode == HttpStatusCode.Redirect &&
page.StatusCode == HttpStatusCode.TemporaryRedirect) && page.Headers.ContainsKey("Location"))
{
return(GetRobotsResponse(new Uri(page.Headers["Location"]), retries, cacheKey));
}
ApplicationCache.Robots.Add(cacheKey, null);
return(null);
}
var parsedRobots = RobotsTextResponse.ParseText(page.Html);
ApplicationCache.Robots.Add(cacheKey, parsedRobots);
return(parsedRobots);
}
