public async Task <RobotsFile> FromUriAsync(Uri robotsUri, CancellationToken cancellationToken = default)
{
var baseUri = new Uri(robotsUri.GetLeftPart(UriPartial.Authority));
robotsUri = new UriBuilder(robotsUri)
{
Path = "/robots.txt"
}.Uri;
using (var response = await HttpClient.GetAsync(robotsUri, cancellationToken))
{
if (response.StatusCode == HttpStatusCode.NotFound)
{
return(RobotsFile.AllowAllRobots(baseUri));
}
else if ((int)response.StatusCode >= 200 && (int)response.StatusCode < 300)
{
using (var stream = await response.Content.ReadAsStreamAsync())
{
return(await FromStreamAsync(stream, baseUri, cancellationToken));
}
}
}
return(RobotsFile.AllowAllRobots(baseUri));
}
public async Task <RobotsFile> FromUriAsync(Uri robotsUri, CancellationToken cancellationToken = default)
{
var baseUri = new Uri(robotsUri.GetLeftPart(UriPartial.Authority));
robotsUri = new UriBuilder(robotsUri)
{
Path = "/robots.txt"
}.Uri;
using (var response = await HttpClient.GetAsync(robotsUri, cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested(); // '.NET Framework' and '.NET Core 2.1' workaround
if (response.StatusCode == HttpStatusCode.NotFound)
{
return(RobotsFile.AllowAllRobots(baseUri));
}
else if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
{
return(RobotsFile.DenyAllRobots(baseUri));
}
else if ((int)response.StatusCode >= 200 && (int)response.StatusCode < 300)
{
using (var stream = await response.Content.ReadAsStreamAsync())
{
cancellationToken.ThrowIfCancellationRequested();
return(await FromStreamAsync(stream, baseUri, cancellationToken));
}
}
}
return(RobotsFile.AllowAllRobots(baseUri));
}
private RobotsFile FromTokens(IEnumerable <Token> tokens, Uri baseUri)
{
var validationResult = PatternValidator.Validate(tokens);
if (validationResult.IsValid)
{
return(new RobotsFile(baseUri)
{
SiteAccessEntries = TokenParser.GetSiteAccessEntries(tokens),
SitemapEntries = TokenParser.GetSitemapUrlEntries(tokens)
});
}
return(RobotsFile.AllowAllRobots(baseUri));
}