public static void UseRobotsTxt(this IApplicationBuilder app, Func <RobotsTxtOptionsBuilder, RobotsTxtOptionsBuilder> builderFunc)
{
var builder = new RobotsTxtOptionsBuilder();
var options = builderFunc(builder).Build();
app.UseRobotsTxt(options);
}
public Benchmarks()
{
_optionsBuilder = GetNewOptionsBuilder()
.AddSection(section =>
section
.AddComment("Allow Googlebot")
.AddUserAgent("Googlebot")
.Allow("/")
)
.AddSection(section =>
section
.AddComment("Allow Bing for most stuff")
.AddUserAgent("Bing")
.Disallow("/bing-should-not-see-this")
.Allow("/")
)
.AddSection(section =>
section
.AddComment("Disallow the rest")
.AddUserAgent("*")
.AddCrawlDelay(TimeSpan.FromSeconds(10))
.Disallow("/")
)
.AddSitemap("https://example.com/sitemap.xml");
_options = _optionsBuilder.Build();
#if V1
_middleware = new RobotsTxtMiddleware(RequestDelegateAsync, _options);
#endif
#if V2
_middleware = new RobotsTxtMiddleware(RequestDelegateAsync);
_provider = new StaticRobotsTxtProvider(_options);
#endif
}
public void ShouldClearPreviousSectionsWhenCallingDenyAll()
{
var builder = new RobotsTxtOptionsBuilder();
builder
.AddSection(section => section.AddUserAgent("Googlebot"))
.AddSection(section => section.AddUserAgent("Bingbot"))
.DenyAll();
var options = builder.Build();
options.Sections.Count.ShouldBe(1);
}
public Task <RobotsTxtResult> GetResultAsync(CancellationToken cancellationToken)
{
var settings = await _context.Settings.FirstAsync();
var builder = new RobotsTxtOptionsBuilder();
string content;
if (settings.RobotsTxt.AllowAll)
{
content = builder.AllowAll().Build().ToString();
}
else
{
content = builder.DenyAll().Build().ToString();
}
var buffer = Encoding.UTF8.GetBytes(content).AsMemory();
return(new RobotsTxtResult(buffer, settings.RobotsTxt.MaxAge));
}
public void ShouldNotThrowOnValidSitemapUrl()
{
var builder = new RobotsTxtOptionsBuilder();
Should.NotThrow(() => builder.AddSitemap("https://example.com/sitemap.xml"));
}
public void ShouldThrowOnInvalidSitemapUrl(string url)
{
var builder = new RobotsTxtOptionsBuilder();
Should.Throw <ArgumentException>(() => builder.AddSitemap(url));
}
