public Benchmarks()
{
_optionsBuilder = GetNewOptionsBuilder()
.AddSection(section =>
section
.AddComment("Allow Googlebot")
.AddUserAgent("Googlebot")
.Allow("/")
)
.AddSection(section =>
section
.AddComment("Allow Bing for most stuff")
.AddUserAgent("Bing")
.Disallow("/bing-should-not-see-this")
.Allow("/")
)
.AddSection(section =>
section
.AddComment("Disallow the rest")
.AddUserAgent("*")
.AddCrawlDelay(TimeSpan.FromSeconds(10))
.Disallow("/")
)
.AddSitemap("https://example.com/sitemap.xml");
_options = _optionsBuilder.Build();
#if V1
_middleware = new RobotsTxtMiddleware(RequestDelegateAsync, _options);
#endif
#if V2
_middleware = new RobotsTxtMiddleware(RequestDelegateAsync);
_provider = new StaticRobotsTxtProvider(_options);
#endif
}
public void ShouldClearPreviousSectionsWhenCallingDenyAll()
{
var builder = new RobotsTxtOptionsBuilder();
builder
.AddSection(section => section.AddUserAgent("Googlebot"))
.AddSection(section => section.AddUserAgent("Bingbot"))
.DenyAll();
var options = builder.Build();
options.Sections.Count.ShouldBe(1);
}