public Benchmarks()
{
_optionsBuilder = GetNewOptionsBuilder()
.AddSection(section =>
section
.AddComment("Allow Googlebot")
.AddUserAgent("Googlebot")
.Allow("/")
)
.AddSection(section =>
section
.AddComment("Allow Bing for most stuff")
.AddUserAgent("Bing")
.Disallow("/bing-should-not-see-this")
.Allow("/")
)
.AddSection(section =>
section
.AddComment("Disallow the rest")
.AddUserAgent("*")
.AddCrawlDelay(TimeSpan.FromSeconds(10))
.Disallow("/")
)
.AddSitemap("https://example.com/sitemap.xml");
_options = _optionsBuilder.Build();
#if V1
_middleware = new RobotsTxtMiddleware(RequestDelegateAsync, _options);
#endif
#if V2
_middleware = new RobotsTxtMiddleware(RequestDelegateAsync);
_provider = new StaticRobotsTxtProvider(_options);
#endif
}
public static void UseRobotsTxt(this IApplicationBuilder app, RobotsTxtOptions options)
{
app.UseMiddleware <RobotsTxtMiddleware>(options);
}
