public async Task RobotsTxtExample()
{
var loader = new SitemapLoader();
Sitemap robotSitemap = await loader.LoadFromRobotsTxtAsync(new Uri("https://www.google.com"));
Assert.Equal(SitemapType.RobotsTxt, robotSitemap.SitemapType);
Assert.NotEmpty(robotSitemap.Sitemaps); //We expect at least some Sitemaps to be in list
Assert.Empty(robotSitemap.Items); //Robots.txt can only link to Sitemaps (Not content items)
Sitemap firstSitemap = robotSitemap.Sitemaps.First();
Assert.False(firstSitemap.IsLoaded); //We only have sitemap location. Contents not yet loaded nor parsed
var firstLoadedSitemap = await loader.LoadAsync(firstSitemap);
Assert.True(firstLoadedSitemap.IsLoaded); //Now items are loaded!
//We have to check type as we can either have links to other sitemaps (i.e. index sitemaps)
//-or- links to actual sitemap items (i.e. links to content)
switch (firstLoadedSitemap.SitemapType)
{
case SitemapType.Index: Assert.NotEmpty(firstLoadedSitemap.Sitemaps); break;
case SitemapType.Items: Assert.NotEmpty(firstLoadedSitemap.Items); break;
default: throw new NotSupportedException($"SitemapType {firstLoadedSitemap.SitemapType} not expected here");
}
}
public RobotsSitemapLoader(ISitemapFetcher fetcher = null, ISitemapParser sitemapParser = null, IRobotsTxtParser robotsParser = null)
{
_adapteeObject = new SitemapLoader(fetcher, sitemapParser, robotsParser);
}
public LoewTest(string url, ILinkStorage linkStorage)
{
_linkStorage = linkStorage ?? new ConsoleLinkStorage();
_url = url;
_loader = new SitemapLoader();
}
