public async Task RobotsTxtExample() { var loader = new SitemapLoader(); Sitemap robotSitemap = await loader.LoadFromRobotsTxtAsync(new Uri("https://www.google.com")); Assert.Equal(SitemapType.RobotsTxt, robotSitemap.SitemapType); Assert.NotEmpty(robotSitemap.Sitemaps); //We expect at least some Sitemaps to be in list Assert.Empty(robotSitemap.Items); //Robots.txt can only link to Sitemaps (Not content items) Sitemap firstSitemap = robotSitemap.Sitemaps.First(); Assert.False(firstSitemap.IsLoaded); //We only have sitemap location. Contents not yet loaded nor parsed var firstLoadedSitemap = await loader.LoadAsync(firstSitemap); Assert.True(firstLoadedSitemap.IsLoaded); //Now items are loaded! //We have to check type as we can either have links to other sitemaps (i.e. index sitemaps) //-or- links to actual sitemap items (i.e. links to content) switch (firstLoadedSitemap.SitemapType) { case SitemapType.Index: Assert.NotEmpty(firstLoadedSitemap.Sitemaps); break; case SitemapType.Items: Assert.NotEmpty(firstLoadedSitemap.Items); break; default: throw new NotSupportedException($"SitemapType {firstLoadedSitemap.SitemapType} not expected here"); } }
public RobotsSitemapLoader(ISitemapFetcher fetcher = null, ISitemapParser sitemapParser = null, IRobotsTxtParser robotsParser = null) { _adapteeObject = new SitemapLoader(fetcher, sitemapParser, robotsParser); }
public LoewTest(string url, ILinkStorage linkStorage) { _linkStorage = linkStorage ?? new ConsoleLinkStorage(); _url = url; _loader = new SitemapLoader(); }