private RobotsFile FromTokens(IEnumerable <Token> tokens, Uri baseUri) { var validationResult = PatternValidator.Validate(tokens); if (validationResult.IsValid) { return(new RobotsFile(baseUri) { SiteAccessEntries = TokenParser.GetSiteAccessEntries(tokens), SitemapEntries = TokenParser.GetSitemapUrlEntries(tokens) }); } return(RobotsFile.AllowAllRobots(baseUri)); }