public async Task <ContentResult> Robot()
{
logger.LogInformation("Generating Robots.txt");
var robot = new Robot();
await AppendShellRobot(robot).ConfigureAwait(false);
// get all the registered application robots.txt
var applicationRobotModels = await GetApplicationRobotsAsync().ConfigureAwait(false);
AppendApplicationsRobots(robot, applicationRobotModels);
// add the Shell sitemap route to the bottom
var sitemapRouteUrl = Url.RouteUrl("Sitemap", null);
if (sitemapRouteUrl != null)
{
robot.Add($"Sitemap: {Request.Scheme}://{Request.Host}{sitemapRouteUrl}");
}
logger.LogInformation("Generated Robots.txt");
return(Content(robot.Data, MediaTypeNames.Text.Plain));
}
private static void AppendApplicationRobotData(ApplicationRobotModel applicationRobotModel, string applicationRobotsText, string baseUrl, Robot robot)
{
var robotsLines = applicationRobotsText.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries);
var robotResults = ProcessRobotsLines(applicationRobotModel, baseUrl, robotsLines);
foreach (var robotResult in robotResults)
{
if (!robot.Lines.Contains(robotResult))
{
robot.Add(robotResult);
}
}
}
private Robot GenerateThisSiteRobot()
{
var robot = new Robot();
string robotsFilePath = System.IO.Path.Combine(hostingEnvironment.WebRootPath, "StaticRobots.txt");
if (System.IO.File.Exists(robotsFilePath))
{
// output the composite UI default (static) robots data from the StaticRobots.txt file
string staticRobotsText = System.IO.File.ReadAllText(robotsFilePath);
if (!string.IsNullOrWhiteSpace(staticRobotsText))
{
robot.Add(staticRobotsText);
}
}
return(robot);
}
private Robot GenerateThisSiteRobot()
{
var robot = new Robot();
string robotsFilePath = System.IO.Path.Combine(_hostingEnvironment.WebRootPath, "StaticRobots.txt");
if (System.IO.File.Exists(robotsFilePath))
{
// output the composite UI default (static) robots data from the StaticRobots.txt file
string staticRobotsText = System.IO.File.ReadAllText(robotsFilePath);
if (!string.IsNullOrEmpty(staticRobotsText))
{
robot.Add(staticRobotsText);
}
}
// add any dynamic robots data form the Shell app
//robot.Add("<<add any dynamic text or other here>>");
return(robot);
}
public void BuildArms()
{
_robot.Add("Sorry, animal robots do not have arms");
}
public void BuildArms()
{
_robot.Add("Adding 2 human robot arms");
}
