public RobotifyOptionsConfigurer Add(string userAgent, StringValues disallowPaths, StringValues allowPaths)
{
var existingGroup = options.Groups.FirstOrDefault(x => x.UserAgent.Equals(userAgent, StringComparison.CurrentCultureIgnoreCase));
if (existingGroup == null)
{
var group = new RobotsGroup {
UserAgent = userAgent, Disallow = disallowPaths, Allow = allowPaths
};
options.Add(group);
}
else
{
existingGroup.MergePaths(disallowPaths, allowPaths);
}
return(this);
}
public RobotifyOptionsConfigurer DenyAll(string userAgent = "*")
{
var denyAll = new RobotsGroup {
UserAgent = userAgent, Disallow = new [] { "/" }
};
var matchingGroups = options.Groups.Where(s => s.UserAgent.Equals(userAgent));
foreach (var group in matchingGroups)
{
options.Groups.Remove(group);
}
if (!options.Groups.Contains(denyAll))
{
options.Add(denyAll);
}
return(this);
}
public RobotifyOptions Add(RobotsGroup group)
{
Groups.Add(group);
return(this);
}
protected virtual string GetRobotsGroup(RobotsGroup group)
{
return(group.ToString());
}
