public RobotifyOptionsConfigurer Add(string userAgent, StringValues disallowPaths, StringValues allowPaths) { var existingGroup = options.Groups.FirstOrDefault(x => x.UserAgent.Equals(userAgent, StringComparison.CurrentCultureIgnoreCase)); if (existingGroup == null) { var group = new RobotsGroup { UserAgent = userAgent, Disallow = disallowPaths, Allow = allowPaths }; options.Add(group); } else { existingGroup.MergePaths(disallowPaths, allowPaths); } return(this); }
public RobotifyOptionsConfigurer DenyAll(string userAgent = "*") { var denyAll = new RobotsGroup { UserAgent = userAgent, Disallow = new [] { "/" } }; var matchingGroups = options.Groups.Where(s => s.UserAgent.Equals(userAgent)); foreach (var group in matchingGroups) { options.Groups.Remove(group); } if (!options.Groups.Contains(denyAll)) { options.Add(denyAll); } return(this); }
public RobotifyOptions Add(RobotsGroup group) { Groups.Add(group); return(this); }
protected virtual string GetRobotsGroup(RobotsGroup group) { return(group.ToString()); }