/// <summary>
/// IGuard implementation
/// see: https://developers.google.com/search/reference/robots_txt
/// </summary>
/// <param name="alias"></param>
/// <param name="path"></param>
/// <param name="userAgents"></param>
/// <returns></returns>
public IDecision Allow(string alias, string path, ref IRobotsTxt model)
{
// allow is undefined by default
bool?allowed = null;
// return undefined when no user-agents are present
if (model?.Useragents == null || !model.Useragents.Any() || string.IsNullOrEmpty(path))
{
return(new Decision(alias, path, "RobotsTxt or Useragent is null", true));
}
// set alias to all if empty
if (string.IsNullOrEmpty(alias))
{
alias = "*";
}
// find useragent
var userAgent = model.Useragents
.FirstOrDefault(ua => ua.Alias.Equals(alias, StringComparison.InvariantCultureIgnoreCase));
// if no user agent then return
if (userAgent == null)
{
return(new Decision(alias, path, $"Useragent {userAgent.Alias} not found", true));
}
// todo: disallow anything if no diallow directives are present but a user agent is
// check if any disallow patterns match
var match = userAgent.Disallow
.FirstOrDefault(disallow => Regex.IsMatch(path, MakePattern(disallow)));
// if match is not empty then disallow
if (!string.IsNullOrEmpty(match))
{
allowed = false;
}
// return if allowed
if (allowed.HasValue && allowed.Value)
{
return(new Decision(alias, path, string.Empty, true));
}
else if (allowed.HasValue && !allowed.Value)
{
// check if it is allowed by directive
var allowMatch = userAgent.Allow
.FirstOrDefault(allow => Regex.IsMatch(path, MakePattern(allow)));
if (string.IsNullOrEmpty(allowMatch))
{
return(new Decision(alias, path, match, false));
}
else
{
return(new Decision(alias, path, allowMatch, true));
}
}
return(new Decision(alias, path, "No matches", true));
}
/// <summary>
/// Convert to string
/// </summary>
/// <param name="context"></param>
/// <param name="culture"></param>
/// <param name="value"></param>
/// <param name="destinationType"></param>
/// <returns></returns>
public override object ConvertTo(ITypeDescriptorContext context, CultureInfo culture, object value, Type destinationType)
{
IRobotsTxt robotsTxt = (IRobotsTxt)value;
if (destinationType == typeof(string))
{
var sb = new StringBuilder();
if (robotsTxt.Sitemaps != null && robotsTxt.Sitemaps.Any())
{
foreach (var sitemap in robotsTxt.Sitemaps)
{
sb.AppendLine($"sitemap: { sitemap }");
}
sb.AppendLine();
}
foreach (var useragent in robotsTxt.Useragents)
{
sb.AppendLine($"user-agent: { useragent.Alias }");
foreach (var disallow in useragent.Disallow)
{
sb.AppendLine($"disallow: { disallow }");
}
foreach (var allow in useragent.Allow)
{
sb.AppendLine($"allow: { allow }");
}
sb.AppendLine();
}
return(sb.ToString());
}
return(base.ConvertTo(context, culture, value, destinationType));
}
