public RobotsBuilder DisallowWithComment(string url, string comment)
{
var disallowEntry = new DisallowEntry {
Url = url, Comment = comment
};
_lastUserAgent.AddEntry(disallowEntry);
return(this);
}
public RobotsBuilder DisallowWithComment(Uri uri, string comment)
{
var disallowEntry = new DisallowEntry {
Url = uri.PathAndQuery, Comment = comment
};
_lastUserAgent.AddEntry(disallowEntry);
return(this);
}
private static bool CheckDisallowedEntry(DisallowEntry entry, string[] uriParts, out bool allow)
{
allow = true;
string[] robotInstructionUriParts = entry.Url.PathAndQuery.Split(new[] { '/', '?' }, StringSplitOptions.RemoveEmptyEntries);
if (robotInstructionUriParts.Length > uriParts.Length)
{
return(false);
}
bool lastPartIsComplete = entry.Url.PathAndQuery.EndsWith("/");
int end = Math.Min(robotInstructionUriParts.Length, uriParts.Length);
bool mismatch = false;
for (int i = 0; i < end; i++)
{
int index1 = i, index2 = i;
if (entry.Inverted)
{
index1 = robotInstructionUriParts.Length - i - 1;
index2 = uriParts.Length - i - 1;
}
bool partIsComplete = i < end - 1 || lastPartIsComplete;
if (IsMismatch(robotInstructionUriParts[index1], uriParts[index2], partIsComplete))
{
mismatch = true;
break;
}
}
if (!mismatch)
{
allow = false;
return(true);
}
return(false);
}
public void EntryType_Test()
{
var target = new DisallowEntry();
Assert.AreEqual(EntryType.Disallow, target.Type);
}
private static bool CheckDisallowedEntry(DisallowEntry entry, Uri uri)
{
return(entry.Regex.IsMatch(uri.PathAndQuery));
}