public void HasCommentTest()
{
Entry target = new UserAgentEntry(); // TODO: Initialize to an appropriate value
bool actual = target.HasComment;
Assert.AreEqual(false, actual);
}
public void HasCommentTest2()
{
Entry target = new UserAgentEntry {
Comment = "comment"
};
bool actual = target.HasComment;
Assert.AreEqual(true, actual);
}
public void Add_disallow_entry_Test()
{
var target = new UserAgentEntry();
Entry entry = new CommentEntry();
target.AddEntry(entry);
Assert.NotEmpty(target.Entries);
Assert.Empty(target.AllowEntries);
Assert.Empty(target.DisallowEntries);
}
public void CommentTest()
{
Entry target = new UserAgentEntry(); // TODO: Initialize to an appropriate value
const string expected = "comment";
target.Comment = expected;
string actual = target.Comment;
Assert.AreEqual(expected, actual);
}
public RobotsBuilder ForUserAgent(string userAgent)
{
_lastUserAgent = new UserAgentEntry {
UserAgent = userAgent
};
_robots.AddEntry(_lastUserAgent);
return(this);
}
public void Add_disallow_entry_Test()
{
var target = new UserAgentEntry();
Entry entry = new CommentEntry();
target.AddEntry(entry);
Assert.AreNotEqual(0, target.Entries.Count());
Assert.AreEqual(0, target.AllowEntries.Count());
Assert.AreEqual(0, target.DisallowEntries.Count());
}
private bool CheckExplicitlyAllowed(UserAgentEntry userAgentEntry, Uri uri)
{
//if (!IsInternalToDomain(uri))
// return true;
foreach (var allowEntry in userAgentEntry.AllowEntries)
{
if (CheckAllowedEntry(allowEntry, uri))
{
return(true);
}
}
return(false);
}
private bool CheckExplicitlyAllowed(UserAgentEntry userAgentEntry, Uri uri)
{
//if (!IsInternalToDomain(uri))
// return true;
string[] uriParts = uri.PathAndQuery.Split(new[] { '/', '?' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var allowEntry in userAgentEntry.AllowEntries)
{
bool result;
if (CheckAllowedEntry(allowEntry, uriParts, out result))
{
return(result);
}
}
return(false);
}
private bool CheckExplicitlyAllowed(UserAgentEntry userAgentEntry, Uri uri)
{
//if (!IsInternalToDomain(uri))
// return true;
string path = uri.PathAndQuery;
foreach (var allowEntry in userAgentEntry.AllowEntries)
{
if (Matches(path, allowEntry.Pattern))
{
return(true);
}
}
return(false);
}
private UserAgentEntry FindExplicitUserAgentEntry(string userAgent)
{
UserAgentEntry targetEntry = null;
foreach (var entry in _entries)
{
var userAgentEntry = entry as UserAgentEntry;
if (userAgentEntry == null || userAgentEntry.Type != EntryType.UserAgent)
{
continue;
}
if (string.Compare(userAgentEntry.UserAgent, userAgent, true) == 0)
{
targetEntry = userAgentEntry;
break;
}
}
return(targetEntry);
}
private UserAgentEntry FindUserAgentEntry(string userAgent)
{
UserAgentEntry allAgentsEntry = null;
foreach (var entry in _entries)
{
var userAgentEntry = entry as UserAgentEntry;
if (userAgentEntry == null || userAgentEntry.Type != EntryType.UserAgent)
{
continue;
}
if (string.Compare(userAgentEntry.UserAgent, userAgent, true) == 0)
{
return(userAgentEntry);
}
if (string.Compare(userAgentEntry.UserAgent, UserAgents.AllAgents, true) == 0)
{
allAgentsEntry = userAgentEntry;
}
}
return(allAgentsEntry);
}
/// <summary>
/// Loads the robots.txt from the specified TextReader.
/// </summary>
/// <param name="reader">The TextReader used to feed the robots.txt data into the document. May not be null.</param>
/// <param name="baseUri"></param>
public void Load(TextReader reader, Uri baseUri)
{
if (reader == null)
{
throw new ArgumentNullException("reader");
}
_baseUri = baseUri;
List <UserAgentEntry> userAgentsGroup = new List <UserAgentEntry>();
bool addedEntriesToUserAgent = false;
do
{
string line = reader.ReadLine();
if (line == null)
{
break;
}
if (string.IsNullOrEmpty(line))
{
continue;
}
Entry entry;
if (!Entry.TryParse(_baseUri, line, out entry))
{
continue;
}
if (entry.Type == EntryType.UserAgent)
{
UserAgentEntry userAgentEntry = (UserAgentEntry)entry;
if (addedEntriesToUserAgent)
{
userAgentsGroup.Clear();
addedEntriesToUserAgent = false;
}
UserAgentEntry foundUserAgentEntry = FindExplicitUserAgentEntry(userAgentEntry.UserAgent);
if (foundUserAgentEntry == null)
{
_entries.Add(userAgentEntry);
userAgentsGroup.Add(userAgentEntry);
}
else
{
userAgentsGroup.Add(foundUserAgentEntry);
}
}
else if (entry.Type == EntryType.Comment)
{
_entries.Add(entry);
}
else if (entry.Type == EntryType.Sitemap)
{
_entries.Add(entry);
}
else if (userAgentsGroup.Count > 0)
{
foreach (UserAgentEntry userAgent in userAgentsGroup)
{
userAgent.AddEntry(entry);
}
addedEntriesToUserAgent = true;
}
else
{
continue;
}
} while (true);
}
public void AllowEntriesTest()
{
var target = new UserAgentEntry();
Assert.Empty(target.AllowEntries);
}
public void EntryType_Test()
{
var target = new UserAgentEntry();
Assert.Equal(EntryType.UserAgent, target.Type);
}
public void AllowEntriesTest()
{
var target = new UserAgentEntry();
Assert.AreEqual(0, target.AllowEntries.Count());
}
