private void Process(TextReader reader)
{
string userAgent = "";
// add the defaults
addStringToMatcher(Disallow,
"*",
new FiniteStateMachine <FiniteStateMachineBooleanState, char>()
);
addStringToMatcher(Allow,
"*",
new FiniteStateMachine <FiniteStateMachineBooleanState, char>()
);
while (reader.Peek() > -1)
{
string line = reader.ReadLine().Trim();
// skip comments
if (line.StartsWith("#", StringComparison.InvariantCultureIgnoreCase))
{
continue;
}
if (line.StartsWith("user-agent:", StringComparison.InvariantCultureIgnoreCase))
{
userAgent = "";
if (line.Length > 11)
{
userAgent = line.Substring(11, line.Length - 11).Trim();
}
if (userAgent == FetchoConfiguration.Current?.UserAgent)
{
log.Error(this.Uri + " has a specific restriction for our user-agent");
}
addStringToMatcher(Disallow,
userAgent,
new FiniteStateMachine <FiniteStateMachineBooleanState, char>()
);
addStringToMatcher(Allow,
userAgent,
new FiniteStateMachine <FiniteStateMachineBooleanState, char>()
);
}
else
{
if (line.EndsWith("*", StringComparison.InvariantCultureIgnoreCase))
{
line = line.Substring(0, line.Length - 1); // chop it
}
if (line.StartsWith("disallow:", StringComparison.InvariantCultureIgnoreCase))
{
var disallow_matcher = Disallow.GetState(userAgent);
if (disallow_matcher.Count() == 0)
{
throw new FetchoException("No default disallow matcher available for '" + userAgent + "' uri " + Uri);
}
if (line.Length > 9)
{
addStringToMatcher(disallow_matcher.First(),
line.Substring(9, line.Length - 9).Trim(),
FiniteStateMachineBooleanState.Accept);
}
}
else if (line.StartsWith("allow:", StringComparison.InvariantCultureIgnoreCase))
{
var allow_matcher = Allow.GetState(userAgent);
if (allow_matcher.Count() == 0)
{
throw new FetchoException("No default allow matcher available for '" + userAgent + "' uri " + Uri);
}
if (line.Length > 6)
{
addStringToMatcher(allow_matcher.First(),
line.Substring(6, line.Length - 6).Trim(),
FiniteStateMachineBooleanState.Accept);
}
}
else if (line.StartsWith("sitemap:", StringComparison.InvariantCultureIgnoreCase))
{
if (line.Length > 8)
{
SiteMaps.Add(line.Substring(8, line.Length - 8).Trim());
}
}
}
}
}
