private void Process(TextReader reader) { string userAgent = ""; // add the defaults addStringToMatcher(Disallow, "*", new FiniteStateMachine <FiniteStateMachineBooleanState, char>() ); addStringToMatcher(Allow, "*", new FiniteStateMachine <FiniteStateMachineBooleanState, char>() ); while (reader.Peek() > -1) { string line = reader.ReadLine().Trim(); // skip comments if (line.StartsWith("#", StringComparison.InvariantCultureIgnoreCase)) { continue; } if (line.StartsWith("user-agent:", StringComparison.InvariantCultureIgnoreCase)) { userAgent = ""; if (line.Length > 11) { userAgent = line.Substring(11, line.Length - 11).Trim(); } if (userAgent == FetchoConfiguration.Current?.UserAgent) { log.Error(this.Uri + " has a specific restriction for our user-agent"); } addStringToMatcher(Disallow, userAgent, new FiniteStateMachine <FiniteStateMachineBooleanState, char>() ); addStringToMatcher(Allow, userAgent, new FiniteStateMachine <FiniteStateMachineBooleanState, char>() ); } else { if (line.EndsWith("*", StringComparison.InvariantCultureIgnoreCase)) { line = line.Substring(0, line.Length - 1); // chop it } if (line.StartsWith("disallow:", StringComparison.InvariantCultureIgnoreCase)) { var disallow_matcher = Disallow.GetState(userAgent); if (disallow_matcher.Count() == 0) { throw new FetchoException("No default disallow matcher available for '" + userAgent + "' uri " + Uri); } if (line.Length > 9) { addStringToMatcher(disallow_matcher.First(), line.Substring(9, line.Length - 9).Trim(), FiniteStateMachineBooleanState.Accept); } } else if (line.StartsWith("allow:", StringComparison.InvariantCultureIgnoreCase)) { var allow_matcher = Allow.GetState(userAgent); if (allow_matcher.Count() == 0) { throw new FetchoException("No default allow matcher available for '" + userAgent + "' uri " + Uri); } if (line.Length > 6) { addStringToMatcher(allow_matcher.First(), line.Substring(6, line.Length - 6).Trim(), FiniteStateMachineBooleanState.Accept); } } else if (line.StartsWith("sitemap:", StringComparison.InvariantCultureIgnoreCase)) { if (line.Length > 8) { SiteMaps.Add(line.Substring(8, line.Length - 8).Trim()); } } } } }