public void handleDirective(
Parser.DirectiveType directiveType, String directiveValue)
{
switch (directiveType)
{
case Parser.DirectiveType.USER_AGENT:
{
handleUserAgent(directiveValue);
break;
}
case Parser.DirectiveType.ALLOW:
case Parser.DirectiveType.DISALLOW:
{
foundContent = true;
if (currentGroup.isGlobal() || currentGroup.getUserAgents().size() > 0)
{
String path = maybeEscapePattern(directiveValue);
currentGroup.addRule(directiveType, path);
if (directiveType == Parser.DirectiveType.ALLOW)
{
// Google-specific optimization: 'index.htm' and 'index.html' are normalized to '/'.
int slashPos = path.lastIndexOf('/');
if (slashPos != -1)
{
String fileName = path.substring(slashPos + 1);
if ("index.htm".equals(fileName) || "index.html".equals(fileName))
{
String normalizedPath = path.substring(0, slashPos + 1) + '$';
if (!currentGroup.hasRule(Parser.DirectiveType.ALLOW, normalizedPath))
{
//FITIT: logging later
//logger.atInfo().log(
// "Allowing normalized path: \"%s\" -> \"%s\"",
// directiveValue, normalizedPath);
currentGroup.addRule(Parser.DirectiveType.ALLOW, normalizedPath);
}
}
}
}
}
break;
}
case Parser.DirectiveType.SITEMAP:
case Parser.DirectiveType.UNKNOWN:
{
foundContent = true;
break;
}
}
}
/**
* Computes {@link Match} priorities for ALLOW and DISALLOW verdicts. Rules are considered
* effective if at least one user agent is listed in "user-agent" directives or applied globally
* (if global rules are not ignored).
*
* @param userAgents list of interested user agents
* @param path target path
* @param ignoreGlobal global rules will not be considered if set to {@code true}
* @return pair of {@link Match} representing ALLOW and DISALLOW priorities respectively
*/
private java.util.MapNS.Entry <Match, Match> computeMatchPriorities(
java.util.List <String> userAgents, String path, bool ignoreGlobal)
{
Match allow = new Match();
Match disallow = new Match();
bool foundSpecificGroup = false;
java.util.Iterator <RobotsContents.Group> iter1 = robotsContents.getGroups().iterator();
while (iter1.hasNext())
{
RobotsContents.Group group = iter1.next();
//foreach (RobotsContents.Group group in robotsContents.getGroups()) {
bool isSpecificGroup = false;
for (int i = 0; i < userAgents.size() && !isSpecificGroup; i++) // userAgents.stream()
{
java.util.Iterator <String> iter3 = group.getUserAgents().iterator(); //group.getUserAgents().stream()
while (iter3.hasNext())
{
String next3 = iter3.next().ToLower(); //userAgent::equalsIgnoreCase
if (userAgents.get(i).ToLower().Equals(next3)) // anyMatch/anyMatch
{
isSpecificGroup = true;
}
}
}
/* see lines before - better with LINQ, maybe later
* bool isSpecificGroup =
* userAgents.stream()
* .anyMatch(
* userAgent ->
* group.getUserAgents().stream().anyMatch(userAgent::equalsIgnoreCase));
*/
foundSpecificGroup |= isSpecificGroup;
if (!isSpecificGroup && (ignoreGlobal || !group.isGlobal()))
{
continue;
}
java.util.Iterator <RobotsContents.Group.Rule> iter2 = group.getRules().iterator();
while (iter2.hasNext())
{
RobotsContents.Group.Rule rule = iter2.next();
//foreach (RobotsContents.Group.Rule rule in group.getRules()) {
switch (rule.getDirectiveType())
{
case Parser.DirectiveType.ALLOW:
{
int priority =
matchingStrategy.matchAllowPriority(path, rule.getDirectiveValue());
if (isSpecificGroup)
{
allow.updateSpecific(priority);
}
if (!ignoreGlobal && group.isGlobal())
{
allow.updateGlobal(priority);
}
break;
}
case Parser.DirectiveType.DISALLOW:
{
int priority =
matchingStrategy.matchDisallowPriority(path, rule.getDirectiveValue());
if (isSpecificGroup)
{
disallow.updateSpecific(priority);
}
if (!ignoreGlobal && group.isGlobal())
{
disallow.updateGlobal(priority);
}
break;
}
case Parser.DirectiveType.SITEMAP:
case Parser.DirectiveType.UNKNOWN:
case Parser.DirectiveType.USER_AGENT:
break;
}
}
}
// If there is at least one group specific for current agents, global groups should be
// disregarded.
if (foundSpecificGroup)
{
allow.resetGlobal();
disallow.resetGlobal();
}
return(java.util.Map <Match, Match> .entry(allow, disallow));
}
