예제 #1
0
        public void handleDirective(
            Parser.DirectiveType directiveType, String directiveValue)
        {
            switch (directiveType)
            {
            case Parser.DirectiveType.USER_AGENT:
            {
                handleUserAgent(directiveValue);
                break;
            }

            case Parser.DirectiveType.ALLOW:
            case Parser.DirectiveType.DISALLOW:
            {
                foundContent = true;
                if (currentGroup.isGlobal() || currentGroup.getUserAgents().size() > 0)
                {
                    String path = maybeEscapePattern(directiveValue);
                    currentGroup.addRule(directiveType, path);

                    if (directiveType == Parser.DirectiveType.ALLOW)
                    {
                        // Google-specific optimization: 'index.htm' and 'index.html' are normalized to '/'.
                        int slashPos = path.lastIndexOf('/');

                        if (slashPos != -1)
                        {
                            String fileName = path.substring(slashPos + 1);
                            if ("index.htm".equals(fileName) || "index.html".equals(fileName))
                            {
                                String normalizedPath = path.substring(0, slashPos + 1) + '$';

                                if (!currentGroup.hasRule(Parser.DirectiveType.ALLOW, normalizedPath))
                                {
                                    //FITIT: logging later
                                    //logger.atInfo().log(
                                    //    "Allowing normalized path: \"%s\" -> \"%s\"",
                                    //    directiveValue, normalizedPath);
                                    currentGroup.addRule(Parser.DirectiveType.ALLOW, normalizedPath);
                                }
                            }
                        }
                    }
                }
                break;
            }

            case Parser.DirectiveType.SITEMAP:
            case Parser.DirectiveType.UNKNOWN:
            {
                foundContent = true;
                break;
            }
            }
        }
        /**
         * Computes {@link Match} priorities for ALLOW and DISALLOW verdicts. Rules are considered
         * effective if at least one user agent is listed in "user-agent" directives or applied globally
         * (if global rules are not ignored).
         *
         * @param userAgents list of interested user agents
         * @param path target path
         * @param ignoreGlobal global rules will not be considered if set to {@code true}
         * @return pair of {@link Match} representing ALLOW and DISALLOW priorities respectively
         */
        private java.util.MapNS.Entry <Match, Match> computeMatchPriorities(
            java.util.List <String> userAgents, String path, bool ignoreGlobal)
        {
            Match allow              = new Match();
            Match disallow           = new Match();
            bool  foundSpecificGroup = false;

            java.util.Iterator <RobotsContents.Group> iter1 = robotsContents.getGroups().iterator();
            while (iter1.hasNext())
            {
                RobotsContents.Group group = iter1.next();
                //foreach (RobotsContents.Group group in robotsContents.getGroups()) {
                bool isSpecificGroup = false;
                for (int i = 0; i < userAgents.size() && !isSpecificGroup; i++)           // userAgents.stream()
                {
                    java.util.Iterator <String> iter3 = group.getUserAgents().iterator(); //group.getUserAgents().stream()
                    while (iter3.hasNext())
                    {
                        String next3 = iter3.next().ToLower();         //userAgent::equalsIgnoreCase
                        if (userAgents.get(i).ToLower().Equals(next3)) // anyMatch/anyMatch
                        {
                            isSpecificGroup = true;
                        }
                    }
                }

                /* see lines before - better with LINQ, maybe later
                 * bool isSpecificGroup =
                 *  userAgents.stream()
                 *      .anyMatch(
                 *          userAgent ->
                 *              group.getUserAgents().stream().anyMatch(userAgent::equalsIgnoreCase));
                 */
                foundSpecificGroup |= isSpecificGroup;
                if (!isSpecificGroup && (ignoreGlobal || !group.isGlobal()))
                {
                    continue;
                }

                java.util.Iterator <RobotsContents.Group.Rule> iter2 = group.getRules().iterator();
                while (iter2.hasNext())
                {
                    RobotsContents.Group.Rule rule = iter2.next();
                    //foreach (RobotsContents.Group.Rule rule in group.getRules()) {
                    switch (rule.getDirectiveType())
                    {
                    case Parser.DirectiveType.ALLOW:
                    {
                        int priority =
                            matchingStrategy.matchAllowPriority(path, rule.getDirectiveValue());
                        if (isSpecificGroup)
                        {
                            allow.updateSpecific(priority);
                        }
                        if (!ignoreGlobal && group.isGlobal())
                        {
                            allow.updateGlobal(priority);
                        }
                        break;
                    }

                    case Parser.DirectiveType.DISALLOW:
                    {
                        int priority =
                            matchingStrategy.matchDisallowPriority(path, rule.getDirectiveValue());
                        if (isSpecificGroup)
                        {
                            disallow.updateSpecific(priority);
                        }
                        if (!ignoreGlobal && group.isGlobal())
                        {
                            disallow.updateGlobal(priority);
                        }
                        break;
                    }

                    case Parser.DirectiveType.SITEMAP:
                    case Parser.DirectiveType.UNKNOWN:
                    case Parser.DirectiveType.USER_AGENT:
                        break;
                    }
                }
            }

            // If there is at least one group specific for current agents, global groups should be
            // disregarded.
            if (foundSpecificGroup)
            {
                allow.resetGlobal();
                disallow.resetGlobal();
            }

            return(java.util.Map <Match, Match> .entry(allow, disallow));
        }