예제 #1
0
        public virtual void BuildNormalRuleFunction(Rule r, RuleFunction function)
        {
            CodeGenerator gen = @delegate.GetGenerator();
            // TRIGGER factory functions for rule alts, elements
            GrammarASTAdaptor    adaptor = new GrammarASTAdaptor(r.ast.Token.InputStream);
            GrammarAST           blk     = (GrammarAST)r.ast.GetFirstChildWithType(ANTLRParser.BLOCK);
            CommonTreeNodeStream nodes   = new CommonTreeNodeStream(adaptor, blk);

            walker = new SourceGenTriggers(nodes, this);
            try
            {
                // walk AST of rule alts/elements
                function.code = DefaultOutputModelFactory.List(walker.block(null, null));
                function.hasLookaheadBlock = walker.hasLookaheadBlock;
            }
            catch (Antlr.Runtime.RecognitionException e)
            {
                @delegate.GetGenerator().tool.ConsoleError.WriteLine(e.Message);
                @delegate.GetGenerator().tool.ConsoleError.WriteLine(e.StackTrace);
            }

            function.ctxType = @delegate.GetTarget().GetRuleFunctionContextStructName(function);

            function.postamble = RulePostamble(function, r);
        }
        public virtual RuleAST ParseArtificialRule(Grammar g, string ruleText)
        {
            ANTLRLexer        lexer   = new ANTLRLexer(new ANTLRStringStream(ruleText));
            GrammarASTAdaptor adaptor = new GrammarASTAdaptor(lexer.CharStream);
            CommonTokenStream tokens  = new CommonTokenStream(lexer);

            lexer.tokens = tokens;
            ToolANTLRParser p = new ToolANTLRParser(tokens, tool);

            p.TreeAdaptor = adaptor;
            IToken ruleStart = null;

            try
            {
                Antlr.Runtime.AstParserRuleReturnScope <GrammarAST, IToken> r = p.rule();
                RuleAST tree = (RuleAST)r.Tree;
                ruleStart = r.Start;
                GrammarTransformPipeline.SetGrammarPtr(g, tree);
                GrammarTransformPipeline.AugmentTokensWithOriginalPosition(g, tree);
                return(tree);
            }
            catch (Exception e)
            {
                tool.errMgr.ToolError(ErrorType.INTERNAL_ERROR,
                                      e,
                                      ruleStart,
                                      "error parsing rule created during left-recursion detection: " + ruleText);
            }
            return(null);
        }
예제 #3
0
        public virtual GrammarAST DupTree()
        {
            GrammarAST        t       = this;
            ICharStream       input   = this.Token.InputStream;
            GrammarASTAdaptor adaptor = new GrammarASTAdaptor(input);

            return((GrammarAST)adaptor.DupTree(t));
        }
        public virtual void ReduceBlocksToSets(GrammarAST root)
        {
            CommonTreeNodeStream nodes       = new CommonTreeNodeStream(new GrammarASTAdaptor(), root);
            GrammarASTAdaptor    adaptor     = new GrammarASTAdaptor();
            BlockSetTransformer  transformer = new BlockSetTransformer(nodes, g);

            transformer.TreeAdaptor = adaptor;
            transformer.Downup(root);
        }
예제 #5
0
        public virtual string ToTokenString()
        {
            ICharStream          input   = this.Token.InputStream;
            GrammarASTAdaptor    adaptor = new GrammarASTAdaptor(input);
            CommonTreeNodeStream nodes   =
                new CommonTreeNodeStream(adaptor, this);
            StringBuilder buf  = new StringBuilder();
            GrammarAST    o    = (GrammarAST)nodes.LT(1);
            int           type = adaptor.GetType(o);

            while (type != TokenTypes.EndOfFile)
            {
                buf.Append(" ");
                buf.Append(o.Text);
                nodes.Consume();
                o    = (GrammarAST)nodes.LT(1);
                type = adaptor.GetType(o);
            }

            return(buf.ToString());
        }
예제 #6
0
        protected virtual void _CreateATN([NotNull] ICollection <Rule> rules)
        {
            CreateRuleStartAndStopATNStates();

            GrammarASTAdaptor adaptor = new GrammarASTAdaptor();

            foreach (Rule r in rules)
            {
                // find rule's block
                GrammarAST           blk   = (GrammarAST)r.ast.GetFirstChildWithType(ANTLRParser.BLOCK);
                CommonTreeNodeStream nodes = new CommonTreeNodeStream(adaptor, blk);
                ATNBuilder           b     = new ATNBuilder(nodes, this);
                try {
                    SetCurrentRuleName(r.name);
                    Handle h = b.ruleBlock(null);
                    Rule(r.ast, r.name, h);
                }
                catch (RecognitionException re) {
                    g.tool.errMgr.FatalInternalError("bad grammar AST structure", re);
                }
            }
        }
예제 #7
0
 public virtual GrammarRootAST Parse(string fileName, ICharStream @in)
 {
     try
     {
         GrammarASTAdaptor adaptor = new GrammarASTAdaptor(@in);
         ToolANTLRLexer    lexer   = new ToolANTLRLexer(@in, this);
         CommonTokenStream tokens  = new CommonTokenStream(lexer);
         lexer.tokens = tokens;
         ToolANTLRParser p = new ToolANTLRParser(tokens, this);
         p.TreeAdaptor = adaptor;
         try
         {
             var        r    = p.grammarSpec();
             GrammarAST root = (GrammarAST)r.Tree;
             if (root is GrammarRootAST)
             {
                 ((GrammarRootAST)root).hasErrors = lexer.NumberOfSyntaxErrors > 0 || p.NumberOfSyntaxErrors > 0;
                 Debug.Assert(((GrammarRootAST)root).tokenStream == tokens);
                 if (grammarOptions != null)
                 {
                     ((GrammarRootAST)root).cmdLineOptions = grammarOptions;
                 }
                 return((GrammarRootAST)root);
             }
         }
         catch (v3TreeGrammarException e)
         {
             errMgr.GrammarError(ErrorType.V3_TREE_GRAMMAR, fileName, e.location);
         }
         return(null);
     }
     catch (RecognitionException)
     {
         // TODO: do we gen errors now?
         ErrorManager.InternalError("can't generate this message at moment; antlr recovers");
     }
     return(null);
 }
        /** Build lexer grammar from combined grammar that looks like:
         *
         *  (COMBINED_GRAMMAR A
         *      (tokens { X (= Y 'y'))
         *      (OPTIONS (= x 'y'))
         *      (@ members {foo})
         *      (@ lexer header {package jj;})
         *      (RULES (RULE .+)))
         *
         *  Move rules and actions to new tree, don't dup. Split AST apart.
         *  We'll have this Grammar share token symbols later; don't generate
         *  tokenVocab or tokens{} section.  Copy over named actions.
         *
         *  Side-effects: it removes children from GRAMMAR &amp; RULES nodes
         *                in combined AST.  Anything cut out is dup'd before
         *                adding to lexer to avoid "who's ur daddy" issues
         */
        public virtual GrammarRootAST ExtractImplicitLexer(Grammar combinedGrammar)
        {
            GrammarRootAST combinedAST = combinedGrammar.ast;
            //tool.log("grammar", "before="+combinedAST.toStringTree());
            GrammarASTAdaptor adaptor = new GrammarASTAdaptor(combinedAST.Token.InputStream);

            GrammarAST[] elements = combinedAST.GetChildrenAsArray();

            // MAKE A GRAMMAR ROOT and ID
            string         lexerName = combinedAST.GetChild(0).Text + "Lexer";
            GrammarRootAST lexerAST  =
                new GrammarRootAST(new CommonToken(ANTLRParser.GRAMMAR, "LEXER_GRAMMAR"), combinedGrammar.ast.tokenStream);

            lexerAST.grammarType       = ANTLRParser.LEXER;
            lexerAST.Token.InputStream = combinedAST.Token.InputStream;
            lexerAST.AddChild((ITree)adaptor.Create(ANTLRParser.ID, lexerName));

            // COPY OPTIONS
            GrammarAST optionsRoot =
                (GrammarAST)combinedAST.GetFirstChildWithType(ANTLRParser.OPTIONS);

            if (optionsRoot != null && optionsRoot.ChildCount != 0)
            {
                GrammarAST lexerOptionsRoot = (GrammarAST)adaptor.DupNode(optionsRoot);
                lexerAST.AddChild(lexerOptionsRoot);
                GrammarAST[] options = optionsRoot.GetChildrenAsArray();
                foreach (GrammarAST o in options)
                {
                    string optionName = o.GetChild(0).Text;
                    if (Grammar.lexerOptions.Contains(optionName) &&
                        !Grammar.doNotCopyOptionsToLexer.Contains(optionName))
                    {
                        GrammarAST optionTree = (GrammarAST)adaptor.DupTree(o);
                        lexerOptionsRoot.AddChild(optionTree);
                        lexerAST.SetOption(optionName, (GrammarAST)optionTree.GetChild(1));
                    }
                }
            }

            // COPY all named actions, but only move those with lexer:: scope
            IList <GrammarAST> actionsWeMoved = new List <GrammarAST>();

            foreach (GrammarAST e in elements)
            {
                if (e.Type == ANTLRParser.AT)
                {
                    lexerAST.AddChild((ITree)adaptor.DupTree(e));
                    if (e.GetChild(0).Text.Equals("lexer"))
                    {
                        actionsWeMoved.Add(e);
                    }
                }
            }

            foreach (GrammarAST r in actionsWeMoved)
            {
                combinedAST.DeleteChild(r);
            }

            GrammarAST combinedRulesRoot =
                (GrammarAST)combinedAST.GetFirstChildWithType(ANTLRParser.RULES);

            if (combinedRulesRoot == null)
            {
                return(lexerAST);
            }

            // MOVE lexer rules

            GrammarAST lexerRulesRoot = (GrammarAST)adaptor.Create(ANTLRParser.RULES, "RULES");

            lexerAST.AddChild(lexerRulesRoot);
            IList <GrammarAST> rulesWeMoved = new List <GrammarAST>();

            GrammarASTWithOptions[] rules;
            if (combinedRulesRoot.ChildCount > 0)
            {
                rules = combinedRulesRoot.Children.Cast <GrammarASTWithOptions>().ToArray();
            }
            else
            {
                rules = new GrammarASTWithOptions[0];
            }

            foreach (GrammarASTWithOptions r in rules)
            {
                string ruleName = r.GetChild(0).Text;
                if (Grammar.IsTokenName(ruleName))
                {
                    lexerRulesRoot.AddChild((ITree)adaptor.DupTree(r));
                    rulesWeMoved.Add(r);
                }
            }

            foreach (GrammarAST r in rulesWeMoved)
            {
                combinedRulesRoot.DeleteChild(r);
            }

            // Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if'
            IList <System.Tuple <GrammarAST, GrammarAST> > litAliases =
                Grammar.GetStringLiteralAliasesFromLexerRules(lexerAST);

            ISet <string> stringLiterals = combinedGrammar.GetStringLiterals();
            // add strings from combined grammar (and imported grammars) into lexer
            // put them first as they are keywords; must resolve ambigs to these rules
            //		tool.log("grammar", "strings from parser: "+stringLiterals);
            int insertIndex = 0;

            foreach (string lit in stringLiterals)
            {
                // if lexer already has a rule for literal, continue
                if (litAliases != null)
                {
                    foreach (System.Tuple <GrammarAST, GrammarAST> pair in litAliases)
                    {
                        GrammarAST litAST = pair.Item2;
                        if (lit.Equals(litAST.Text))
                        {
                            goto continueNextLit;
                        }
                    }
                }
                // create for each literal: (RULE <uniquename> (BLOCK (ALT <lit>))
                string rname = combinedGrammar.GetStringLiteralLexerRuleName(lit);
                // can't use wizard; need special node types
                GrammarAST  litRule = new RuleAST(ANTLRParser.RULE);
                BlockAST    blk     = new BlockAST(ANTLRParser.BLOCK);
                AltAST      alt     = new AltAST(ANTLRParser.ALT);
                TerminalAST slit    = new TerminalAST(new CommonToken(ANTLRParser.STRING_LITERAL, lit));
                alt.AddChild(slit);
                blk.AddChild(alt);
                CommonToken idToken = new CommonToken(ANTLRParser.TOKEN_REF, rname);
                litRule.AddChild(new TerminalAST(idToken));
                litRule.AddChild(blk);
                lexerRulesRoot.InsertChild(insertIndex, litRule);
                //			lexerRulesRoot.getChildren().add(0, litRule);
                lexerRulesRoot.FreshenParentAndChildIndexes(); // reset indexes and set litRule parent

                // next literal will be added after the one just added
                insertIndex++;

continueNextLit:
                ;
            }

            // TODO: take out after stable if slow
            lexerAST.SanityCheckParentAndChildIndexes();
            combinedAST.SanityCheckParentAndChildIndexes();
            //		tool.log("grammar", combinedAST.toTokenString());

            combinedGrammar.tool.Log("grammar", "after extract implicit lexer =" + combinedAST.ToStringTree());
            combinedGrammar.tool.Log("grammar", "lexer =" + lexerAST.ToStringTree());

            if (lexerRulesRoot.ChildCount == 0)
            {
                return(null);
            }
            return(lexerAST);
        }
        /** Merge all the rules, token definitions, and named actions from
         *  imported grammars into the root grammar tree.  Perform:
         *
         *  (tokens { X (= Y 'y')) + (tokens { Z )	-&gt;	(tokens { X (= Y 'y') Z)
         *
         *  (@ members {foo}) + (@ members {bar})	-&gt;	(@ members {foobar})
         *
         *  (RULES (RULE x y)) + (RULES (RULE z))	-&gt;	(RULES (RULE x y z))
         *
         *  Rules in root prevent same rule from being appended to RULES node.
         *
         *  The goal is a complete combined grammar so we can ignore subordinate
         *  grammars.
         */
        public virtual void IntegrateImportedGrammars(Grammar rootGrammar)
        {
            IList <Grammar> imports = rootGrammar.GetAllImportedGrammars();

            if (imports == null)
            {
                return;
            }

            GrammarAST        root    = rootGrammar.ast;
            GrammarAST        id      = (GrammarAST)root.GetChild(0);
            GrammarASTAdaptor adaptor = new GrammarASTAdaptor(id.Token.InputStream);

            GrammarAST tokensRoot = (GrammarAST)root.GetFirstChildWithType(ANTLRParser.TOKENS_SPEC);

            IList <GrammarAST> actionRoots = root.GetNodesWithType(ANTLRParser.AT);

            // Compute list of rules in root grammar and ensure we have a RULES node
            GrammarAST    RULES         = (GrammarAST)root.GetFirstChildWithType(ANTLRParser.RULES);
            ISet <string> rootRuleNames = new HashSet <string>();
            // make list of rules we have in root grammar
            IList <GrammarAST> rootRules = RULES.GetNodesWithType(ANTLRParser.RULE);

            foreach (GrammarAST r in rootRules)
            {
                rootRuleNames.Add(r.GetChild(0).Text);
            }

            foreach (Grammar imp in imports)
            {
                // COPY TOKENS
                GrammarAST imp_tokensRoot = (GrammarAST)imp.ast.GetFirstChildWithType(ANTLRParser.TOKENS_SPEC);
                if (imp_tokensRoot != null)
                {
                    rootGrammar.tool.Log("grammar", "imported tokens: " + imp_tokensRoot.Children);
                    if (tokensRoot == null)
                    {
                        tokensRoot   = (GrammarAST)adaptor.Create(ANTLRParser.TOKENS_SPEC, "TOKENS");
                        tokensRoot.g = rootGrammar;
                        root.InsertChild(1, tokensRoot); // ^(GRAMMAR ID TOKENS...)
                    }
                    tokensRoot.AddChildren(imp_tokensRoot.Children);
                }

                IList <GrammarAST> all_actionRoots = new List <GrammarAST>();
                IList <GrammarAST> imp_actionRoots = imp.ast.GetAllChildrenWithType(ANTLRParser.AT);
                if (actionRoots != null)
                {
                    foreach (var actionRoot in actionRoots)
                    {
                        all_actionRoots.Add(actionRoot);
                    }
                }

                foreach (var actionRoot in imp_actionRoots)
                {
                    all_actionRoots.Add(actionRoot);
                }

                // COPY ACTIONS
                if (imp_actionRoots != null)
                {
                    IDictionary <System.Tuple <string, string>, GrammarAST> namedActions =
                        new Dictionary <System.Tuple <string, string>, GrammarAST>();

                    rootGrammar.tool.Log("grammar", "imported actions: " + imp_actionRoots);
                    foreach (GrammarAST at in all_actionRoots)
                    {
                        string     scopeName = rootGrammar.GetDefaultActionScope();
                        GrammarAST scope, name, action;
                        if (at.ChildCount > 2)
                        { // must have a scope
                            scope     = (GrammarAST)at.GetChild(0);
                            scopeName = scope.Text;
                            name      = (GrammarAST)at.GetChild(1);
                            action    = (GrammarAST)at.GetChild(2);
                        }
                        else
                        {
                            name   = (GrammarAST)at.GetChild(0);
                            action = (GrammarAST)at.GetChild(1);
                        }
                        GrammarAST prevAction;
                        if (!namedActions.TryGetValue(Tuple.Create(scopeName, name.Text), out prevAction) || prevAction == null)
                        {
                            namedActions[Tuple.Create(scopeName, name.Text)] = action;
                        }
                        else
                        {
                            if (prevAction.g == at.g)
                            {
                                rootGrammar.tool.errMgr.GrammarError(ErrorType.ACTION_REDEFINITION,
                                                                     at.g.fileName, name.Token, name.Text);
                            }
                            else
                            {
                                string s1 = prevAction.Text;
                                s1 = s1.Substring(1, s1.Length - 2);
                                string s2 = action.Text;
                                s2 = s2.Substring(1, s2.Length - 2);
                                string combinedAction = "{" + s1 + '\n' + s2 + "}";
                                prevAction.Token.Text = combinedAction;
                            }
                        }
                    }
                    // at this point, we have complete list of combined actions,
                    // some of which are already living in root grammar.
                    // Merge in any actions not in root grammar into root's tree.
                    foreach (string scopeName in namedActions.Keys.Select(i => i.Item1).Distinct())
                    {
                        foreach (string name in namedActions.Keys.Where(i => i.Item1 == scopeName).Select(i => i.Item2))
                        {
                            GrammarAST action = namedActions[Tuple.Create(scopeName, name)];
                            rootGrammar.tool.Log("grammar", action.g.name + " " + scopeName + ":" + name + "=" + action.Text);
                            if (action.g != rootGrammar)
                            {
                                root.InsertChild(1, action.Parent);
                            }
                        }
                    }
                }

                // COPY RULES
                IList <GrammarAST> rules = imp.ast.GetNodesWithType(ANTLRParser.RULE);
                if (rules != null)
                {
                    foreach (GrammarAST r in rules)
                    {
                        rootGrammar.tool.Log("grammar", "imported rule: " + r.ToStringTree());
                        string name = r.GetChild(0).Text;
                        bool   rootAlreadyHasRule = rootRuleNames.Contains(name);
                        if (!rootAlreadyHasRule)
                        {
                            RULES.AddChild(r); // merge in if not overridden
                            rootRuleNames.Add(name);
                        }
                    }
                }

                GrammarAST optionsRoot = (GrammarAST)imp.ast.GetFirstChildWithType(ANTLRParser.OPTIONS);
                if (optionsRoot != null)
                {
                    // suppress the warning if the options match the options specified
                    // in the root grammar
                    // https://github.com/antlr/antlr4/issues/707

                    bool hasNewOption = false;
                    foreach (KeyValuePair <string, GrammarAST> option in imp.ast.GetOptions())
                    {
                        string importOption = imp.ast.GetOptionString(option.Key);
                        if (importOption == null)
                        {
                            continue;
                        }

                        string rootOption = rootGrammar.ast.GetOptionString(option.Key);
                        if (!importOption.Equals(rootOption))
                        {
                            hasNewOption = true;
                            break;
                        }
                    }

                    if (hasNewOption)
                    {
                        rootGrammar.tool.errMgr.GrammarError(ErrorType.OPTIONS_IN_DELEGATE,
                                                             optionsRoot.g.fileName, optionsRoot.Token, imp.name);
                    }
                }
            }
            rootGrammar.tool.Log("grammar", "Grammar: " + rootGrammar.ast.ToStringTree());
        }