ParseAndBuildAST() public method

public ParseAndBuildAST ( ) : void
return void
        public void Test3LevelImport()
        {
            ErrorQueue equeue = new ErrorQueue();
            ErrorManager.SetErrorListener( equeue );
            string slave =
                "parser grammar T;\n" +
                "a : T ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "T.g", slave );
            string slave2 =
                "parser grammar S;\n" + // A, B, C token type order
                "import T;\n" +
                "a : S ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "S.g", slave2 );

            string master =
                "grammar M;\n" +
                "import S;\n" +
                "a : M ;\n";
            writeFile( tmpdir, "M.g", master );
            AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } );
            CompositeGrammar composite = new CompositeGrammar();
            Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite );
            composite.SetDelegationRoot( g );
            g.ParseAndBuildAST();
            g.composite.AssignTokenTypes();
            g.composite.DefineGrammarSymbols();

            string expectedTokenIDToTypeMap = "[M=6, S=5, T=4]";
            string expectedStringLiteralToTypeMap = "{}";
            string expectedTypeToTokenList = "[T, S, M]";

            assertEquals( expectedTokenIDToTypeMap,
                         realElements( g.composite.tokenIDToTypeMap ).ToElementString() );
            assertEquals( expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.ToElementString() );
            assertEquals( expectedTypeToTokenList,
                         realElements( g.composite.typeToTokenList ).ToElementString() );

            assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );

            bool ok =
                rawGenerateAndBuildRecognizer( "M.g", master, "MParser", null, false );
            bool expecting = true; // should be ok
            assertEquals( expecting, ok );
        }
Beispiel #2
0
        public void TestArgsOnTokenInLexerRuleOfCombined()
        {
            ErrorQueue equeue = new ErrorQueue();
            ErrorManager.SetErrorListener( equeue );
            Grammar g = new Grammar(
                "grammar t;\n" +
                "a : R;\n" +
                "R : 'z' ID[32] ;\n" +
                "ID : 'a';\n" );

            string lexerGrammarStr = g.GetLexerGrammar();
            System.IO.StringReader sr = new System.IO.StringReader( lexerGrammarStr );
            Grammar lexerGrammar = new Grammar();
            lexerGrammar.FileName = "<internally-generated-lexer>";
            lexerGrammar.ImportTokenVocabulary( g );
            lexerGrammar.ParseAndBuildAST( sr );
            lexerGrammar.DefineGrammarSymbols();
            lexerGrammar.CheckNameSpaceAndActions();
            sr.Close();

            AntlrTool antlr = newTool();
            antlr.SetOutputDirectory( null ); // write to /dev/null
            CodeGenerator generator = new CodeGenerator( antlr, lexerGrammar, "Java" );
            lexerGrammar.CodeGenerator = generator;
            generator.GenRecognizer();

            int expectedMsgID = ErrorManager.MSG_RULE_HAS_NO_ARGS;
            object expectedArg = "ID";
            object expectedArg2 = null;
            GrammarSemanticsMessage expectedMessage =
                new GrammarSemanticsMessage( expectedMsgID, lexerGrammar, null, expectedArg, expectedArg2 );
            checkError( equeue, expectedMessage );
        }
Beispiel #3
0
        /** Import the rules/tokens of a delegate grammar. All delegate grammars are
         *  read during the ctor of first Grammar created.
         *
         *  Do not create NFA here because NFA construction needs to hook up with
         *  overridden rules in delegation root grammar.
         */
        public virtual void ImportGrammar( IToken grammarNameToken, string label )
        {
            string grammarName = grammarNameToken.Text;
            //[email protected]("import "+gfile.getName());
            string gname = grammarName + GrammarFileExtension;
            TextReader br = null;
            try
            {
                string fullName = Tool.GetLibraryFile( gname );
                if (!File.Exists(fullName))
                {
                    gname = grammarName + AltGrammarFileExtension;
                    fullName = Tool.GetLibraryFile(gname);
                }

                //FileReader fr = new FileReader( fullName );
                //br = new BufferedReader( fr );
                br = new StringReader( System.IO.File.ReadAllText( fullName ) );
                Grammar delegateGrammar = null;
                delegateGrammar = new Grammar( Tool, gname, composite );
                delegateGrammar.label = label;

                AddDelegateGrammar( delegateGrammar );

                delegateGrammar.ParseAndBuildAST( br );
                delegateGrammar.AddRulesForSyntacticPredicates();
                if ( !ValidImport( delegateGrammar ) )
                {
                    ErrorManager.GrammarError( ErrorManager.MSG_INVALID_IMPORT,
                                              this,
                                              grammarNameToken,
                                              this,
                                              delegateGrammar );
                    return;
                }
                if ( this.type == GrammarType.Combined &&
                     ( delegateGrammar.name.Equals( this.name + grammarTypeToFileNameSuffix[(int)GrammarType.Lexer] ) ||
                      delegateGrammar.name.Equals( this.name + grammarTypeToFileNameSuffix[(int)GrammarType.Parser] ) ) )
                {
                    ErrorManager.GrammarError( ErrorManager.MSG_IMPORT_NAME_CLASH,
                                              this,
                                              grammarNameToken,
                                              this,
                                              delegateGrammar );
                    return;
                }
                if ( delegateGrammar.grammarTree != null )
                {
                    // we have a valid grammar
                    // deal with combined grammars
                    if ( delegateGrammar.type == GrammarType.Lexer && this.type == GrammarType.Combined )
                    {
                        // ooops, we wasted some effort; tell lexer to read it in
                        // later
                        LexerGrammarTemplate.SetAttribute( "imports", grammarName );
                        // but, this parser grammar will need the vocab
                        // so add to composite anyway so we suck in the tokens later
                    }
                }
                //[email protected]("Got grammar:\n"+delegateGrammar);
            }
            catch ( IOException ioe )
            {
                ErrorManager.Error( ErrorManager.MSG_CANNOT_OPEN_FILE,
                                   gname,
                                   ioe );
            }
            finally
            {
                if ( br != null )
                {
                    try
                    {
                        br.Close();
                    }
                    catch ( IOException ioe )
                    {
                        ErrorManager.Error( ErrorManager.MSG_CANNOT_CLOSE_FILE,
                                           gname,
                                           ioe );
                    }
                }
            }
        }
        public void TestSyntaxErrorsInImportsNotThrownOut2()
        {
            ErrorQueue equeue = new ErrorQueue();
            ErrorManager.SetErrorListener( equeue );
            string slave =
                "parser grammar S;\n" +
                ": A {System.out.println(\"S.x\");} ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "S.g", slave );

            string master =
                "grammar M;\n" +
                "import S;\n" +
                "s : x ;\n" +
                "WS : (' '|'\\n') {skip();} ;\n";
            writeFile( tmpdir, "M.g", master );
            AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } );
            CompositeGrammar composite = new CompositeGrammar();
            Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite );
            composite.SetDelegationRoot( g );
            g.ParseAndBuildAST();
            g.composite.AssignTokenTypes();

            // whole bunch of errors from bad S.g file
            assertEquals( "unexpected errors: " + equeue, 3, equeue.errors.Count );
        }
        public void TestSameStringTwoNames()
        {
            Assert.Inconclusive( "May be failing on just my port..." );
            ErrorQueue equeue = new ErrorQueue();
            ErrorManager.SetErrorListener( equeue );
            string slave =
                "parser grammar S;\n" +
                "tokens { A='a'; }\n" +
                "x : A {System.out.println(\"S.x\");} ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "S.g", slave );
            string slave2 =
                "parser grammar T;\n" +
                "tokens { X='a'; }\n" +
                "y : X {System.out.println(\"T.y\");} ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "T.g", slave2 );

            string master =
                "grammar M;\n" +
                "import S,T;\n" +
                "s : x y ;\n" +
                "WS : (' '|'\\n') {skip();} ;\n";
            writeFile( tmpdir, "M.g", master );
            AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } );
            CompositeGrammar composite = new CompositeGrammar();
            Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite );
            composite.SetDelegationRoot( g );
            g.ParseAndBuildAST();
            g.composite.AssignTokenTypes();

            string expectedTokenIDToTypeMap = "[A=4, WS=6, X=5]";
            string expectedStringLiteralToTypeMap = "{'a'=4}";
            string expectedTypeToTokenList = "[A, X, WS]";

            assertEquals( expectedTokenIDToTypeMap,
                         realElements( g.composite.tokenIDToTypeMap ).ToElementString() );
            assertEquals( expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.ToString() );
            assertEquals( expectedTypeToTokenList,
                         realElements( g.composite.typeToTokenList ).ToElementString() );

            object expectedArg = "X='a'";
            object expectedArg2 = "A";
            int expectedMsgID = ErrorManager.MSG_TOKEN_ALIAS_CONFLICT;
            GrammarSemanticsMessage expectedMessage =
                new GrammarSemanticsMessage( expectedMsgID, g, null, expectedArg, expectedArg2 );
            checkGrammarSemanticsError( equeue, expectedMessage );

            assertEquals( "unexpected errors: " + equeue, 1, equeue.errors.Count );

            string expectedError =
                "error(158): T.g:2:10: cannot alias X='a'; string already assigned to A";
            assertEquals( expectedError, equeue.errors[0].ToString() );
        }
        public void TestRulesVisibleThroughMultilevelImport()
        {
            ErrorQueue equeue = new ErrorQueue();
            ErrorManager.SetErrorListener( equeue );
            string slave =
                "parser grammar T;\n" +
                "x : T ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "T.g", slave );
            string slave2 =
                "parser grammar S;\n" + // A, B, C token type order
                "import T;\n" +
                "a : S ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "S.g", slave2 );

            string master =
                "grammar M;\n" +
                "import S;\n" +
                "a : M x ;\n"; // x MUST BE VISIBLE TO M
            writeFile( tmpdir, "M.g", master );
            AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } );
            CompositeGrammar composite = new CompositeGrammar();
            Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite );
            composite.SetDelegationRoot( g );
            g.ParseAndBuildAST();
            g.composite.AssignTokenTypes();
            g.composite.DefineGrammarSymbols();

            string expectedTokenIDToTypeMap = "[M=6, S=5, T=4]";
            string expectedStringLiteralToTypeMap = "{}";
            string expectedTypeToTokenList = "[T, S, M]";

            assertEquals( expectedTokenIDToTypeMap,
                         realElements( g.composite.tokenIDToTypeMap ).ToElementString() );
            assertEquals( expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.ToElementString() );
            assertEquals( expectedTypeToTokenList,
                         realElements( g.composite.typeToTokenList ).ToElementString() );

            assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
        }
        public void TestLexerDelegatorRuleOverridesDelegateLeavingNoRules()
        {
            // M.Tokens has nothing to predict tokens from S.  Should
            // not include S.Tokens alt in this case?
            string slave =
                "lexer grammar S;\n" +
                "A : 'a' {System.out.println(\"S.A\");} ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "S.g", slave );
            string master =
                "lexer grammar M;\n" +
                "import S;\n" +
                "A : 'a' {System.out.println(\"M.A\");} ;\n" +
                "WS : (' '|'\\n') {skip();} ;\n";
            writeFile( tmpdir, "M.g", master );

            ErrorQueue equeue = new ErrorQueue();
            ErrorManager.SetErrorListener( equeue );
            AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } );
            CompositeGrammar composite = new CompositeGrammar();
            Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite );
            composite.SetDelegationRoot( g );
            g.ParseAndBuildAST();
            composite.AssignTokenTypes();
            composite.DefineGrammarSymbols();
            composite.CreateNFAs();
            g.CreateLookaheadDFAs( false );

            // predict only alts from M not S
            string expectingDFA =
                ".s0-'a'->.s1\n" +
                ".s0-{'\\n', ' '}->:s3=>2\n" +
                ".s1-<EOT>->:s2=>1\n";
            Antlr3.Analysis.DFA dfa = g.GetLookaheadDFA( 1 );
            FASerializer serializer = new FASerializer( g );
            string result = serializer.Serialize( dfa.startState );
            assertEquals( expectingDFA, result );

            // must not be a "unreachable alt: Tokens" error
            assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
        }
        public void TestBigTreeOfImports()
        {
            ErrorQueue equeue = new ErrorQueue();
            ErrorManager.SetErrorListener( equeue );
            string slave =
                "parser grammar T;\n" +
                "x : T ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "T.g", slave );
            slave =
                "parser grammar S;\n" +
                "import T;\n" +
                "y : S ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "S.g", slave );

            slave =
                "parser grammar C;\n" +
                "i : C ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "C.g", slave );
            slave =
                "parser grammar B;\n" +
                "j : B ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "B.g", slave );
            slave =
                "parser grammar A;\n" +
                "import B,C;\n" +
                "k : A ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "A.g", slave );

            string master =
                "grammar M;\n" +
                "import S,A;\n" +
                "a : M ;\n";
            writeFile( tmpdir, "M.g", master );
            AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } );
            CompositeGrammar composite = new CompositeGrammar();
            Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite );
            composite.SetDelegationRoot( g );
            g.ParseAndBuildAST();
            g.composite.AssignTokenTypes();
            g.composite.DefineGrammarSymbols();

            string expectedTokenIDToTypeMap = "[A=4, B=5, C=6, M=7, S=8, T=9]";
            string expectedStringLiteralToTypeMap = "{}";
            string expectedTypeToTokenList = "[A, B, C, M, S, T]";

            Assert.AreEqual( expectedTokenIDToTypeMap,
                         realElements( g.composite.TokenIDToTypeMap ).ToElementString() );
            Assert.AreEqual( expectedStringLiteralToTypeMap, g.composite.StringLiteralToTypeMap.ToElementString() );
            Assert.AreEqual( expectedTypeToTokenList,
                         realElements( g.composite.TypeToTokenList ).ToElementString() );

            Assert.AreEqual(0, equeue.errors.Count, "unexpected errors: " + equeue);

            bool ok =
                rawGenerateAndBuildRecognizer( "M.g", master, "MParser", null, false );
            bool expecting = true; // should be ok
            Assert.AreEqual( expecting, ok );
        }
        public void TestInvalidImportMechanism()
        {
            // M.Tokens has nothing to predict tokens from S.  Should
            // not include S.Tokens alt in this case?
            string slave =
                "lexer grammar S;\n" +
                "A : 'a' {System.out.println(\"S.A\");} ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "S.g", slave );
            string master =
                "tree grammar M;\n" +
                "import S;\n" +
                "a : A ;";
            writeFile( tmpdir, "M.g", master );

            ErrorQueue equeue = new ErrorQueue();
            ErrorManager.SetErrorListener( equeue );
            AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } );
            CompositeGrammar composite = new CompositeGrammar();
            Grammar g = new Grammar( antlr, System.IO.Path.Combine( tmpdir, "M.g" ), composite );
            composite.SetDelegationRoot( g );
            g.ParseAndBuildAST();

            assertEquals( "unexpected errors: " + equeue, 1, equeue.errors.Count );
            assertEquals( "unexpected errors: " + equeue, 0, equeue.warnings.Count );

            string expectedError =
                "error(161): " + tmpdir.ToString().replaceFirst( "\\-[0-9]+", "" ) + "\\M.g:2:8: tree grammar M cannot import lexer grammar S";
            assertEquals( expectedError, equeue.errors[0].ToString().replaceFirst( "\\-[0-9]+", "" ) );
        }
        public void TestImportedTokenVocabWorksInRoot()
        {
            Assert.Inconclusive( "May be failing on just my port..." );
            ErrorQueue equeue = new ErrorQueue();
            ErrorManager.SetErrorListener( equeue );
            string slave =
                "parser grammar S;\n" +
                "tokens { A='a'; }\n" +
                "x : A {System.out.println(\"S.x\");} ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "S.g", slave );

            string tokens =
                "A=99\n";
            writeFile( tmpdir, "Test.tokens", tokens );

            string master =
                "grammar M;\n" +
                "options {tokenVocab=Test;}\n" +
                "import S;\n" +
                "s : x ;\n" +
                "WS : (' '|'\\n') {skip();} ;\n";
            writeFile( tmpdir, "M.g", master );
            AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } );
            CompositeGrammar composite = new CompositeGrammar();
            Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite );
            composite.SetDelegationRoot( g );
            g.ParseAndBuildAST();
            g.composite.AssignTokenTypes();

            string expectedTokenIDToTypeMap = "[A=99, WS=101]";
            string expectedStringLiteralToTypeMap = "{'a'=100}";
            string expectedTypeToTokenList = "[A, 'a', WS]";

            assertEquals( expectedTokenIDToTypeMap,
                         realElements( g.composite.tokenIDToTypeMap ).ToElementString() );
            assertEquals( expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.ToString() );
            assertEquals( expectedTypeToTokenList,
                         realElements( g.composite.typeToTokenList ).ToElementString() );

            assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
        }
        public void TestImportedTokenVocabIgnoredWithWarning()
        {
            ErrorQueue equeue = new ErrorQueue();
            ErrorManager.SetErrorListener( equeue );
            string slave =
                "parser grammar S;\n" +
                "options {tokenVocab=whatever;}\n" +
                "tokens { A='a'; }\n" +
                "x : A {System.out.println(\"S.x\");} ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "S.g", slave );

            string master =
                "grammar M;\n" +
                "import S;\n" +
                "s : x ;\n" +
                "WS : (' '|'\\n') {skip();} ;\n";
            writeFile( tmpdir, "M.g", master );
            AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } );
            CompositeGrammar composite = new CompositeGrammar();
            Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite );
            composite.SetDelegationRoot( g );
            g.ParseAndBuildAST();
            g.composite.AssignTokenTypes();

            object expectedArg = "S";
            int expectedMsgID = ErrorManager.MSG_TOKEN_VOCAB_IN_DELEGATE;
            GrammarSemanticsMessage expectedMessage =
                new GrammarSemanticsMessage( expectedMsgID, g, null, expectedArg );
            checkGrammarSemanticsWarning( equeue, expectedMessage );

            assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
            assertEquals( "unexpected errors: " + equeue, 1, equeue.warnings.Count );

            string expectedError =
                "warning(160): S.g:2:10: tokenVocab option ignored in imported grammar S";
            assertEquals( expectedError, equeue.warnings[0].ToString() );
        }
        public void TestDelegatesSeeSameTokenType2()
        {
            ErrorQueue equeue = new ErrorQueue();
            ErrorManager.SetErrorListener( equeue );
            string slave =
                "parser grammar S;\n" + // A, B, C token type order
                "tokens { A; B; C; }\n" +
                "x : A {System.out.println(\"S.x\");} ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "S.g", slave );
            string slave2 =
                "parser grammar T;\n" +
                "tokens { C; B; A; }\n" + // reverse order
                "y : A {System.out.println(\"T.y\");} ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "T.g", slave2 );

            string master =
                "grammar M;\n" +
                "import S,T;\n" +
                "s : x y ;\n" + // matches AA, which should be "aa"
                "B : 'b' ;\n" + // another order: B, A, C
                "A : 'a' ;\n" +
                "C : 'c' ;\n" +
                "WS : (' '|'\\n') {skip();} ;\n";
            writeFile( tmpdir, "M.g", master );
            AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } );
            CompositeGrammar composite = new CompositeGrammar();
            Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite );
            composite.SetDelegationRoot( g );
            g.ParseAndBuildAST();
            g.composite.AssignTokenTypes();

            string expectedTokenIDToTypeMap = "[A=4, B=5, C=6, WS=7]";
            string expectedStringLiteralToTypeMap = "{}";
            string expectedTypeToTokenList = "[A, B, C, WS]";

            assertEquals( expectedTokenIDToTypeMap,
                         realElements( g.composite.tokenIDToTypeMap ).ToElementString() );
            assertEquals( expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.ToElementString() );
            assertEquals( expectedTypeToTokenList,
                         realElements( g.composite.typeToTokenList ).ToElementString() );

            assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
        }
        public void TestCombinedImportsCombined()
        {
            //Assert.Inconclusive( "May be failing on just my port..." );
            // for now, we don't allow combined to import combined
            ErrorQueue equeue = new ErrorQueue();
            ErrorManager.SetErrorListener( equeue );
            string slave =
                "grammar S;\n" + // A, B, C token type order
                "tokens { A; B; C; }\n" +
                "x : 'x' INT {System.out.println(\"S.x\");} ;\n" +
                "INT : '0'..'9'+ ;\n" +
                "WS : (' '|'\\n') {skip();} ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "S.g", slave );

            string master =
                "grammar M;\n" +
                "import S;\n" +
                "s : x INT ;\n";
            writeFile( tmpdir, "M.g", master );
            AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } );
            CompositeGrammar composite = new CompositeGrammar();
            Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite );
            composite.SetDelegationRoot( g );
            g.ParseAndBuildAST();
            g.composite.AssignTokenTypes();

            assertEquals( "unexpected errors: " + equeue, 1, equeue.errors.Count );
            string expectedError = "error(161): " + tmpdir.ToString().replaceFirst( "\\-[0-9]+", "" ) + "/M.g:2:8: combined grammar M cannot import combined grammar S";
            assertEquals( "unexpected errors: " + equeue, expectedError, equeue.errors[0].ToString().replaceFirst( "\\-[0-9]+", "" ) );
        }
        public void TestSameNameTwoStrings()
        {
            ErrorQueue equeue = new ErrorQueue();
            ErrorManager.SetErrorListener( equeue );
            string slave =
                "parser grammar S;\n" +
                "tokens { A='a'; }\n" +
                "x : A {System.out.println(\"S.x\");} ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "S.g", slave );
            string slave2 =
                "parser grammar T;\n" +
                "tokens { A='x'; }\n" +
                "y : A {System.out.println(\"T.y\");} ;\n";
            writeFile( tmpdir, "T.g", slave2 );

            string master =
                "grammar M;\n" +
                "import S,T;\n" +
                "s : x y ;\n" +
                "WS : (' '|'\\n') {skip();} ;\n";
            writeFile( tmpdir, "M.g", master );
            AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } );
            CompositeGrammar composite = new CompositeGrammar();
            Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite );
            composite.SetDelegationRoot( g );
            g.ParseAndBuildAST();
            g.composite.AssignTokenTypes();

            string expectedTokenIDToTypeMap = "[A=4, T__6=6, WS=5]";
            string expectedStringLiteralToTypeMap = "{'a'=4, 'x'=6}";
            string expectedTypeToTokenList = "[A, WS, T__6]";

            Assert.AreEqual( expectedTokenIDToTypeMap,
                         realElements( g.composite.TokenIDToTypeMap ).ToElementString() );
            Assert.AreEqual( expectedStringLiteralToTypeMap, sortMapToString( g.composite.StringLiteralToTypeMap ) );
            Assert.AreEqual( expectedTypeToTokenList,
                         realElements( g.composite.TypeToTokenList ).ToElementString() );

            object expectedArg = "A='x'";
            object expectedArg2 = "'a'";
            int expectedMsgID = ErrorManager.MSG_TOKEN_ALIAS_REASSIGNMENT;
            GrammarSemanticsMessage expectedMessage =
                new GrammarSemanticsMessage( expectedMsgID, g, null, expectedArg, expectedArg2 );
            checkGrammarSemanticsError( equeue, expectedMessage );

            Assert.AreEqual(1, equeue.errors.Count, "unexpected errors: " + equeue);

            string expectedError =
                "error(159): T.g:2:10: cannot alias A='x'; token name already assigned to 'a'";
            Assert.AreEqual( expectedError, equeue.errors[0].ToString() );
        }
        public void TestNestedComposite()
        {
            // Wasn't compiling. http://www.antlr.org/jira/browse/ANTLR-438
            ErrorQueue equeue = new ErrorQueue();
            ErrorManager.SetErrorListener(equeue);
            string gstr =
                "lexer grammar L;\n" +
                "T1: '1';\n" +
                "T2: '2';\n" +
                "T3: '3';\n" +
                "T4: '4';\n";
            mkdir(tmpdir);
            writeFile(tmpdir, "L.g", gstr);
            gstr =
                "parser grammar G1;\n" +
                "s: a | b;\n" +
                "a: T1;\n" +
                "b: T2;\n";
            mkdir(tmpdir);
            writeFile(tmpdir, "G1.g", gstr);

            gstr =
                "parser grammar G2;\n" +
                "import G1;\n" +
                "a: T3;\n";
            mkdir(tmpdir);
            writeFile(tmpdir, "G2.g", gstr);
            string G3str =
                "grammar G3;\n" +
                "import G2;\n" +
                "b: T4;\n";
            mkdir(tmpdir);
            writeFile(tmpdir, "G3.g", G3str);

            AntlrTool antlr = newTool(new string[] { "-lib", tmpdir });
            CompositeGrammar composite = new CompositeGrammar();
            Grammar g = new Grammar(antlr, tmpdir + "/G3.g", composite);
            composite.SetDelegationRoot(g);
            g.ParseAndBuildAST();
            g.composite.AssignTokenTypes();
            g.composite.DefineGrammarSymbols();

            string expectedTokenIDToTypeMap = "[T1=4, T2=5, T3=6, T4=7]";
            string expectedStringLiteralToTypeMap = "{}";
            string expectedTypeToTokenList = "[T1, T2, T3, T4]";

            Assert.AreEqual(expectedTokenIDToTypeMap, realElements(g.composite.TokenIDToTypeMap).ToElementString());
            Assert.AreEqual(expectedStringLiteralToTypeMap, g.composite.StringLiteralToTypeMap.ToElementString());
            Assert.AreEqual(expectedTypeToTokenList, realElements(g.composite.TypeToTokenList).ToElementString());

            Assert.AreEqual(0, equeue.errors.Count, "unexpected errors: " + equeue);

            bool ok =
                rawGenerateAndBuildRecognizer("G3.g", G3str, "G3Parser", null, false);
            bool expecting = true; // should be ok
            Assert.AreEqual(expecting, ok);
        }
Beispiel #16
0
        /** Get a grammar mentioned on the command-line and any delegates */
        public virtual Grammar GetRootGrammar( string grammarFileName )
        {
            //StringTemplate.setLintMode(true);
            // grammars mentioned on command line are either roots or single grammars.
            // create the necessary composite in case it's got delegates; even
            // single grammar needs it to get token types.
            CompositeGrammar composite = new CompositeGrammar();
            Grammar grammar = new Grammar( this, grammarFileName, composite );
            if ( TestMode )
                grammar.DefaultRuleModifier = "public";
            composite.SetDelegationRoot( grammar );

            string f = null;

            if ( haveInputDir )
            {
                f = Path.Combine( inputDirectory, grammarFileName );
            }
            else
            {
                f = grammarFileName;
            }

            // Store the location of this grammar as if we import files, we can then
            // search for imports in the same location as the original grammar as well as in
            // the lib directory.
            //
            parentGrammarDirectory = Path.GetDirectoryName( f );

            if ( grammarFileName.LastIndexOfAny( new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar } ) == -1 )
            {
                grammarOutputDirectory = ".";
            }
            else
            {
                grammarOutputDirectory = grammarFileName.Substring( 0, grammarFileName.LastIndexOfAny( new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar } ) );
            }

            StringReader reader = new StringReader( System.IO.File.ReadAllText( f ) );
            grammar.ParseAndBuildAST( reader );
            composite.WatchNFAConversion = internalOption_watchNFAConversion;
            return grammar;
        }
Beispiel #17
0
        public virtual void Process()
        {
            bool exceptionWhenWritingLexerFile = false;
            string lexerGrammarFileName = null;		// necessary at this scope to have access in the catch below

            Stopwatch timer = Stopwatch.StartNew();

            // Have to be tricky here when Maven or build tools call in and must new Tool()
            // before setting options. The banner won't display that way!
            if ( Verbose && showBanner )
            {
                ErrorManager.Info( "ANTLR Parser Generator  Version " + AssemblyInformationalVersion );
                showBanner = false;
            }

            try
            {
                SortGrammarFiles(); // update grammarFileNames
            }
            catch ( Exception e )
            {
                ErrorManager.Error( ErrorManager.MSG_INTERNAL_ERROR, e );
            }

            foreach ( string grammarFileName in GrammarFileNames )
            {
                // If we are in make mode (to support build tools like Maven) and the
                // file is already up to date, then we do not build it (and in verbose mode
                // we will say so).
                if ( Make )
                {
                    try
                    {
                        if ( !BuildRequired( grammarFileName ) )
                            continue;
                    }
                    catch ( Exception e )
                    {
                        ErrorManager.Error( ErrorManager.MSG_INTERNAL_ERROR, e );
                    }
                }

                if ( Verbose && !Depend )
                {
                    Console.Out.WriteLine( grammarFileName );
                }
                try
                {
                    if ( Depend )
                    {
                        BuildDependencyGenerator dep = new BuildDependencyGenerator( this, grammarFileName );
            #if false
                        IList<string> outputFiles = dep.getGeneratedFileList();
                        IList<string> dependents = dep.getDependenciesFileList();
                        Console.Out.WriteLine( "output: " + outputFiles );
                        Console.Out.WriteLine( "dependents: " + dependents );
            #endif
                        Console.Out.WriteLine( dep.GetDependencies().Render() );
                        continue;
                    }

                    Grammar rootGrammar = GetRootGrammar( grammarFileName );
                    // we now have all grammars read in as ASTs
                    // (i.e., root and all delegates)
                    rootGrammar.composite.AssignTokenTypes();
                    //rootGrammar.composite.TranslateLeftRecursiveRules();
                    rootGrammar.AddRulesForSyntacticPredicates();
                    rootGrammar.composite.DefineGrammarSymbols();
                    rootGrammar.composite.CreateNFAs();

                    GenerateRecognizer( rootGrammar );

                    if ( PrintGrammar )
                    {
                        rootGrammar.PrintGrammar( Console.Out );
                    }

                    if (Report)
                    {
                        GrammarReport2 greport = new GrammarReport2(rootGrammar);
                        Console.WriteLine(greport.ToString());
                    }

                    if ( Profile )
                    {
                        GrammarReport report = new GrammarReport(rootGrammar);
                        Stats.WriteReport( GrammarReport.GRAMMAR_STATS_FILENAME,
                                          report.ToNotifyString() );
                    }

                    // now handle the lexer if one was created for a merged spec
                    string lexerGrammarStr = rootGrammar.GetLexerGrammar();
                    //[email protected]("lexer grammar:\n"+lexerGrammarStr);
                    if ( rootGrammar.type == GrammarType.Combined && lexerGrammarStr != null )
                    {
                        lexerGrammarFileName = rootGrammar.ImplicitlyGeneratedLexerFileName;
                        try
                        {
                            TextWriter w = GetOutputFile( rootGrammar, lexerGrammarFileName );
                            w.Write( lexerGrammarStr );
                            w.Close();
                        }
                        catch (IOException)
                        {
                            // emit different error message when creating the implicit lexer fails
                            // due to write permission error
                            exceptionWhenWritingLexerFile = true;
                            throw;
                        }
                        try
                        {
                            StringReader sr = new StringReader( lexerGrammarStr );
                            Grammar lexerGrammar = new Grammar(this);
                            lexerGrammar.composite.WatchNFAConversion = internalOption_watchNFAConversion;
                            lexerGrammar.implicitLexer = true;
                            if ( TestMode )
                                lexerGrammar.DefaultRuleModifier = "public";
                            FileInfo lexerGrammarFullFile = new FileInfo( System.IO.Path.Combine( GetFileDirectory( lexerGrammarFileName ), lexerGrammarFileName ) );
                            lexerGrammar.FileName = lexerGrammarFullFile.ToString();

                            lexerGrammar.ImportTokenVocabulary( rootGrammar );
                            lexerGrammar.ParseAndBuildAST( sr );

                            sr.Close();

                            lexerGrammar.composite.AssignTokenTypes();
                            lexerGrammar.AddRulesForSyntacticPredicates();
                            lexerGrammar.composite.DefineGrammarSymbols();
                            lexerGrammar.composite.CreateNFAs();

                            GenerateRecognizer( lexerGrammar );
                        }
                        finally
                        {
                            // make sure we clean up
                            if ( deleteTempLexer )
                            {
                                System.IO.DirectoryInfo outputDir = GetOutputDirectory( lexerGrammarFileName );
                                FileInfo outputFile = new FileInfo( System.IO.Path.Combine( outputDir.FullName, lexerGrammarFileName ) );
                                outputFile.Delete();
                            }
                        }
                    }
                }
                catch ( IOException e )
                {
                    if ( exceptionWhenWritingLexerFile )
                    {
                        ErrorManager.Error( ErrorManager.MSG_CANNOT_WRITE_FILE, e );
                    }
                    else
                    {
                        ErrorManager.Error( ErrorManager.MSG_CANNOT_OPEN_FILE,
                                           grammarFileName, e );
                    }
                }
                catch ( Exception e )
                {
                    ErrorManager.Error( ErrorManager.MSG_INTERNAL_ERROR, grammarFileName, e );
                }
            #if false
                finally
                {
                    Console.Out.WriteLine( "creates=" + Interval.creates );
                    Console.Out.WriteLine( "hits=" + Interval.hits );
                    Console.Out.WriteLine( "misses=" + Interval.misses );
                    Console.Out.WriteLine( "outOfRange=" + Interval.outOfRange );
                }
            #endif
            }

            if (_showTimer)
            {
                Console.WriteLine("Total parse time: {0}ms", timer.ElapsedMilliseconds);
            }
        }
        public void TestTokenVocabStringUsedInLexer()
        {
            ErrorQueue equeue = new ErrorQueue();
            ErrorManager.SetErrorListener( equeue );
            string tokens =
                "';'=4\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "T.tokens", tokens );

            string importer =
                "lexer grammar B; \n" +
                "options\t{tokenVocab=T;} \n" +
                "SEMI:';' ; \n";
            writeFile( tmpdir, "B.g", importer );
            AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } );
            CompositeGrammar composite = new CompositeGrammar();
            Grammar g = new Grammar( antlr, tmpdir + "/B.g", composite );
            g.ParseAndBuildAST();
            g.composite.AssignTokenTypes();

            string expectedTokenIDToTypeMap = "[SEMI=4]";
            string expectedStringLiteralToTypeMap = "{';'=4}";
            string expectedTypeToTokenList = "[SEMI]";

            Assert.AreEqual( expectedTokenIDToTypeMap,
                         realElements( g.composite.TokenIDToTypeMap ).ToElementString() );
            Assert.AreEqual( expectedStringLiteralToTypeMap, g.composite.StringLiteralToTypeMap.ToElementString() );
            Assert.AreEqual( expectedTypeToTokenList,
                         realElements( g.composite.TypeToTokenList ).ToElementString() );

            Assert.AreEqual(0, equeue.errors.Count, "unexpected errors: " + equeue);
        }