public void Test3LevelImport() { ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); string slave = "parser grammar T;\n" + "a : T ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "T.g", slave ); string slave2 = "parser grammar S;\n" + // A, B, C token type order "import T;\n" + "a : S ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "S.g", slave2 ); string master = "grammar M;\n" + "import S;\n" + "a : M ;\n"; writeFile( tmpdir, "M.g", master ); AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } ); CompositeGrammar composite = new CompositeGrammar(); Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite ); composite.SetDelegationRoot( g ); g.ParseAndBuildAST(); g.composite.AssignTokenTypes(); g.composite.DefineGrammarSymbols(); string expectedTokenIDToTypeMap = "[M=6, S=5, T=4]"; string expectedStringLiteralToTypeMap = "{}"; string expectedTypeToTokenList = "[T, S, M]"; assertEquals( expectedTokenIDToTypeMap, realElements( g.composite.tokenIDToTypeMap ).ToElementString() ); assertEquals( expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.ToElementString() ); assertEquals( expectedTypeToTokenList, realElements( g.composite.typeToTokenList ).ToElementString() ); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); bool ok = rawGenerateAndBuildRecognizer( "M.g", master, "MParser", null, false ); bool expecting = true; // should be ok assertEquals( expecting, ok ); }
public void TestArgsOnTokenInLexerRuleOfCombined() { ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); Grammar g = new Grammar( "grammar t;\n" + "a : R;\n" + "R : 'z' ID[32] ;\n" + "ID : 'a';\n" ); string lexerGrammarStr = g.GetLexerGrammar(); System.IO.StringReader sr = new System.IO.StringReader( lexerGrammarStr ); Grammar lexerGrammar = new Grammar(); lexerGrammar.FileName = "<internally-generated-lexer>"; lexerGrammar.ImportTokenVocabulary( g ); lexerGrammar.ParseAndBuildAST( sr ); lexerGrammar.DefineGrammarSymbols(); lexerGrammar.CheckNameSpaceAndActions(); sr.Close(); AntlrTool antlr = newTool(); antlr.SetOutputDirectory( null ); // write to /dev/null CodeGenerator generator = new CodeGenerator( antlr, lexerGrammar, "Java" ); lexerGrammar.CodeGenerator = generator; generator.GenRecognizer(); int expectedMsgID = ErrorManager.MSG_RULE_HAS_NO_ARGS; object expectedArg = "ID"; object expectedArg2 = null; GrammarSemanticsMessage expectedMessage = new GrammarSemanticsMessage( expectedMsgID, lexerGrammar, null, expectedArg, expectedArg2 ); checkError( equeue, expectedMessage ); }
/** Import the rules/tokens of a delegate grammar. All delegate grammars are * read during the ctor of first Grammar created. * * Do not create NFA here because NFA construction needs to hook up with * overridden rules in delegation root grammar. */ public virtual void ImportGrammar( IToken grammarNameToken, string label ) { string grammarName = grammarNameToken.Text; //[email protected]("import "+gfile.getName()); string gname = grammarName + GrammarFileExtension; TextReader br = null; try { string fullName = Tool.GetLibraryFile( gname ); if (!File.Exists(fullName)) { gname = grammarName + AltGrammarFileExtension; fullName = Tool.GetLibraryFile(gname); } //FileReader fr = new FileReader( fullName ); //br = new BufferedReader( fr ); br = new StringReader( System.IO.File.ReadAllText( fullName ) ); Grammar delegateGrammar = null; delegateGrammar = new Grammar( Tool, gname, composite ); delegateGrammar.label = label; AddDelegateGrammar( delegateGrammar ); delegateGrammar.ParseAndBuildAST( br ); delegateGrammar.AddRulesForSyntacticPredicates(); if ( !ValidImport( delegateGrammar ) ) { ErrorManager.GrammarError( ErrorManager.MSG_INVALID_IMPORT, this, grammarNameToken, this, delegateGrammar ); return; } if ( this.type == GrammarType.Combined && ( delegateGrammar.name.Equals( this.name + grammarTypeToFileNameSuffix[(int)GrammarType.Lexer] ) || delegateGrammar.name.Equals( this.name + grammarTypeToFileNameSuffix[(int)GrammarType.Parser] ) ) ) { ErrorManager.GrammarError( ErrorManager.MSG_IMPORT_NAME_CLASH, this, grammarNameToken, this, delegateGrammar ); return; } if ( delegateGrammar.grammarTree != null ) { // we have a valid grammar // deal with combined grammars if ( delegateGrammar.type == GrammarType.Lexer && this.type == GrammarType.Combined ) { // ooops, we wasted some effort; tell lexer to read it in // later LexerGrammarTemplate.SetAttribute( "imports", grammarName ); // but, this parser grammar will need the vocab // so add to composite anyway so we suck in the tokens later } } //[email protected]("Got grammar:\n"+delegateGrammar); } catch ( IOException ioe ) { ErrorManager.Error( ErrorManager.MSG_CANNOT_OPEN_FILE, gname, ioe ); } finally { if ( br != null ) { try { br.Close(); } catch ( IOException ioe ) { ErrorManager.Error( ErrorManager.MSG_CANNOT_CLOSE_FILE, gname, ioe ); } } } }
public void TestSyntaxErrorsInImportsNotThrownOut2() { ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); string slave = "parser grammar S;\n" + ": A {System.out.println(\"S.x\");} ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "S.g", slave ); string master = "grammar M;\n" + "import S;\n" + "s : x ;\n" + "WS : (' '|'\\n') {skip();} ;\n"; writeFile( tmpdir, "M.g", master ); AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } ); CompositeGrammar composite = new CompositeGrammar(); Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite ); composite.SetDelegationRoot( g ); g.ParseAndBuildAST(); g.composite.AssignTokenTypes(); // whole bunch of errors from bad S.g file assertEquals( "unexpected errors: " + equeue, 3, equeue.errors.Count ); }
public void TestSameStringTwoNames() { Assert.Inconclusive( "May be failing on just my port..." ); ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); string slave = "parser grammar S;\n" + "tokens { A='a'; }\n" + "x : A {System.out.println(\"S.x\");} ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "S.g", slave ); string slave2 = "parser grammar T;\n" + "tokens { X='a'; }\n" + "y : X {System.out.println(\"T.y\");} ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "T.g", slave2 ); string master = "grammar M;\n" + "import S,T;\n" + "s : x y ;\n" + "WS : (' '|'\\n') {skip();} ;\n"; writeFile( tmpdir, "M.g", master ); AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } ); CompositeGrammar composite = new CompositeGrammar(); Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite ); composite.SetDelegationRoot( g ); g.ParseAndBuildAST(); g.composite.AssignTokenTypes(); string expectedTokenIDToTypeMap = "[A=4, WS=6, X=5]"; string expectedStringLiteralToTypeMap = "{'a'=4}"; string expectedTypeToTokenList = "[A, X, WS]"; assertEquals( expectedTokenIDToTypeMap, realElements( g.composite.tokenIDToTypeMap ).ToElementString() ); assertEquals( expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.ToString() ); assertEquals( expectedTypeToTokenList, realElements( g.composite.typeToTokenList ).ToElementString() ); object expectedArg = "X='a'"; object expectedArg2 = "A"; int expectedMsgID = ErrorManager.MSG_TOKEN_ALIAS_CONFLICT; GrammarSemanticsMessage expectedMessage = new GrammarSemanticsMessage( expectedMsgID, g, null, expectedArg, expectedArg2 ); checkGrammarSemanticsError( equeue, expectedMessage ); assertEquals( "unexpected errors: " + equeue, 1, equeue.errors.Count ); string expectedError = "error(158): T.g:2:10: cannot alias X='a'; string already assigned to A"; assertEquals( expectedError, equeue.errors[0].ToString() ); }
public void TestRulesVisibleThroughMultilevelImport() { ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); string slave = "parser grammar T;\n" + "x : T ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "T.g", slave ); string slave2 = "parser grammar S;\n" + // A, B, C token type order "import T;\n" + "a : S ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "S.g", slave2 ); string master = "grammar M;\n" + "import S;\n" + "a : M x ;\n"; // x MUST BE VISIBLE TO M writeFile( tmpdir, "M.g", master ); AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } ); CompositeGrammar composite = new CompositeGrammar(); Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite ); composite.SetDelegationRoot( g ); g.ParseAndBuildAST(); g.composite.AssignTokenTypes(); g.composite.DefineGrammarSymbols(); string expectedTokenIDToTypeMap = "[M=6, S=5, T=4]"; string expectedStringLiteralToTypeMap = "{}"; string expectedTypeToTokenList = "[T, S, M]"; assertEquals( expectedTokenIDToTypeMap, realElements( g.composite.tokenIDToTypeMap ).ToElementString() ); assertEquals( expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.ToElementString() ); assertEquals( expectedTypeToTokenList, realElements( g.composite.typeToTokenList ).ToElementString() ); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); }
public void TestLexerDelegatorRuleOverridesDelegateLeavingNoRules() { // M.Tokens has nothing to predict tokens from S. Should // not include S.Tokens alt in this case? string slave = "lexer grammar S;\n" + "A : 'a' {System.out.println(\"S.A\");} ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "S.g", slave ); string master = "lexer grammar M;\n" + "import S;\n" + "A : 'a' {System.out.println(\"M.A\");} ;\n" + "WS : (' '|'\\n') {skip();} ;\n"; writeFile( tmpdir, "M.g", master ); ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } ); CompositeGrammar composite = new CompositeGrammar(); Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite ); composite.SetDelegationRoot( g ); g.ParseAndBuildAST(); composite.AssignTokenTypes(); composite.DefineGrammarSymbols(); composite.CreateNFAs(); g.CreateLookaheadDFAs( false ); // predict only alts from M not S string expectingDFA = ".s0-'a'->.s1\n" + ".s0-{'\\n', ' '}->:s3=>2\n" + ".s1-<EOT>->:s2=>1\n"; Antlr3.Analysis.DFA dfa = g.GetLookaheadDFA( 1 ); FASerializer serializer = new FASerializer( g ); string result = serializer.Serialize( dfa.startState ); assertEquals( expectingDFA, result ); // must not be a "unreachable alt: Tokens" error assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); }
public void TestBigTreeOfImports() { ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); string slave = "parser grammar T;\n" + "x : T ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "T.g", slave ); slave = "parser grammar S;\n" + "import T;\n" + "y : S ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "S.g", slave ); slave = "parser grammar C;\n" + "i : C ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "C.g", slave ); slave = "parser grammar B;\n" + "j : B ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "B.g", slave ); slave = "parser grammar A;\n" + "import B,C;\n" + "k : A ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "A.g", slave ); string master = "grammar M;\n" + "import S,A;\n" + "a : M ;\n"; writeFile( tmpdir, "M.g", master ); AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } ); CompositeGrammar composite = new CompositeGrammar(); Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite ); composite.SetDelegationRoot( g ); g.ParseAndBuildAST(); g.composite.AssignTokenTypes(); g.composite.DefineGrammarSymbols(); string expectedTokenIDToTypeMap = "[A=4, B=5, C=6, M=7, S=8, T=9]"; string expectedStringLiteralToTypeMap = "{}"; string expectedTypeToTokenList = "[A, B, C, M, S, T]"; Assert.AreEqual( expectedTokenIDToTypeMap, realElements( g.composite.TokenIDToTypeMap ).ToElementString() ); Assert.AreEqual( expectedStringLiteralToTypeMap, g.composite.StringLiteralToTypeMap.ToElementString() ); Assert.AreEqual( expectedTypeToTokenList, realElements( g.composite.TypeToTokenList ).ToElementString() ); Assert.AreEqual(0, equeue.errors.Count, "unexpected errors: " + equeue); bool ok = rawGenerateAndBuildRecognizer( "M.g", master, "MParser", null, false ); bool expecting = true; // should be ok Assert.AreEqual( expecting, ok ); }
public void TestInvalidImportMechanism() { // M.Tokens has nothing to predict tokens from S. Should // not include S.Tokens alt in this case? string slave = "lexer grammar S;\n" + "A : 'a' {System.out.println(\"S.A\");} ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "S.g", slave ); string master = "tree grammar M;\n" + "import S;\n" + "a : A ;"; writeFile( tmpdir, "M.g", master ); ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } ); CompositeGrammar composite = new CompositeGrammar(); Grammar g = new Grammar( antlr, System.IO.Path.Combine( tmpdir, "M.g" ), composite ); composite.SetDelegationRoot( g ); g.ParseAndBuildAST(); assertEquals( "unexpected errors: " + equeue, 1, equeue.errors.Count ); assertEquals( "unexpected errors: " + equeue, 0, equeue.warnings.Count ); string expectedError = "error(161): " + tmpdir.ToString().replaceFirst( "\\-[0-9]+", "" ) + "\\M.g:2:8: tree grammar M cannot import lexer grammar S"; assertEquals( expectedError, equeue.errors[0].ToString().replaceFirst( "\\-[0-9]+", "" ) ); }
public void TestImportedTokenVocabWorksInRoot() { Assert.Inconclusive( "May be failing on just my port..." ); ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); string slave = "parser grammar S;\n" + "tokens { A='a'; }\n" + "x : A {System.out.println(\"S.x\");} ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "S.g", slave ); string tokens = "A=99\n"; writeFile( tmpdir, "Test.tokens", tokens ); string master = "grammar M;\n" + "options {tokenVocab=Test;}\n" + "import S;\n" + "s : x ;\n" + "WS : (' '|'\\n') {skip();} ;\n"; writeFile( tmpdir, "M.g", master ); AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } ); CompositeGrammar composite = new CompositeGrammar(); Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite ); composite.SetDelegationRoot( g ); g.ParseAndBuildAST(); g.composite.AssignTokenTypes(); string expectedTokenIDToTypeMap = "[A=99, WS=101]"; string expectedStringLiteralToTypeMap = "{'a'=100}"; string expectedTypeToTokenList = "[A, 'a', WS]"; assertEquals( expectedTokenIDToTypeMap, realElements( g.composite.tokenIDToTypeMap ).ToElementString() ); assertEquals( expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.ToString() ); assertEquals( expectedTypeToTokenList, realElements( g.composite.typeToTokenList ).ToElementString() ); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); }
public void TestImportedTokenVocabIgnoredWithWarning() { ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); string slave = "parser grammar S;\n" + "options {tokenVocab=whatever;}\n" + "tokens { A='a'; }\n" + "x : A {System.out.println(\"S.x\");} ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "S.g", slave ); string master = "grammar M;\n" + "import S;\n" + "s : x ;\n" + "WS : (' '|'\\n') {skip();} ;\n"; writeFile( tmpdir, "M.g", master ); AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } ); CompositeGrammar composite = new CompositeGrammar(); Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite ); composite.SetDelegationRoot( g ); g.ParseAndBuildAST(); g.composite.AssignTokenTypes(); object expectedArg = "S"; int expectedMsgID = ErrorManager.MSG_TOKEN_VOCAB_IN_DELEGATE; GrammarSemanticsMessage expectedMessage = new GrammarSemanticsMessage( expectedMsgID, g, null, expectedArg ); checkGrammarSemanticsWarning( equeue, expectedMessage ); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); assertEquals( "unexpected errors: " + equeue, 1, equeue.warnings.Count ); string expectedError = "warning(160): S.g:2:10: tokenVocab option ignored in imported grammar S"; assertEquals( expectedError, equeue.warnings[0].ToString() ); }
public void TestDelegatesSeeSameTokenType2() { ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); string slave = "parser grammar S;\n" + // A, B, C token type order "tokens { A; B; C; }\n" + "x : A {System.out.println(\"S.x\");} ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "S.g", slave ); string slave2 = "parser grammar T;\n" + "tokens { C; B; A; }\n" + // reverse order "y : A {System.out.println(\"T.y\");} ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "T.g", slave2 ); string master = "grammar M;\n" + "import S,T;\n" + "s : x y ;\n" + // matches AA, which should be "aa" "B : 'b' ;\n" + // another order: B, A, C "A : 'a' ;\n" + "C : 'c' ;\n" + "WS : (' '|'\\n') {skip();} ;\n"; writeFile( tmpdir, "M.g", master ); AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } ); CompositeGrammar composite = new CompositeGrammar(); Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite ); composite.SetDelegationRoot( g ); g.ParseAndBuildAST(); g.composite.AssignTokenTypes(); string expectedTokenIDToTypeMap = "[A=4, B=5, C=6, WS=7]"; string expectedStringLiteralToTypeMap = "{}"; string expectedTypeToTokenList = "[A, B, C, WS]"; assertEquals( expectedTokenIDToTypeMap, realElements( g.composite.tokenIDToTypeMap ).ToElementString() ); assertEquals( expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.ToElementString() ); assertEquals( expectedTypeToTokenList, realElements( g.composite.typeToTokenList ).ToElementString() ); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); }
public void TestCombinedImportsCombined() { //Assert.Inconclusive( "May be failing on just my port..." ); // for now, we don't allow combined to import combined ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); string slave = "grammar S;\n" + // A, B, C token type order "tokens { A; B; C; }\n" + "x : 'x' INT {System.out.println(\"S.x\");} ;\n" + "INT : '0'..'9'+ ;\n" + "WS : (' '|'\\n') {skip();} ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "S.g", slave ); string master = "grammar M;\n" + "import S;\n" + "s : x INT ;\n"; writeFile( tmpdir, "M.g", master ); AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } ); CompositeGrammar composite = new CompositeGrammar(); Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite ); composite.SetDelegationRoot( g ); g.ParseAndBuildAST(); g.composite.AssignTokenTypes(); assertEquals( "unexpected errors: " + equeue, 1, equeue.errors.Count ); string expectedError = "error(161): " + tmpdir.ToString().replaceFirst( "\\-[0-9]+", "" ) + "/M.g:2:8: combined grammar M cannot import combined grammar S"; assertEquals( "unexpected errors: " + equeue, expectedError, equeue.errors[0].ToString().replaceFirst( "\\-[0-9]+", "" ) ); }
public void TestSameNameTwoStrings() { ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); string slave = "parser grammar S;\n" + "tokens { A='a'; }\n" + "x : A {System.out.println(\"S.x\");} ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "S.g", slave ); string slave2 = "parser grammar T;\n" + "tokens { A='x'; }\n" + "y : A {System.out.println(\"T.y\");} ;\n"; writeFile( tmpdir, "T.g", slave2 ); string master = "grammar M;\n" + "import S,T;\n" + "s : x y ;\n" + "WS : (' '|'\\n') {skip();} ;\n"; writeFile( tmpdir, "M.g", master ); AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } ); CompositeGrammar composite = new CompositeGrammar(); Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite ); composite.SetDelegationRoot( g ); g.ParseAndBuildAST(); g.composite.AssignTokenTypes(); string expectedTokenIDToTypeMap = "[A=4, T__6=6, WS=5]"; string expectedStringLiteralToTypeMap = "{'a'=4, 'x'=6}"; string expectedTypeToTokenList = "[A, WS, T__6]"; Assert.AreEqual( expectedTokenIDToTypeMap, realElements( g.composite.TokenIDToTypeMap ).ToElementString() ); Assert.AreEqual( expectedStringLiteralToTypeMap, sortMapToString( g.composite.StringLiteralToTypeMap ) ); Assert.AreEqual( expectedTypeToTokenList, realElements( g.composite.TypeToTokenList ).ToElementString() ); object expectedArg = "A='x'"; object expectedArg2 = "'a'"; int expectedMsgID = ErrorManager.MSG_TOKEN_ALIAS_REASSIGNMENT; GrammarSemanticsMessage expectedMessage = new GrammarSemanticsMessage( expectedMsgID, g, null, expectedArg, expectedArg2 ); checkGrammarSemanticsError( equeue, expectedMessage ); Assert.AreEqual(1, equeue.errors.Count, "unexpected errors: " + equeue); string expectedError = "error(159): T.g:2:10: cannot alias A='x'; token name already assigned to 'a'"; Assert.AreEqual( expectedError, equeue.errors[0].ToString() ); }
public void TestNestedComposite() { // Wasn't compiling. http://www.antlr.org/jira/browse/ANTLR-438 ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener(equeue); string gstr = "lexer grammar L;\n" + "T1: '1';\n" + "T2: '2';\n" + "T3: '3';\n" + "T4: '4';\n"; mkdir(tmpdir); writeFile(tmpdir, "L.g", gstr); gstr = "parser grammar G1;\n" + "s: a | b;\n" + "a: T1;\n" + "b: T2;\n"; mkdir(tmpdir); writeFile(tmpdir, "G1.g", gstr); gstr = "parser grammar G2;\n" + "import G1;\n" + "a: T3;\n"; mkdir(tmpdir); writeFile(tmpdir, "G2.g", gstr); string G3str = "grammar G3;\n" + "import G2;\n" + "b: T4;\n"; mkdir(tmpdir); writeFile(tmpdir, "G3.g", G3str); AntlrTool antlr = newTool(new string[] { "-lib", tmpdir }); CompositeGrammar composite = new CompositeGrammar(); Grammar g = new Grammar(antlr, tmpdir + "/G3.g", composite); composite.SetDelegationRoot(g); g.ParseAndBuildAST(); g.composite.AssignTokenTypes(); g.composite.DefineGrammarSymbols(); string expectedTokenIDToTypeMap = "[T1=4, T2=5, T3=6, T4=7]"; string expectedStringLiteralToTypeMap = "{}"; string expectedTypeToTokenList = "[T1, T2, T3, T4]"; Assert.AreEqual(expectedTokenIDToTypeMap, realElements(g.composite.TokenIDToTypeMap).ToElementString()); Assert.AreEqual(expectedStringLiteralToTypeMap, g.composite.StringLiteralToTypeMap.ToElementString()); Assert.AreEqual(expectedTypeToTokenList, realElements(g.composite.TypeToTokenList).ToElementString()); Assert.AreEqual(0, equeue.errors.Count, "unexpected errors: " + equeue); bool ok = rawGenerateAndBuildRecognizer("G3.g", G3str, "G3Parser", null, false); bool expecting = true; // should be ok Assert.AreEqual(expecting, ok); }
/** Get a grammar mentioned on the command-line and any delegates */ public virtual Grammar GetRootGrammar( string grammarFileName ) { //StringTemplate.setLintMode(true); // grammars mentioned on command line are either roots or single grammars. // create the necessary composite in case it's got delegates; even // single grammar needs it to get token types. CompositeGrammar composite = new CompositeGrammar(); Grammar grammar = new Grammar( this, grammarFileName, composite ); if ( TestMode ) grammar.DefaultRuleModifier = "public"; composite.SetDelegationRoot( grammar ); string f = null; if ( haveInputDir ) { f = Path.Combine( inputDirectory, grammarFileName ); } else { f = grammarFileName; } // Store the location of this grammar as if we import files, we can then // search for imports in the same location as the original grammar as well as in // the lib directory. // parentGrammarDirectory = Path.GetDirectoryName( f ); if ( grammarFileName.LastIndexOfAny( new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar } ) == -1 ) { grammarOutputDirectory = "."; } else { grammarOutputDirectory = grammarFileName.Substring( 0, grammarFileName.LastIndexOfAny( new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar } ) ); } StringReader reader = new StringReader( System.IO.File.ReadAllText( f ) ); grammar.ParseAndBuildAST( reader ); composite.WatchNFAConversion = internalOption_watchNFAConversion; return grammar; }
public virtual void Process() { bool exceptionWhenWritingLexerFile = false; string lexerGrammarFileName = null; // necessary at this scope to have access in the catch below Stopwatch timer = Stopwatch.StartNew(); // Have to be tricky here when Maven or build tools call in and must new Tool() // before setting options. The banner won't display that way! if ( Verbose && showBanner ) { ErrorManager.Info( "ANTLR Parser Generator Version " + AssemblyInformationalVersion ); showBanner = false; } try { SortGrammarFiles(); // update grammarFileNames } catch ( Exception e ) { ErrorManager.Error( ErrorManager.MSG_INTERNAL_ERROR, e ); } foreach ( string grammarFileName in GrammarFileNames ) { // If we are in make mode (to support build tools like Maven) and the // file is already up to date, then we do not build it (and in verbose mode // we will say so). if ( Make ) { try { if ( !BuildRequired( grammarFileName ) ) continue; } catch ( Exception e ) { ErrorManager.Error( ErrorManager.MSG_INTERNAL_ERROR, e ); } } if ( Verbose && !Depend ) { Console.Out.WriteLine( grammarFileName ); } try { if ( Depend ) { BuildDependencyGenerator dep = new BuildDependencyGenerator( this, grammarFileName ); #if false IList<string> outputFiles = dep.getGeneratedFileList(); IList<string> dependents = dep.getDependenciesFileList(); Console.Out.WriteLine( "output: " + outputFiles ); Console.Out.WriteLine( "dependents: " + dependents ); #endif Console.Out.WriteLine( dep.GetDependencies().Render() ); continue; } Grammar rootGrammar = GetRootGrammar( grammarFileName ); // we now have all grammars read in as ASTs // (i.e., root and all delegates) rootGrammar.composite.AssignTokenTypes(); //rootGrammar.composite.TranslateLeftRecursiveRules(); rootGrammar.AddRulesForSyntacticPredicates(); rootGrammar.composite.DefineGrammarSymbols(); rootGrammar.composite.CreateNFAs(); GenerateRecognizer( rootGrammar ); if ( PrintGrammar ) { rootGrammar.PrintGrammar( Console.Out ); } if (Report) { GrammarReport2 greport = new GrammarReport2(rootGrammar); Console.WriteLine(greport.ToString()); } if ( Profile ) { GrammarReport report = new GrammarReport(rootGrammar); Stats.WriteReport( GrammarReport.GRAMMAR_STATS_FILENAME, report.ToNotifyString() ); } // now handle the lexer if one was created for a merged spec string lexerGrammarStr = rootGrammar.GetLexerGrammar(); //[email protected]("lexer grammar:\n"+lexerGrammarStr); if ( rootGrammar.type == GrammarType.Combined && lexerGrammarStr != null ) { lexerGrammarFileName = rootGrammar.ImplicitlyGeneratedLexerFileName; try { TextWriter w = GetOutputFile( rootGrammar, lexerGrammarFileName ); w.Write( lexerGrammarStr ); w.Close(); } catch (IOException) { // emit different error message when creating the implicit lexer fails // due to write permission error exceptionWhenWritingLexerFile = true; throw; } try { StringReader sr = new StringReader( lexerGrammarStr ); Grammar lexerGrammar = new Grammar(this); lexerGrammar.composite.WatchNFAConversion = internalOption_watchNFAConversion; lexerGrammar.implicitLexer = true; if ( TestMode ) lexerGrammar.DefaultRuleModifier = "public"; FileInfo lexerGrammarFullFile = new FileInfo( System.IO.Path.Combine( GetFileDirectory( lexerGrammarFileName ), lexerGrammarFileName ) ); lexerGrammar.FileName = lexerGrammarFullFile.ToString(); lexerGrammar.ImportTokenVocabulary( rootGrammar ); lexerGrammar.ParseAndBuildAST( sr ); sr.Close(); lexerGrammar.composite.AssignTokenTypes(); lexerGrammar.AddRulesForSyntacticPredicates(); lexerGrammar.composite.DefineGrammarSymbols(); lexerGrammar.composite.CreateNFAs(); GenerateRecognizer( lexerGrammar ); } finally { // make sure we clean up if ( deleteTempLexer ) { System.IO.DirectoryInfo outputDir = GetOutputDirectory( lexerGrammarFileName ); FileInfo outputFile = new FileInfo( System.IO.Path.Combine( outputDir.FullName, lexerGrammarFileName ) ); outputFile.Delete(); } } } } catch ( IOException e ) { if ( exceptionWhenWritingLexerFile ) { ErrorManager.Error( ErrorManager.MSG_CANNOT_WRITE_FILE, e ); } else { ErrorManager.Error( ErrorManager.MSG_CANNOT_OPEN_FILE, grammarFileName, e ); } } catch ( Exception e ) { ErrorManager.Error( ErrorManager.MSG_INTERNAL_ERROR, grammarFileName, e ); } #if false finally { Console.Out.WriteLine( "creates=" + Interval.creates ); Console.Out.WriteLine( "hits=" + Interval.hits ); Console.Out.WriteLine( "misses=" + Interval.misses ); Console.Out.WriteLine( "outOfRange=" + Interval.outOfRange ); } #endif } if (_showTimer) { Console.WriteLine("Total parse time: {0}ms", timer.ElapsedMilliseconds); } }
public void TestTokenVocabStringUsedInLexer() { ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); string tokens = "';'=4\n"; mkdir( tmpdir ); writeFile( tmpdir, "T.tokens", tokens ); string importer = "lexer grammar B; \n" + "options\t{tokenVocab=T;} \n" + "SEMI:';' ; \n"; writeFile( tmpdir, "B.g", importer ); AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } ); CompositeGrammar composite = new CompositeGrammar(); Grammar g = new Grammar( antlr, tmpdir + "/B.g", composite ); g.ParseAndBuildAST(); g.composite.AssignTokenTypes(); string expectedTokenIDToTypeMap = "[SEMI=4]"; string expectedStringLiteralToTypeMap = "{';'=4}"; string expectedTypeToTokenList = "[SEMI]"; Assert.AreEqual( expectedTokenIDToTypeMap, realElements( g.composite.TokenIDToTypeMap ).ToElementString() ); Assert.AreEqual( expectedStringLiteralToTypeMap, g.composite.StringLiteralToTypeMap.ToElementString() ); Assert.AreEqual( expectedTypeToTokenList, realElements( g.composite.TypeToTokenList ).ToElementString() ); Assert.AreEqual(0, equeue.errors.Count, "unexpected errors: " + equeue); }