public void TestSimpleRangeVersusChar() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a'..'z' '@' | 'k' '$' ;" ); g.CreateLookaheadDFAs(); string expecting = ".s0-'k'->.s1" + NewLine + ".s0-{'a'..'j', 'l'..'z'}->:s2=>1" + NewLine + ".s1-'$'->:s3=>2" + NewLine + ".s1-'@'->:s2=>1" + NewLine; checkDecision( g, 1, expecting, null ); }
//throws Exception protected void checkDecision( Grammar g, int decision, string expecting, int[] expectingUnreachableAlts ) { Antlr3.AntlrTool tool = new Antlr3.AntlrTool(); // mimic actions of org.antlr.Tool first time for grammar g if ( g.CodeGenerator == null ) { CodeGenerator generator = new CodeGenerator( tool, g, "Java" ); g.CodeGenerator = generator; g.BuildNFA(); g.CreateLookaheadDFAs( false ); } DFA dfa = g.GetLookaheadDFA( decision ); Assert.IsNotNull(dfa, "unknown decision #" + decision); FASerializer serializer = new FASerializer( g ); string result = serializer.Serialize( dfa.StartState ); //System.out.print(result); var nonDetAlts = dfa.UnreachableAlts; //System.out.println("alts w/o predict state="+nonDetAlts); // first make sure nondeterministic alts are as expected if ( expectingUnreachableAlts == null ) { if ( nonDetAlts != null && nonDetAlts.Count != 0 ) { Console.Error.WriteLine( "nondeterministic alts (should be empty): " + ( (IList)nonDetAlts ).ToElementString() ); } Assert.AreEqual(0, nonDetAlts != null ? nonDetAlts.Count : 0, "unreachable alts mismatch"); } else { for ( int i = 0; i < expectingUnreachableAlts.Length; i++ ) { Assert.IsTrue(nonDetAlts != null ? nonDetAlts.Contains(expectingUnreachableAlts[i]) : false, "unreachable alts mismatch"); } } Assert.AreEqual( expecting, result ); }
public void TestRecursive() { // this is cool because the 3rd alt includes !(all other possibilities) Grammar g = new Grammar( "lexer grammar duh;\n" + "SUBTEMPLATE\n" + " : '{'\n" + " ( SUBTEMPLATE\n" + " | ESC\n" + " | ~('}'|'\\\\'|'{')\n" + " )*\n" + " '}'\n" + " ;\n" + "fragment\n" + "ESC : '\\\\' . ;" ); g.CreateLookaheadDFAs(); string expecting = ".s0-'\\\\'->:s2=>2" + NewLine + ".s0-'{'->:s1=>1" + NewLine + ".s0-'}'->:s4=>4" + NewLine + ".s0-{'\\u0000'..'[', ']'..'z', '|', '~'..'\\uFFFF'}->:s3=>3" + NewLine; checkDecision( g, 1, expecting, null ); }
public void TestRecursive2() { // this is also cool because it resolves \\ to be ESC alt; it's just // less efficient of a DFA Grammar g = new Grammar( "lexer grammar duh;\n" + "SUBTEMPLATE\n" + " : '{'\n" + " ( SUBTEMPLATE\n" + " | ESC\n" + " | ~('}'|'{')\n" + " )*\n" + " '}'\n" + " ;\n" + "fragment\n" + "ESC : '\\\\' . ;" ); g.CreateLookaheadDFAs(); string expecting = ".s0-'\\\\'->.s3" + NewLine + ".s0-'{'->:s2=>1" + NewLine + ".s0-'}'->:s1=>4" + NewLine + ".s0-{'\\u0000'..'[', ']'..'z', '|', '~'..'\\uFFFF'}->:s5=>3" + NewLine + ".s3-'\\\\'->:s8=>2" + NewLine + ".s3-'{'->:s7=>2" + NewLine + ".s3-'}'->.s4" + NewLine + ".s3-{'\\u0000'..'[', ']'..'z', '|', '~'..'\\uFFFF'}->:s6=>2" + NewLine + ".s4-'\\u0000'..'\\uFFFF'->:s6=>2" + NewLine + ".s4-<EOT>->:s5=>3" + NewLine; checkDecision( g, 1, expecting, null ); }
public void TestNotTokenInLexer() { Grammar g = new Grammar( "lexer grammar T;\n" + "A : 'x' ('a' | ~B {;}) ;\n" + "B : 'a' ;\n" ); g.CreateLookaheadDFAs(); string expecting = ".s0-'a'->:s1=>1" + NewLine + ".s0-{'\\u0000'..'`', 'b'..'\\uFFFF'}->:s2=>2" + NewLine; checkDecision( g, 1, expecting, null ); }
public void TestRangeWithDisjointSet() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a'..'z' '@'\n" + " | ('k'|'9'|'p') '$'\n" + " ;\n" ); g.CreateLookaheadDFAs(); // must break up a..z into {'a'..'j', 'l'..'o', 'q'..'z'} string expecting = ".s0-'9'->:s3=>2" + NewLine + ".s0-{'a'..'j', 'l'..'o', 'q'..'z'}->:s2=>1" + NewLine + ".s0-{'k', 'p'}->.s1" + NewLine + ".s1-'$'->:s3=>2" + NewLine + ".s1-'@'->:s2=>1" + NewLine; checkDecision( g, 1, expecting, null ); }
public void TestDisjointSetCollidingWithTwoRanges() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : ('a'..'z'|'0'..'9') '@'\n" + " | ('k'|'9'|'p') '$'\n" + " ;\n" ); g.CreateLookaheadDFAs( false ); // must break up a..z into {'a'..'j', 'l'..'o', 'q'..'z'} and 0..9 // into 0..8 string expecting = ".s0-{'0'..'8', 'a'..'j', 'l'..'o', 'q'..'z'}->:s2=>1" + NewLine + ".s0-{'9', 'k', 'p'}->.s1" + NewLine + ".s1-'$'->:s3=>2" + NewLine + ".s1-'@'->:s2=>1" + NewLine; checkDecision( g, 1, expecting, null ); }
public void TestNotSetFragmentInLexer() { Grammar g = new Grammar( "lexer grammar T;\n" + "A : B | ~B {;} ;\n" + "fragment B : 'a'|'b' ;\n" ); g.CreateLookaheadDFAs(); string expecting = ".s0-'a'..'b'->:s1=>1" + NewLine + ".s0-{'\\u0000'..'`', 'c'..'\\uFFFF'}->:s2=>2" + NewLine; checkDecision( g, 1, expecting, null ); }
public void TestPredsUsedAfterRecursionOverflow() { // analysis must bail out due to non-LL(*) nature (ovf) // retries with k=1 (but with LL(*) algorithm not optimized version // as it has preds) Grammar g = new Grammar( "parser grammar P;\n" + "s : {p1}? e '.' | {p2}? e ':' ;\n" + "e : '(' e ')' | INT ;\n" ); string expecting = ".s0-'('->.s1" + NewLine + ".s0-INT->.s4" + NewLine + ".s1-{p1}?->:s2=>1" + NewLine + ".s1-{p2}?->:s3=>2" + NewLine + ".s4-{p1}?->:s2=>1" + NewLine + ".s4-{p2}?->:s3=>2" + NewLine; DecisionProbe.verbose = true; // make sure we get all error info ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); CodeGenerator generator = new CodeGenerator( newTool(), g, "Java" ); g.CodeGenerator = generator; if ( g.NumberOfDecisions == 0 ) { g.BuildNFA(); g.CreateLookaheadDFAs( false ); } Assert.AreEqual(0, equeue.size(), "unexpected number of expected problems"); checkDecision( g, 1, expecting, null, null, null, null, null, 0, false ); }
//throws Exception protected void checkDecision( Grammar g, int decision, string expecting, int[] expectingUnreachableAlts, int[] expectingNonDetAlts, string expectingAmbigInput, int[] expectingInsufficientPredAlts, int[] expectingDanglingAlts, int expectingNumWarnings, bool hasPredHiddenByAction ) { DecisionProbe.verbose = true; // make sure we get all error info ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); CodeGenerator generator = new CodeGenerator( newTool(), g, "Java" ); g.CodeGenerator = generator; // mimic actions of org.antlr.Tool first time for grammar g if ( g.NumberOfDecisions == 0 ) { g.BuildNFA(); g.CreateLookaheadDFAs( false ); } if ( equeue.size() != expectingNumWarnings ) { Console.Error.WriteLine( "Warnings issued: " + equeue ); } Assert.AreEqual(expectingNumWarnings, equeue.size(), "unexpected number of expected problems"); DFA dfa = g.GetLookaheadDFA( decision ); FASerializer serializer = new FASerializer( g ); string result = serializer.Serialize( dfa.StartState ); //System.out.print(result); var unreachableAlts = dfa.UnreachableAlts; // make sure unreachable alts are as expected if ( expectingUnreachableAlts != null ) { BitSet s = new BitSet(); s.AddAll( expectingUnreachableAlts ); BitSet s2 = new BitSet(); s2.AddAll( unreachableAlts ); Assert.AreEqual(s, s2, "unreachable alts mismatch"); } else { Assert.AreEqual(0, unreachableAlts != null ? unreachableAlts.Count : 0, "unreachable alts mismatch"); } // check conflicting input if ( expectingAmbigInput != null ) { // first, find nondet message Message msg = getNonDeterminismMessage( equeue.warnings ); Assert.IsNotNull(msg, "no nondeterminism warning?"); Assert.IsTrue(msg is GrammarNonDeterminismMessage, "expecting nondeterminism; found " + msg.GetType().Name); GrammarNonDeterminismMessage nondetMsg = getNonDeterminismMessage( equeue.warnings ); var labels = nondetMsg.probe.GetSampleNonDeterministicInputSequence( nondetMsg.problemState ); string input = nondetMsg.probe.GetInputSequenceDisplay( labels ); Assert.AreEqual( expectingAmbigInput, input ); } // check nondet alts if ( expectingNonDetAlts != null ) { GrammarNonDeterminismMessage nondetMsg = getNonDeterminismMessage( equeue.warnings ); Assert.IsNotNull(nondetMsg, "found no nondet alts; expecting: " + str(expectingNonDetAlts)); var nonDetAlts = nondetMsg.probe.GetNonDeterministicAltsForState( nondetMsg.problemState ); // compare nonDetAlts with expectingNonDetAlts BitSet s = new BitSet(); s.AddAll( expectingNonDetAlts ); BitSet s2 = new BitSet(); s2.AddAll( nonDetAlts ); Assert.AreEqual(s, s2, "nondet alts mismatch"); Assert.AreEqual(hasPredHiddenByAction, nondetMsg.problemState.Dfa.HasPredicateBlockedByAction, "mismatch between expected hasPredHiddenByAction"); } else { // not expecting any nondet alts, make sure there are none GrammarNonDeterminismMessage nondetMsg = getNonDeterminismMessage( equeue.warnings ); Assert.IsNull(nondetMsg, "found nondet alts, but expecting none"); } if ( expectingInsufficientPredAlts != null ) { GrammarInsufficientPredicatesMessage insuffPredMsg = getGrammarInsufficientPredicatesMessage( equeue.warnings ); Assert.IsNotNull(insuffPredMsg, "found no GrammarInsufficientPredicatesMessage alts; expecting: " + str(expectingNonDetAlts)); var locations = insuffPredMsg.altToLocations; var actualAlts = locations.Keys; BitSet s = new BitSet(); s.AddAll( expectingInsufficientPredAlts ); BitSet s2 = new BitSet(); s2.AddAll( actualAlts ); Assert.AreEqual(s, s2, "mismatch between insufficiently covered alts"); Assert.AreEqual(hasPredHiddenByAction, insuffPredMsg.problemState.Dfa.HasPredicateBlockedByAction, "mismatch between expected hasPredHiddenByAction"); } else { // not expecting any nondet alts, make sure there are none GrammarInsufficientPredicatesMessage nondetMsg = getGrammarInsufficientPredicatesMessage( equeue.warnings ); if ( nondetMsg != null ) { Console.Out.WriteLine( equeue.warnings ); } Assert.IsNull(nondetMsg, "found insufficiently covered alts, but expecting none"); } Assert.AreEqual( expecting, result ); }
public void TestLeftRecursivePred() { // No analysis possible. but probably good to fail. Not sure we really want // left-recursion even if guarded with pred. Grammar g = new Grammar( "parser grammar P;\n" + "s : a ;\n" + "a : {p1}? a | ID ;\n" ); DecisionProbe.verbose = true; // make sure we get all error info ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); CodeGenerator generator = new CodeGenerator( newTool(), g, "Java" ); g.CodeGenerator = generator; if ( g.NumberOfDecisions == 0 ) { g.BuildNFA(); g.CreateLookaheadDFAs( false ); } DFA dfa = g.GetLookaheadDFA( 1 ); Assert.AreEqual( null, dfa ); // can't analyze. /* String expecting = ".s0-ID->.s1" + NewLine + ".s1-{p1}?->:s2=>1" + NewLine + ".s1-{true}?->:s3=>2" + NewLine; String result = serializer.serialize(dfa.startState); Assert.AreEqual(expecting, result); */ Assert.AreEqual(1, equeue.size(), "unexpected number of expected problems"); Message msg = equeue.errors[0]; Assert.IsTrue(msg is LeftRecursionCyclesMessage, "warning must be a left recursion msg"); }
//throws Exception protected void checkDecision( Grammar g, int decision, string expecting, int[] expectingUnreachableAlts, int[] expectingNonDetAlts, string expectingAmbigInput, int[] expectingDanglingAlts, int expectingNumWarnings ) { DecisionProbe.verbose = true; // make sure we get all error info ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); // mimic actions of org.antlr.Tool first time for grammar g if ( g.NumberOfDecisions == 0 ) { g.BuildNFA(); g.CreateLookaheadDFAs( false ); } CodeGenerator generator = new CodeGenerator( newTool(), g, "Java" ); g.CodeGenerator = generator; if ( equeue.size() != expectingNumWarnings ) { Console.Error.WriteLine( "Warnings issued: " + equeue ); } Assert.AreEqual(expectingNumWarnings, equeue.size(), "unexpected number of expected problems"); DFA dfa = g.GetLookaheadDFA( decision ); Assert.IsNotNull( dfa, "no DFA for decision " + decision ); FASerializer serializer = new FASerializer( g ); string result = serializer.Serialize( dfa.StartState ); var unreachableAlts = dfa.UnreachableAlts; // make sure unreachable alts are as expected if ( expectingUnreachableAlts != null ) { BitSet s = new BitSet(); s.AddAll( expectingUnreachableAlts ); BitSet s2 = new BitSet(); s2.AddAll( unreachableAlts ); Assert.AreEqual(s, s2, "unreachable alts mismatch"); } else { Assert.AreEqual(0, unreachableAlts != null ? unreachableAlts.Count : 0, "number of unreachable alts"); } // check conflicting input if ( expectingAmbigInput != null ) { // first, find nondet message Message msg = (Message)equeue.warnings[0]; Assert.IsTrue(msg is GrammarNonDeterminismMessage, "expecting nondeterminism; found " + msg.GetType().Name); GrammarNonDeterminismMessage nondetMsg = getNonDeterminismMessage( equeue.warnings ); var labels = nondetMsg.probe.GetSampleNonDeterministicInputSequence( nondetMsg.problemState ); string input = nondetMsg.probe.GetInputSequenceDisplay( labels ); Assert.AreEqual( expectingAmbigInput, input ); } // check nondet alts if ( expectingNonDetAlts != null ) { RecursionOverflowMessage recMsg = null; GrammarNonDeterminismMessage nondetMsg = getNonDeterminismMessage( equeue.warnings ); IList<int> nonDetAlts = null; if ( nondetMsg != null ) { nonDetAlts = nondetMsg.probe.GetNonDeterministicAltsForState( nondetMsg.problemState ); } else { recMsg = getRecursionOverflowMessage( equeue.warnings ); if ( recMsg != null ) { //nonDetAlts = new ArrayList(recMsg.alts); } } // compare nonDetAlts with expectingNonDetAlts BitSet s = new BitSet(); s.AddAll( expectingNonDetAlts ); BitSet s2 = new BitSet(); s2.AddAll( nonDetAlts ); Assert.AreEqual(s, s2, "nondet alts mismatch"); Assert.IsTrue(nondetMsg != null || recMsg != null, "found no nondet alts; expecting: " + str(expectingNonDetAlts)); } else { // not expecting any nondet alts, make sure there are none GrammarNonDeterminismMessage nondetMsg = getNonDeterminismMessage( equeue.warnings ); Assert.IsNull(nondetMsg, "found nondet alts, but expecting none"); } Assert.AreEqual( expecting, result ); }
protected void assertRecursionOverflow( Grammar g, IList expectedTargetRules, int expectedAlt ) { DecisionProbe.verbose = true; // make sure we get all error info ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); // mimic actions of org.antlr.Tool first time for grammar g if ( g.NumberOfDecisions == 0 ) { g.BuildNFA(); g.CreateLookaheadDFAs( false ); } RecursionOverflowMessage msg = getRecursionOverflowMessage( equeue.errors ); Assert.IsNotNull(msg, "missing expected recursion overflow msg" + msg); Assert.AreEqual(expectedTargetRules.ToElementString(), msg.targetRules.ToList().ToElementString(), "target rules mismatch"); Assert.AreEqual(expectedAlt, msg.alt, "mismatched alt"); }
protected void assertNonLLStar( Grammar g, IList<int> expectedBadAlts ) { DecisionProbe.verbose = true; // make sure we get all error info ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); // mimic actions of org.antlr.Tool first time for grammar g if ( g.NumberOfDecisions == 0 ) { g.BuildNFA(); g.CreateLookaheadDFAs( false ); } NonRegularDecisionMessage msg = getNonRegularDecisionMessage( equeue.errors ); Assert.IsNotNull(msg, "expected fatal non-LL(*) msg"); List<int> alts = new List<int>(); alts.AddRange( msg.altsWithRecursion ); alts.Sort(); //Collections.sort( alts ); //Assert.AreEqual( expectedBadAlts, alts ); Assert.IsTrue( expectedBadAlts.SequenceEqual( alts ) ); }
public void TestLexerDelegatorRuleOverridesDelegateLeavingNoRules() { // M.Tokens has nothing to predict tokens from S. Should // not include S.Tokens alt in this case? string slave = "lexer grammar S;\n" + "A : 'a' {System.out.println(\"S.A\");} ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "S.g", slave ); string master = "lexer grammar M;\n" + "import S;\n" + "A : 'a' {System.out.println(\"M.A\");} ;\n" + "WS : (' '|'\\n') {skip();} ;\n"; writeFile( tmpdir, "M.g", master ); ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } ); CompositeGrammar composite = new CompositeGrammar(); Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite ); composite.SetDelegationRoot( g ); g.ParseAndBuildAST(); composite.AssignTokenTypes(); composite.DefineGrammarSymbols(); composite.CreateNFAs(); g.CreateLookaheadDFAs( false ); // predict only alts from M not S string expectingDFA = ".s0-'a'->.s1\n" + ".s0-{'\\n', ' '}->:s3=>2\n" + ".s1-<EOT>->:s2=>1\n"; Antlr3.Analysis.DFA dfa = g.GetLookaheadDFA( 1 ); FASerializer serializer = new FASerializer( g ); string result = serializer.Serialize( dfa.startState ); assertEquals( expectingDFA, result ); // must not be a "unreachable alt: Tokens" error assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); }