//throws Exception protected void checkDecision( Grammar g, int decision, string expecting, int[] expectingUnreachableAlts ) { Antlr3.AntlrTool tool = new Antlr3.AntlrTool(); // mimic actions of org.antlr.Tool first time for grammar g if ( g.CodeGenerator == null ) { CodeGenerator generator = new CodeGenerator( tool, g, "Java" ); g.CodeGenerator = generator; g.BuildNFA(); g.CreateLookaheadDFAs( false ); } DFA dfa = g.GetLookaheadDFA( decision ); Assert.IsNotNull(dfa, "unknown decision #" + decision); FASerializer serializer = new FASerializer( g ); string result = serializer.Serialize( dfa.StartState ); //System.out.print(result); var nonDetAlts = dfa.UnreachableAlts; //System.out.println("alts w/o predict state="+nonDetAlts); // first make sure nondeterministic alts are as expected if ( expectingUnreachableAlts == null ) { if ( nonDetAlts != null && nonDetAlts.Count != 0 ) { Console.Error.WriteLine( "nondeterministic alts (should be empty): " + ( (IList)nonDetAlts ).ToElementString() ); } Assert.AreEqual(0, nonDetAlts != null ? nonDetAlts.Count : 0, "unreachable alts mismatch"); } else { for ( int i = 0; i < expectingUnreachableAlts.Length; i++ ) { Assert.IsTrue(nonDetAlts != null ? nonDetAlts.Contains(expectingUnreachableAlts[i]) : false, "unreachable alts mismatch"); } } Assert.AreEqual( expecting, result ); }
public virtual void GenerateDFAs( Grammar g ) { for ( int d = 1; d <= g.NumberOfDecisions; d++ ) { DFA dfa = g.GetLookaheadDFA( d ); if ( dfa == null ) { continue; // not there for some reason, ignore } IGraphGenerator generator; if (GenerateDgmlGraphs) { generator = new DgmlGenerator(g); } else { generator = new DOTGenerator(g); } string graph = generator.GenerateGraph( dfa.StartState ); string graphFileName = g.name + "." + "dec-" + d; if ( g.implicitLexer ) { graphFileName = g.name + Grammar.grammarTypeToFileNameSuffix[(int)g.type] + "." + "dec-" + d; } try { WriteGraphFile( g, graphFileName, graph, generator.FileExtension ); } catch ( IOException ioe ) { ErrorManager.Error( ErrorManager.MSG_CANNOT_GEN_DOT_FILE, graphFileName, ioe ); } } }
public void TestLoopsWithOptimizedOutExitBranches() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'x'* ~'x'+ ;\n" ); string expecting = ".s0-'x'->:s1=>1" + NewLine + ".s0-{'\\u0000'..'w', 'y'..'\\uFFFF'}->:s2=>2" + NewLine; checkDecision( g, 1, expecting, null ); // The optimizer yanks out all exit branches from EBNF blocks // This is ok because we've already verified there are no problems // with the enter/exit decision DFAOptimizer optimizer = new DFAOptimizer( g ); optimizer.Optimize(); FASerializer serializer = new FASerializer( g ); DFA dfa = g.GetLookaheadDFA( 1 ); string result = serializer.Serialize( dfa.StartState ); expecting = ".s0-'x'->:s1=>1" + NewLine; Assert.AreEqual( expecting, result ); }
//throws Exception protected void checkDecision( Grammar g, int decision, string expecting, int[] expectingUnreachableAlts, int[] expectingNonDetAlts, string expectingAmbigInput, int[] expectingInsufficientPredAlts, int[] expectingDanglingAlts, int expectingNumWarnings, bool hasPredHiddenByAction ) { DecisionProbe.verbose = true; // make sure we get all error info ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); CodeGenerator generator = new CodeGenerator( newTool(), g, "Java" ); g.CodeGenerator = generator; // mimic actions of org.antlr.Tool first time for grammar g if ( g.NumberOfDecisions == 0 ) { g.BuildNFA(); g.CreateLookaheadDFAs( false ); } if ( equeue.size() != expectingNumWarnings ) { Console.Error.WriteLine( "Warnings issued: " + equeue ); } Assert.AreEqual(expectingNumWarnings, equeue.size(), "unexpected number of expected problems"); DFA dfa = g.GetLookaheadDFA( decision ); FASerializer serializer = new FASerializer( g ); string result = serializer.Serialize( dfa.StartState ); //System.out.print(result); var unreachableAlts = dfa.UnreachableAlts; // make sure unreachable alts are as expected if ( expectingUnreachableAlts != null ) { BitSet s = new BitSet(); s.AddAll( expectingUnreachableAlts ); BitSet s2 = new BitSet(); s2.AddAll( unreachableAlts ); Assert.AreEqual(s, s2, "unreachable alts mismatch"); } else { Assert.AreEqual(0, unreachableAlts != null ? unreachableAlts.Count : 0, "unreachable alts mismatch"); } // check conflicting input if ( expectingAmbigInput != null ) { // first, find nondet message Message msg = getNonDeterminismMessage( equeue.warnings ); Assert.IsNotNull(msg, "no nondeterminism warning?"); Assert.IsTrue(msg is GrammarNonDeterminismMessage, "expecting nondeterminism; found " + msg.GetType().Name); GrammarNonDeterminismMessage nondetMsg = getNonDeterminismMessage( equeue.warnings ); var labels = nondetMsg.probe.GetSampleNonDeterministicInputSequence( nondetMsg.problemState ); string input = nondetMsg.probe.GetInputSequenceDisplay( labels ); Assert.AreEqual( expectingAmbigInput, input ); } // check nondet alts if ( expectingNonDetAlts != null ) { GrammarNonDeterminismMessage nondetMsg = getNonDeterminismMessage( equeue.warnings ); Assert.IsNotNull(nondetMsg, "found no nondet alts; expecting: " + str(expectingNonDetAlts)); var nonDetAlts = nondetMsg.probe.GetNonDeterministicAltsForState( nondetMsg.problemState ); // compare nonDetAlts with expectingNonDetAlts BitSet s = new BitSet(); s.AddAll( expectingNonDetAlts ); BitSet s2 = new BitSet(); s2.AddAll( nonDetAlts ); Assert.AreEqual(s, s2, "nondet alts mismatch"); Assert.AreEqual(hasPredHiddenByAction, nondetMsg.problemState.Dfa.HasPredicateBlockedByAction, "mismatch between expected hasPredHiddenByAction"); } else { // not expecting any nondet alts, make sure there are none GrammarNonDeterminismMessage nondetMsg = getNonDeterminismMessage( equeue.warnings ); Assert.IsNull(nondetMsg, "found nondet alts, but expecting none"); } if ( expectingInsufficientPredAlts != null ) { GrammarInsufficientPredicatesMessage insuffPredMsg = getGrammarInsufficientPredicatesMessage( equeue.warnings ); Assert.IsNotNull(insuffPredMsg, "found no GrammarInsufficientPredicatesMessage alts; expecting: " + str(expectingNonDetAlts)); var locations = insuffPredMsg.altToLocations; var actualAlts = locations.Keys; BitSet s = new BitSet(); s.AddAll( expectingInsufficientPredAlts ); BitSet s2 = new BitSet(); s2.AddAll( actualAlts ); Assert.AreEqual(s, s2, "mismatch between insufficiently covered alts"); Assert.AreEqual(hasPredHiddenByAction, insuffPredMsg.problemState.Dfa.HasPredicateBlockedByAction, "mismatch between expected hasPredHiddenByAction"); } else { // not expecting any nondet alts, make sure there are none GrammarInsufficientPredicatesMessage nondetMsg = getGrammarInsufficientPredicatesMessage( equeue.warnings ); if ( nondetMsg != null ) { Console.Out.WriteLine( equeue.warnings ); } Assert.IsNull(nondetMsg, "found insufficiently covered alts, but expecting none"); } Assert.AreEqual( expecting, result ); }
public void TestLeftRecursivePred() { // No analysis possible. but probably good to fail. Not sure we really want // left-recursion even if guarded with pred. Grammar g = new Grammar( "parser grammar P;\n" + "s : a ;\n" + "a : {p1}? a | ID ;\n" ); DecisionProbe.verbose = true; // make sure we get all error info ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); CodeGenerator generator = new CodeGenerator( newTool(), g, "Java" ); g.CodeGenerator = generator; if ( g.NumberOfDecisions == 0 ) { g.BuildNFA(); g.CreateLookaheadDFAs( false ); } DFA dfa = g.GetLookaheadDFA( 1 ); Assert.AreEqual( null, dfa ); // can't analyze. /* String expecting = ".s0-ID->.s1" + NewLine + ".s1-{p1}?->:s2=>1" + NewLine + ".s1-{true}?->:s3=>2" + NewLine; String result = serializer.serialize(dfa.startState); Assert.AreEqual(expecting, result); */ Assert.AreEqual(1, equeue.size(), "unexpected number of expected problems"); Message msg = equeue.errors[0]; Assert.IsTrue(msg is LeftRecursionCyclesMessage, "warning must be a left recursion msg"); }
//throws Exception protected void checkDecision( Grammar g, int decision, string expecting, int[] expectingUnreachableAlts, int[] expectingNonDetAlts, string expectingAmbigInput, int[] expectingDanglingAlts, int expectingNumWarnings ) { DecisionProbe.verbose = true; // make sure we get all error info ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); // mimic actions of org.antlr.Tool first time for grammar g if ( g.NumberOfDecisions == 0 ) { g.BuildNFA(); g.CreateLookaheadDFAs( false ); } CodeGenerator generator = new CodeGenerator( newTool(), g, "Java" ); g.CodeGenerator = generator; if ( equeue.size() != expectingNumWarnings ) { Console.Error.WriteLine( "Warnings issued: " + equeue ); } Assert.AreEqual(expectingNumWarnings, equeue.size(), "unexpected number of expected problems"); DFA dfa = g.GetLookaheadDFA( decision ); Assert.IsNotNull( dfa, "no DFA for decision " + decision ); FASerializer serializer = new FASerializer( g ); string result = serializer.Serialize( dfa.StartState ); var unreachableAlts = dfa.UnreachableAlts; // make sure unreachable alts are as expected if ( expectingUnreachableAlts != null ) { BitSet s = new BitSet(); s.AddAll( expectingUnreachableAlts ); BitSet s2 = new BitSet(); s2.AddAll( unreachableAlts ); Assert.AreEqual(s, s2, "unreachable alts mismatch"); } else { Assert.AreEqual(0, unreachableAlts != null ? unreachableAlts.Count : 0, "number of unreachable alts"); } // check conflicting input if ( expectingAmbigInput != null ) { // first, find nondet message Message msg = (Message)equeue.warnings[0]; Assert.IsTrue(msg is GrammarNonDeterminismMessage, "expecting nondeterminism; found " + msg.GetType().Name); GrammarNonDeterminismMessage nondetMsg = getNonDeterminismMessage( equeue.warnings ); var labels = nondetMsg.probe.GetSampleNonDeterministicInputSequence( nondetMsg.problemState ); string input = nondetMsg.probe.GetInputSequenceDisplay( labels ); Assert.AreEqual( expectingAmbigInput, input ); } // check nondet alts if ( expectingNonDetAlts != null ) { RecursionOverflowMessage recMsg = null; GrammarNonDeterminismMessage nondetMsg = getNonDeterminismMessage( equeue.warnings ); IList<int> nonDetAlts = null; if ( nondetMsg != null ) { nonDetAlts = nondetMsg.probe.GetNonDeterministicAltsForState( nondetMsg.problemState ); } else { recMsg = getRecursionOverflowMessage( equeue.warnings ); if ( recMsg != null ) { //nonDetAlts = new ArrayList(recMsg.alts); } } // compare nonDetAlts with expectingNonDetAlts BitSet s = new BitSet(); s.AddAll( expectingNonDetAlts ); BitSet s2 = new BitSet(); s2.AddAll( nonDetAlts ); Assert.AreEqual(s, s2, "nondet alts mismatch"); Assert.IsTrue(nondetMsg != null || recMsg != null, "found no nondet alts; expecting: " + str(expectingNonDetAlts)); } else { // not expecting any nondet alts, make sure there are none GrammarNonDeterminismMessage nondetMsg = getNonDeterminismMessage( equeue.warnings ); Assert.IsNull(nondetMsg, "found nondet alts, but expecting none"); } Assert.AreEqual( expecting, result ); }
public void TestLexerDelegatorRuleOverridesDelegateLeavingNoRules() { // M.Tokens has nothing to predict tokens from S. Should // not include S.Tokens alt in this case? string slave = "lexer grammar S;\n" + "A : 'a' {System.out.println(\"S.A\");} ;\n"; mkdir( tmpdir ); writeFile( tmpdir, "S.g", slave ); string master = "lexer grammar M;\n" + "import S;\n" + "A : 'a' {System.out.println(\"M.A\");} ;\n" + "WS : (' '|'\\n') {skip();} ;\n"; writeFile( tmpdir, "M.g", master ); ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } ); CompositeGrammar composite = new CompositeGrammar(); Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite ); composite.SetDelegationRoot( g ); g.ParseAndBuildAST(); composite.AssignTokenTypes(); composite.DefineGrammarSymbols(); composite.CreateNFAs(); g.CreateLookaheadDFAs( false ); // predict only alts from M not S string expectingDFA = ".s0-'a'->.s1\n" + ".s0-{'\\n', ' '}->:s3=>2\n" + ".s1-<EOT>->:s2=>1\n"; Antlr3.Analysis.DFA dfa = g.GetLookaheadDFA( 1 ); FASerializer serializer = new FASerializer( g ); string result = serializer.Serialize( dfa.startState ); assertEquals( expectingDFA, result ); // must not be a "unreachable alt: Tokens" error assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); }
public virtual void GenerateDFAs( Grammar g ) { for ( int d = 1; d <= g.NumberOfDecisions; d++ ) { DFA dfa = g.GetLookaheadDFA( d ); if ( dfa == null ) { continue; // not there for some reason, ignore } DOTGenerator dotGenerator = new DOTGenerator( g ); string dot = dotGenerator.GetDOT( dfa.startState ); string dotFileName = g.name + "." + "dec-" + d; if ( g.implicitLexer ) { dotFileName = g.name + Grammar.grammarTypeToFileNameSuffix[(int)g.type] + "." + "dec-" + d; } try { WriteDOTFile( g, dotFileName, dot ); } catch ( IOException ioe ) { ErrorManager.Error( ErrorManager.MSG_CANNOT_GEN_DOT_FILE, dotFileName, ioe ); } } }