Пример #1
0
        public void TestLoopsWithOptimizedOutExitBranches()
        {
            Grammar g = new Grammar(
                "lexer grammar t;\n" +
                "A : 'x'* ~'x'+ ;\n" );
            string expecting =
                ".s0-'x'->:s1=>1" + NewLine +
                ".s0-{'\\u0000'..'w', 'y'..'\\uFFFF'}->:s2=>2" + NewLine;
            checkDecision( g, 1, expecting, null );

            // The optimizer yanks out all exit branches from EBNF blocks
            // This is ok because we've already verified there are no problems
            // with the enter/exit decision
            DFAOptimizer optimizer = new DFAOptimizer( g );
            optimizer.Optimize();
            FASerializer serializer = new FASerializer( g );
            DFA dfa = g.GetLookaheadDFA( 1 );
            string result = serializer.Serialize( dfa.StartState );
            expecting = ".s0-'x'->:s1=>1" + NewLine;
            Assert.AreEqual( expecting, result );
        }
Пример #2
0
        //throws Exception
        protected void checkDecision( Grammar g,
                                     int decision,
                                     string expecting,
                                     int[] expectingUnreachableAlts )
        {
            Antlr3.AntlrTool tool = new Antlr3.AntlrTool();
            // mimic actions of org.antlr.Tool first time for grammar g
            if ( g.CodeGenerator == null )
            {
                CodeGenerator generator = new CodeGenerator( tool, g, "Java" );
                g.CodeGenerator = generator;
                g.BuildNFA();
                g.CreateLookaheadDFAs( false );
            }

            DFA dfa = g.GetLookaheadDFA( decision );
            Assert.IsNotNull(dfa, "unknown decision #" + decision);
            FASerializer serializer = new FASerializer( g );
            string result = serializer.Serialize( dfa.StartState );
            //System.out.print(result);
            var nonDetAlts = dfa.UnreachableAlts;
            //System.out.println("alts w/o predict state="+nonDetAlts);

            // first make sure nondeterministic alts are as expected
            if ( expectingUnreachableAlts == null )
            {
                if ( nonDetAlts != null && nonDetAlts.Count != 0 )
                {
                    Console.Error.WriteLine( "nondeterministic alts (should be empty): " + ( (IList)nonDetAlts ).ToElementString() );
                }
                Assert.AreEqual(0, nonDetAlts != null ? nonDetAlts.Count : 0, "unreachable alts mismatch");
            }
            else
            {
                for ( int i = 0; i < expectingUnreachableAlts.Length; i++ )
                {
                    Assert.IsTrue(nonDetAlts != null ? nonDetAlts.Contains(expectingUnreachableAlts[i]) : false, "unreachable alts mismatch");
                }
            }
            Assert.AreEqual( expecting, result );
        }
Пример #3
0
 public override string ToString()
 {
     FASerializer serializer = new FASerializer( Nfa.Grammar );
     if ( StartState == null )
     {
         return "";
     }
     return serializer.Serialize( StartState, false );
 }
Пример #4
0
        //throws Exception
        protected void checkDecision( Grammar g,
                                     int decision,
                                     string expecting,
                                     int[] expectingUnreachableAlts,
                                     int[] expectingNonDetAlts,
                                     string expectingAmbigInput,
                                     int[] expectingInsufficientPredAlts,
                                     int[] expectingDanglingAlts,
                                     int expectingNumWarnings,
                                     bool hasPredHiddenByAction )
        {
            DecisionProbe.verbose = true; // make sure we get all error info
            ErrorQueue equeue = new ErrorQueue();
            ErrorManager.SetErrorListener( equeue );
            CodeGenerator generator = new CodeGenerator( newTool(), g, "Java" );
            g.CodeGenerator = generator;
            // mimic actions of org.antlr.Tool first time for grammar g
            if ( g.NumberOfDecisions == 0 )
            {
                g.BuildNFA();
                g.CreateLookaheadDFAs( false );
            }

            if ( equeue.size() != expectingNumWarnings )
            {
                Console.Error.WriteLine( "Warnings issued: " + equeue );
            }

            Assert.AreEqual(expectingNumWarnings, equeue.size(), "unexpected number of expected problems");

            DFA dfa = g.GetLookaheadDFA( decision );
            FASerializer serializer = new FASerializer( g );
            string result = serializer.Serialize( dfa.StartState );
            //System.out.print(result);
            var unreachableAlts = dfa.UnreachableAlts;

            // make sure unreachable alts are as expected
            if ( expectingUnreachableAlts != null )
            {
                BitSet s = new BitSet();
                s.AddAll( expectingUnreachableAlts );
                BitSet s2 = new BitSet();
                s2.AddAll( unreachableAlts );
                Assert.AreEqual(s, s2, "unreachable alts mismatch");
            }
            else
            {
                Assert.AreEqual(0, unreachableAlts != null ? unreachableAlts.Count : 0, "unreachable alts mismatch");
            }

            // check conflicting input
            if ( expectingAmbigInput != null )
            {
                // first, find nondet message
                Message msg = getNonDeterminismMessage( equeue.warnings );
                Assert.IsNotNull(msg, "no nondeterminism warning?");
                Assert.IsTrue(msg is GrammarNonDeterminismMessage, "expecting nondeterminism; found " + msg.GetType().Name);
                GrammarNonDeterminismMessage nondetMsg =
                    getNonDeterminismMessage( equeue.warnings );
                var labels =
                    nondetMsg.probe.GetSampleNonDeterministicInputSequence( nondetMsg.problemState );
                string input = nondetMsg.probe.GetInputSequenceDisplay( labels );
                Assert.AreEqual( expectingAmbigInput, input );
            }

            // check nondet alts
            if ( expectingNonDetAlts != null )
            {
                GrammarNonDeterminismMessage nondetMsg =
                    getNonDeterminismMessage( equeue.warnings );
                Assert.IsNotNull(nondetMsg, "found no nondet alts; expecting: " + str(expectingNonDetAlts));
                var nonDetAlts =
                    nondetMsg.probe.GetNonDeterministicAltsForState( nondetMsg.problemState );
                // compare nonDetAlts with expectingNonDetAlts
                BitSet s = new BitSet();
                s.AddAll( expectingNonDetAlts );
                BitSet s2 = new BitSet();
                s2.AddAll( nonDetAlts );
                Assert.AreEqual(s, s2, "nondet alts mismatch");
                Assert.AreEqual(hasPredHiddenByAction, nondetMsg.problemState.Dfa.HasPredicateBlockedByAction, "mismatch between expected hasPredHiddenByAction");
            }
            else
            {
                // not expecting any nondet alts, make sure there are none
                GrammarNonDeterminismMessage nondetMsg =
                    getNonDeterminismMessage( equeue.warnings );
                Assert.IsNull(nondetMsg, "found nondet alts, but expecting none");
            }

            if ( expectingInsufficientPredAlts != null )
            {
                GrammarInsufficientPredicatesMessage insuffPredMsg =
                    getGrammarInsufficientPredicatesMessage( equeue.warnings );
                Assert.IsNotNull(insuffPredMsg, "found no GrammarInsufficientPredicatesMessage alts; expecting: " + str(expectingNonDetAlts));
                var locations = insuffPredMsg.altToLocations;
                var actualAlts = locations.Keys;
                BitSet s = new BitSet();
                s.AddAll( expectingInsufficientPredAlts );
                BitSet s2 = new BitSet();
                s2.AddAll( actualAlts );
                Assert.AreEqual(s, s2, "mismatch between insufficiently covered alts");
                Assert.AreEqual(hasPredHiddenByAction, insuffPredMsg.problemState.Dfa.HasPredicateBlockedByAction, "mismatch between expected hasPredHiddenByAction");
            }
            else
            {
                // not expecting any nondet alts, make sure there are none
                GrammarInsufficientPredicatesMessage nondetMsg =
                    getGrammarInsufficientPredicatesMessage( equeue.warnings );
                if ( nondetMsg != null )
                {
                    Console.Out.WriteLine( equeue.warnings );
                }
                Assert.IsNull(nondetMsg, "found insufficiently covered alts, but expecting none");
            }

            Assert.AreEqual( expecting, result );
        }
Пример #5
0
        public virtual void CreateLookaheadDFAs( bool wackTempStructures )
        {
            if ( nfa == null )
            {
                BuildNFA();
            }

            // CHECK FOR LEFT RECURSION; Make sure we can actually do analysis
            CheckAllRulesForLeftRecursion();

            /*
            // was there a severe problem while sniffing the grammar?
            if ( ErrorManager.doNotAttemptAnalysis() ) {
                return;
            }
            */

            DateTime start = DateTime.Now;

            //[email protected]("### create DFAs");
            int numDecisions = NumberOfDecisions;
            if ( NFAToDFAConverter.SINGLE_THREADED_NFA_CONVERSION )
            {
                for ( int decision = 1; decision <= numDecisions; decision++ )
                {
                    NFAState decisionStartState = GetDecisionNFAStartState( decision );
                    if ( leftRecursiveRules.Contains( decisionStartState.enclosingRule ) )
                    {
                        // don't bother to process decisions within left recursive rules.
                        if ( composite.WatchNFAConversion )
                        {
                            Console.Out.WriteLine( "ignoring decision " + decision +
                                               " within left-recursive rule " + decisionStartState.enclosingRule.Name );
                        }
                        continue;
                    }
                    if ( !externalAnalysisAbort && decisionStartState.NumberOfTransitions > 1 )
                    {
                        Rule r = decisionStartState.enclosingRule;
                        if ( r.IsSynPred && !synPredNamesUsedInDFA.Contains( r.Name ) )
                        {
                            continue;
                        }
                        DFA dfa = null;
                        // if k=* or k=1, try LL(1)
                        if ( GetUserMaxLookahead( decision ) == 0 ||
                             GetUserMaxLookahead( decision ) == 1 )
                        {
                            dfa = CreateLL_1_LookaheadDFA( decision );
                        }
                        if ( dfa == null )
                        {
                            if ( composite.WatchNFAConversion )
                            {
                                Console.Out.WriteLine( "decision " + decision +
                                                   " not suitable for LL(1)-optimized DFA analysis" );
                            }
                            dfa = CreateLookaheadDFA( decision, wackTempStructures );
                        }
                        if ( dfa.StartState == null )
                        {
                            // something went wrong; wipe out DFA
                            SetLookaheadDFA( decision, null );
                        }
                        if ( Tool.internalOption_PrintDFA )
                        {
                            Console.Out.WriteLine( "DFA d=" + decision );
                            FASerializer serializer = new FASerializer( nfa.Grammar );
                            string result = serializer.Serialize( dfa.StartState );
                            Console.Out.WriteLine( result );
                        }
                    }
                }
            }
            else
            {
                ErrorManager.Info( "two-threaded DFA conversion" );
                // create a barrier expecting n DFA and this main creation thread
                Barrier barrier = new Barrier( 3 );
                // assume 2 CPU for now
                int midpoint = numDecisions / 2;
                NFAConversionThread t1 =
                    new NFAConversionThread( this, barrier, 1, midpoint );
                new System.Threading.Thread( t1.Run ).Start();
                //new Thread( t1 ).start();
                if ( midpoint == ( numDecisions / 2 ) )
                {
                    midpoint++;
                }
                NFAConversionThread t2 =
                    new NFAConversionThread( this, barrier, midpoint, numDecisions );
                new System.Threading.Thread( t2.Run ).Start();
                //new Thread( t2 ).start();
                // wait for these two threads to finish
                try
                {
                    barrier.WaitForRelease();
                }
                //catch ( InterruptedException e )
                //{
                //    ErrorManager.internalError( "what the hell? DFA interruptus", e );
                //}
                catch
                {
                    throw new System.NotImplementedException();
                }
            }

            DateTime stop = DateTime.Now;
            DFACreationWallClockTimeInMS = stop - start;

            // indicate that we've finished building DFA (even if #decisions==0)
            allDecisionDFACreated = true;
        }
Пример #6
0
        //throws Exception
        protected void checkDecision( Grammar g,
                                     int decision,
                                     string expecting,
                                     int[] expectingUnreachableAlts,
                                     int[] expectingNonDetAlts,
                                     string expectingAmbigInput,
                                     int[] expectingDanglingAlts,
                                     int expectingNumWarnings )
        {
            DecisionProbe.verbose = true; // make sure we get all error info
            ErrorQueue equeue = new ErrorQueue();
            ErrorManager.SetErrorListener( equeue );

            // mimic actions of org.antlr.Tool first time for grammar g
            if ( g.NumberOfDecisions == 0 )
            {
                g.BuildNFA();
                g.CreateLookaheadDFAs( false );
            }
            CodeGenerator generator = new CodeGenerator( newTool(), g, "Java" );
            g.CodeGenerator = generator;

            if ( equeue.size() != expectingNumWarnings )
            {
                Console.Error.WriteLine( "Warnings issued: " + equeue );
            }

            Assert.AreEqual(expectingNumWarnings, equeue.size(), "unexpected number of expected problems");

            DFA dfa = g.GetLookaheadDFA( decision );
            Assert.IsNotNull( dfa, "no DFA for decision " + decision );
            FASerializer serializer = new FASerializer( g );
            string result = serializer.Serialize( dfa.StartState );

            var unreachableAlts = dfa.UnreachableAlts;

            // make sure unreachable alts are as expected
            if ( expectingUnreachableAlts != null )
            {
                BitSet s = new BitSet();
                s.AddAll( expectingUnreachableAlts );
                BitSet s2 = new BitSet();
                s2.AddAll( unreachableAlts );
                Assert.AreEqual(s, s2, "unreachable alts mismatch");
            }
            else
            {
                Assert.AreEqual(0, unreachableAlts != null ? unreachableAlts.Count : 0, "number of unreachable alts");
            }

            // check conflicting input
            if ( expectingAmbigInput != null )
            {
                // first, find nondet message
                Message msg = (Message)equeue.warnings[0];
                Assert.IsTrue(msg is GrammarNonDeterminismMessage, "expecting nondeterminism; found " + msg.GetType().Name);
                GrammarNonDeterminismMessage nondetMsg =
                    getNonDeterminismMessage( equeue.warnings );
                var labels =
                    nondetMsg.probe.GetSampleNonDeterministicInputSequence( nondetMsg.problemState );
                string input = nondetMsg.probe.GetInputSequenceDisplay( labels );
                Assert.AreEqual( expectingAmbigInput, input );
            }

            // check nondet alts
            if ( expectingNonDetAlts != null )
            {
                RecursionOverflowMessage recMsg = null;
                GrammarNonDeterminismMessage nondetMsg =
                    getNonDeterminismMessage( equeue.warnings );
                IList<int> nonDetAlts = null;
                if ( nondetMsg != null )
                {
                    nonDetAlts =
                        nondetMsg.probe.GetNonDeterministicAltsForState( nondetMsg.problemState );
                }
                else
                {
                    recMsg = getRecursionOverflowMessage( equeue.warnings );
                    if ( recMsg != null )
                    {
                        //nonDetAlts = new ArrayList(recMsg.alts);
                    }
                }
                // compare nonDetAlts with expectingNonDetAlts
                BitSet s = new BitSet();
                s.AddAll( expectingNonDetAlts );
                BitSet s2 = new BitSet();
                s2.AddAll( nonDetAlts );
                Assert.AreEqual(s, s2, "nondet alts mismatch");
                Assert.IsTrue(nondetMsg != null || recMsg != null, "found no nondet alts; expecting: " + str(expectingNonDetAlts));
            }
            else
            {
                // not expecting any nondet alts, make sure there are none
                GrammarNonDeterminismMessage nondetMsg =
                    getNonDeterminismMessage( equeue.warnings );
                Assert.IsNull(nondetMsg, "found nondet alts, but expecting none");
            }

            Assert.AreEqual( expecting, result );
        }
Пример #7
0
        public void TestLexerDelegatorRuleOverridesDelegateLeavingNoRules()
        {
            // M.Tokens has nothing to predict tokens from S.  Should
            // not include S.Tokens alt in this case?
            string slave =
                "lexer grammar S;\n" +
                "A : 'a' {System.out.println(\"S.A\");} ;\n";
            mkdir( tmpdir );
            writeFile( tmpdir, "S.g", slave );
            string master =
                "lexer grammar M;\n" +
                "import S;\n" +
                "A : 'a' {System.out.println(\"M.A\");} ;\n" +
                "WS : (' '|'\\n') {skip();} ;\n";
            writeFile( tmpdir, "M.g", master );

            ErrorQueue equeue = new ErrorQueue();
            ErrorManager.SetErrorListener( equeue );
            AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } );
            CompositeGrammar composite = new CompositeGrammar();
            Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite );
            composite.SetDelegationRoot( g );
            g.ParseAndBuildAST();
            composite.AssignTokenTypes();
            composite.DefineGrammarSymbols();
            composite.CreateNFAs();
            g.CreateLookaheadDFAs( false );

            // predict only alts from M not S
            string expectingDFA =
                ".s0-'a'->.s1\n" +
                ".s0-{'\\n', ' '}->:s3=>2\n" +
                ".s1-<EOT>->:s2=>1\n";
            Antlr3.Analysis.DFA dfa = g.GetLookaheadDFA( 1 );
            FASerializer serializer = new FASerializer( g );
            string result = serializer.Serialize( dfa.startState );
            assertEquals( expectingDFA, result );

            // must not be a "unreachable alt: Tokens" error
            assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
        }