Пример #1
0
        public LookaheadSet Look(NFAState s)
        {
            if (NFAToDFAConverter.debug)
            {
                Console.Out.WriteLine("> LOOK(" + s + ")");
            }
            _lookBusy.Clear();
            LookaheadSet look = FirstCore(s, true);

            // FOLLOW makes no sense (at the moment!) for lexical rules.
            if (_grammar.type != GrammarType.Lexer && look.Member(Label.EOR_TOKEN_TYPE))
            {
                // avoid altering FIRST reset as it is cached
                LookaheadSet f = Follow(s.enclosingRule);
                f.OrInPlace(look);
                f.Remove(Label.EOR_TOKEN_TYPE);
                look = f;
                //look.orInPlace(FOLLOW(s.enclosingRule));
            }
            else if (_grammar.type == GrammarType.Lexer && look.Member(Label.EOT))
            {
                // if this has EOT, lookahead is all char (all char can follow rule)
                //look = new LookaheadSet(Label.EOT);
                look = new LookaheadSet(IntervalSet.COMPLETE_SET);
            }
            if (NFAToDFAConverter.debug)
            {
                Console.Out.WriteLine("< LOOK(" + s + ")=" + look.ToString(_grammar));
            }
            return(look);
        }
Пример #2
0
        public virtual LookaheadSet Intersection(LookaheadSet s)
        {
            IIntSet      i            = this.tokenTypeSet.And(s.tokenTypeSet);
            LookaheadSet intersection = new LookaheadSet(i);

            return(intersection);
        }
Пример #3
0
        /** From an NFA state, s, find the set of all labels reachable from s.
         *  Used to compute follow sets for error recovery.  Never computes
         *  a FOLLOW operation.  FIRST stops at end of rules, returning EOR, unless
         *  invoked from another rule.  I.e., routine properly handles
         *
         *     a : b A ;
         *
         *  where b is nullable.
         *
         *  We record with EOR_TOKEN_TYPE if we hit the end of a rule so we can
         *  know at runtime (when these sets are used) to start walking up the
         *  follow chain to compute the real, correct follow set (as opposed to
         *  the FOLLOW, which is a superset).
         *
         *  This routine will only be used on parser and tree parser grammars.
         */
        public LookaheadSet First(NFAState s)
        {
            //[email protected]("> FIRST("+s.enclosingRule.name+") in rule "+s.enclosingRule);
            _lookBusy.Clear();
            LookaheadSet look = FirstCore(s, false);

            //[email protected]("< FIRST("+s.enclosingRule.name+") in rule "+s.enclosingRule+"="+look.toString(this.grammar));
            return(look);
        }
Пример #4
0
        public LookaheadSet Follow(Rule r)
        {
            //[email protected]("> FOLLOW("+r.name+") in rule "+r.startState.enclosingRule);
            LookaheadSet f = _followCache.get(r);

            if (f != null)
            {
                return(f);
            }
            f = FirstCore(r.stopState, true);
            _followCache[r] = f;
            //[email protected]("< FOLLOW("+r+") in rule "+r.startState.enclosingRule+"="+f.toString(this.grammar));
            return(f);
        }
Пример #5
0
 /** From list of lookahead sets (one per alt in decision), create
  *  an LL(1) DFA.  One edge per set.
  *
  *  s0-{alt1}->:o=>1
  *  | \
  *  |  -{alt2}->:o=>2
  *  |
  *  ...
  */
 public LL1DFA( int decisionNumber, NFAState decisionStartState, LookaheadSet[] altLook )
     : base(decisionNumber, decisionStartState)
 {
     DFAState s0 = NewState();
     StartState = s0;
     UnreachableAlts.Clear();
     for ( int alt = 1; alt < altLook.Length; alt++ )
     {
         DFAState acceptAltState = NewState();
         acceptAltState.IsAcceptState = true;
         SetAcceptState( alt, acceptAltState );
         acceptAltState.LookaheadDepth = 1;
         acceptAltState.CachedUniquelyPredicatedAlt = alt;
         Label e = GetLabelForSet( altLook[alt].TokenTypeSet );
         s0.AddTransition( acceptAltState, e );
     }
 }
Пример #6
0
 public virtual void ComputeRuleFIRSTSets()
 {
     if (NumberOfDecisions == 0)
     {
         createNFAs();
     }
     for (Iterator it = getRules().iterator(); it.hasNext();)
     {
         Rule r = (Rule)it.next();
         if (r.isSynPred)
         {
             continue;
         }
         LookaheadSet s = FIRST(r);
         [email protected]("FIRST(" + r.name + ")=" + s);
     }
 }
Пример #7
0
 /** From list of lookahead sets (one per alt in decision), create
  *  an LL(1) DFA.  One edge per set.
  *
  *  s0-{alt1}->:o=>1
  *  | \
  *  |  -{alt2}->:o=>2
  *  |
  *  ...
  */
 public LL1DFA( int decisionNumber, NFAState decisionStartState, LookaheadSet[] altLook )
 {
     DFAState s0 = NewState();
     startState = s0;
     nfa = decisionStartState.nfa;
     NumberOfAlts = nfa.grammar.GetNumberOfAltsForDecisionNFA( decisionStartState );
     this.decisionNumber = decisionNumber;
     this.NFADecisionStartState = decisionStartState;
     InitAltRelatedInfo();
     UnreachableAlts = null;
     for ( int alt = 1; alt < altLook.Length; alt++ )
     {
         DFAState acceptAltState = NewState();
         acceptAltState.acceptState = true;
         SetAcceptState( alt, acceptAltState );
         acceptAltState.LookaheadDepth = 1;
         acceptAltState.cachedUniquelyPredicatedAlt = alt;
         Label e = GetLabelForSet( altLook[alt].tokenTypeSet );
         s0.AddTransition( acceptAltState, e );
     }
 }
Пример #8
0
 public virtual LookaheadSet Intersection( LookaheadSet s )
 {
     IIntSet i = this.tokenTypeSet.And( s.tokenTypeSet );
     LookaheadSet intersection = new LookaheadSet( i );
     return intersection;
 }
Пример #9
0
 public virtual LookaheadSet Subtract(LookaheadSet other)
 {
     return(new LookaheadSet(this.tokenTypeSet.Subtract(other.tokenTypeSet)));
 }
Пример #10
0
 public virtual LookaheadSet Or(LookaheadSet other)
 {
     return(new LookaheadSet(tokenTypeSet.Or(other.tokenTypeSet)));
 }
Пример #11
0
 public virtual void OrInPlace(LookaheadSet other)
 {
     this.tokenTypeSet.AddAll(other.tokenTypeSet);
 }
Пример #12
0
 public LookaheadSet(LookaheadSet other)
     : this()
 {
     this.tokenTypeSet.AddAll(other.tokenTypeSet);
 }
Пример #13
0
        public virtual DFA CreateLL_1_LookaheadDFA( int decision )
        {
            Decision d = GetDecision( decision );
            string enclosingRule = d.startState.enclosingRule.Name;
            Rule r = d.startState.enclosingRule;
            NFAState decisionStartState = GetDecisionNFAStartState( decision );

            if ( composite.WatchNFAConversion )
            {
                Console.Out.WriteLine( "--------------------\nattempting LL(1) DFA (d="
                                   + decisionStartState.DecisionNumber + ") for " +
                                   decisionStartState.Description );
            }

            if ( r.IsSynPred && !synPredNamesUsedInDFA.Contains( enclosingRule ) )
            {
                return null;
            }

            // compute lookahead for each alt
            int numAlts = GetNumberOfAltsForDecisionNFA( decisionStartState );
            LookaheadSet[] altLook = new LookaheadSet[numAlts + 1];
            for ( int alt = 1; alt <= numAlts; alt++ )
            {
                int walkAlt =
                    decisionStartState.TranslateDisplayAltToWalkAlt( alt );
                NFAState altLeftEdge = GetNFAStateForAltOfDecision( decisionStartState, walkAlt );
                NFAState altStartState = (NFAState)altLeftEdge.transition[0].Target;
                //[email protected]("alt "+alt+" start state = "+altStartState.stateNumber);
                altLook[alt] = ll1Analyzer.Look( altStartState );
                //[email protected]("alt "+alt+": "+altLook[alt].toString(this));
            }

            // compare alt i with alt j for disjointness
            bool decisionIsLL_1 = true;
            for ( int i = 1; i <= numAlts; i++ )
            {
                for ( int j = i + 1; j <= numAlts; j++ )
                {
                    /*
                    [email protected]("compare "+i+", "+j+": "+
                                       altLook[i].toString(this)+" with "+
                                       altLook[j].toString(this));
                    */
                    LookaheadSet collision = altLook[i].Intersection( altLook[j] );
                    if ( !collision.IsNil )
                    {
                        //[email protected]("collision (non-LL(1)): "+collision.toString(this));
                        decisionIsLL_1 = false;
                        goto outer;
                    }
                }
            }

            outer:
            bool foundConfoundingPredicate =
                ll1Analyzer.DetectConfoundingPredicates( decisionStartState );
            if ( decisionIsLL_1 && !foundConfoundingPredicate )
            {
                // build an LL(1) optimized DFA with edge for each altLook[i]
                if ( NFAToDFAConverter.debug )
                {
                    Console.Out.WriteLine( "decision " + decision + " is simple LL(1)" );
                }
                DFA lookaheadDFA2 = new LL1DFA( decision, decisionStartState, altLook );
                SetLookaheadDFA( decision, lookaheadDFA2 );
                UpdateLineColumnToLookaheadDFAMap( lookaheadDFA2 );
                return lookaheadDFA2;
            }

            // not LL(1) but perhaps we can solve with simplified predicate search
            // even if k=1 set manually, only resolve here if we have preds; i.e.,
            // don't resolve etc...

            /*
            SemanticContext visiblePredicates =
                ll1Analyzer.getPredicates(decisionStartState);
            boolean foundConfoundingPredicate =
                ll1Analyzer.detectConfoundingPredicates(decisionStartState);
                */

            // exit if not forced k=1 or we found a predicate situation we
            // can't handle: predicates in rules invoked from this decision.
            if ( GetUserMaxLookahead( decision ) != 1 || // not manually set to k=1
                 !GetAutoBacktrackMode( decision ) ||
                 foundConfoundingPredicate )
            {
                //[email protected]("trying LL(*)");
                return null;
            }

            IList<IIntSet> edges = new List<IIntSet>();
            for ( int i = 1; i < altLook.Length; i++ )
            {
                LookaheadSet s = altLook[i];
                edges.Add( (IntervalSet)s.TokenTypeSet );
            }
            IList<IIntSet> disjoint = MakeEdgeSetsDisjoint( edges );
            //[email protected]("disjoint="+disjoint);

            MultiMap<IntervalSet, int> edgeMap = new MultiMap<IntervalSet, int>();
            for ( int i = 0; i < disjoint.Count; i++ )
            {
                IntervalSet ds = (IntervalSet)disjoint[i];
                for ( int alt = 1; alt < altLook.Length; alt++ )
                {
                    LookaheadSet look = altLook[alt];
                    if ( !ds.And( look.TokenTypeSet ).IsNil )
                    {
                        edgeMap.Map( ds, alt );
                    }
                }
            }
            //[email protected]("edge map: "+edgeMap);

            // TODO: how do we know we covered stuff?

            // build an LL(1) optimized DFA with edge for each altLook[i]
            DFA lookaheadDFA = new LL1DFA( decision, decisionStartState, edgeMap );
            SetLookaheadDFA( decision, lookaheadDFA );

            // create map from line:col to decision DFA (for ANTLRWorks)
            UpdateLineColumnToLookaheadDFAMap( lookaheadDFA );

            return lookaheadDFA;
        }
Пример #14
0
 public LookaheadSet( LookaheadSet other )
     : this()
 {
     this.tokenTypeSet.AddAll( other.tokenTypeSet );
 }
Пример #15
0
 public virtual LookaheadSet Subtract( LookaheadSet other )
 {
     return new LookaheadSet( this.tokenTypeSet.Subtract( other.tokenTypeSet ) );
 }
Пример #16
0
 public virtual void OrInPlace( LookaheadSet other )
 {
     this.tokenTypeSet.AddAll( other.tokenTypeSet );
 }
Пример #17
0
 public virtual LookaheadSet Or( LookaheadSet other )
 {
     return new LookaheadSet( tokenTypeSet.Or( other.tokenTypeSet ) );
 }
Пример #18
0
        protected virtual LookaheadSet FirstCore(NFAState s, bool chaseFollowTransitions)
        {
            /*
             * [email protected]("_LOOK("+s+") in rule "+s.enclosingRule);
             * if ( s.transition[0] instanceof RuleClosureTransition ) {
             *  [email protected]("go to rule "+((NFAState)s.transition[0].target).enclosingRule);
             * }
             */
            if (!chaseFollowTransitions && s.IsAcceptState)
            {
                if (_grammar.type == GrammarType.Lexer)
                {
                    // FOLLOW makes no sense (at the moment!) for lexical rules.
                    // assume all char can follow
                    return(new LookaheadSet(IntervalSet.COMPLETE_SET));
                }
                return(new LookaheadSet(Label.EOR_TOKEN_TYPE));
            }

            if (_lookBusy.Contains(s))
            {
                // return a copy of an empty set; we may modify set inline
                return(new LookaheadSet());
            }
            _lookBusy.Add(s);

            Transition transition0 = s.transition[0];

            if (transition0 == null)
            {
                return(null);
            }

            if (transition0.label.IsAtom)
            {
                int atom = transition0.label.Atom;
                return(new LookaheadSet(atom));
            }
            if (transition0.label.IsSet)
            {
                IIntSet sl = transition0.label.Set;
                return(new LookaheadSet(sl));
            }

            // compute FIRST of transition 0
            LookaheadSet tset = null;

            // if transition 0 is a rule call and we don't want FOLLOW, check cache
            if (!chaseFollowTransitions && transition0 is RuleClosureTransition)
            {
                LookaheadSet prev = _firstCache.get((NFAState)transition0.target);
                if (prev != null)
                {
                    tset = new LookaheadSet(prev);
                }
            }

            // if not in cache, must compute
            if (tset == null)
            {
                tset = FirstCore((NFAState)transition0.target, chaseFollowTransitions);
                // save FIRST cache for transition 0 if rule call
                if (!chaseFollowTransitions && transition0 is RuleClosureTransition)
                {
                    _firstCache[(NFAState)transition0.target] = tset;
                }
            }

            // did we fall off the end?
            if (_grammar.type != GrammarType.Lexer && tset.Member(Label.EOR_TOKEN_TYPE))
            {
                if (transition0 is RuleClosureTransition)
                {
                    // we called a rule that found the end of the rule.
                    // That means the rule is nullable and we need to
                    // keep looking at what follows the rule ref.  E.g.,
                    // a : b A ; where b is nullable means that LOOK(a)
                    // should include A.
                    RuleClosureTransition ruleInvocationTrans =
                        (RuleClosureTransition)transition0;
                    // remove the EOR and get what follows
                    //tset.remove(Label.EOR_TOKEN_TYPE);
                    NFAState     following = (NFAState)ruleInvocationTrans.followState;
                    LookaheadSet fset      = FirstCore(following, chaseFollowTransitions);
                    fset.OrInPlace(tset);   // tset cached; or into new set
                    fset.Remove(Label.EOR_TOKEN_TYPE);
                    tset = fset;
                }
            }

            Transition transition1 = s.transition[1];

            if (transition1 != null)
            {
                LookaheadSet tset1 =
                    FirstCore((NFAState)transition1.target, chaseFollowTransitions);
                tset1.OrInPlace(tset);   // tset cached; or into new set
                tset = tset1;
            }

            return(tset);
        }
Пример #19
0
 public LookaheadSet Look( NFAState s )
 {
     if ( NFAToDFAConverter.debug )
     {
         Console.Out.WriteLine( "> LOOK(" + s + ")" );
     }
     _lookBusy.Clear();
     LookaheadSet look = FirstCore( s, true );
     // FOLLOW makes no sense (at the moment!) for lexical rules.
     if ( _grammar.type != GrammarType.Lexer && look.Member( Label.EOR_TOKEN_TYPE ) )
     {
         // avoid altering FIRST reset as it is cached
         LookaheadSet f = Follow( s.enclosingRule );
         f.OrInPlace( look );
         f.Remove( Label.EOR_TOKEN_TYPE );
         look = f;
         //look.orInPlace(FOLLOW(s.enclosingRule));
     }
     else if ( _grammar.type == GrammarType.Lexer && look.Member( Label.EOT ) )
     {
         // if this has EOT, lookahead is all char (all char can follow rule)
         //look = new LookaheadSet(Label.EOT);
         look = new LookaheadSet( IntervalSet.COMPLETE_SET );
     }
     if ( NFAToDFAConverter.debug )
     {
         Console.Out.WriteLine( "< LOOK(" + s + ")=" + look.ToString( _grammar ) );
     }
     return look;
 }
Пример #20
0
        protected virtual LookaheadSet FirstCore( NFAState s, bool chaseFollowTransitions )
        {
            /*
            [email protected]("_LOOK("+s+") in rule "+s.enclosingRule);
            if ( s.transition[0] instanceof RuleClosureTransition ) {
                [email protected]("go to rule "+((NFAState)s.transition[0].target).enclosingRule);
            }
            */
            if ( !chaseFollowTransitions && s.IsAcceptState )
            {
                if ( _grammar.type == GrammarType.Lexer )
                {
                    // FOLLOW makes no sense (at the moment!) for lexical rules.
                    // assume all char can follow
                    return new LookaheadSet( IntervalSet.COMPLETE_SET );
                }
                return new LookaheadSet( Label.EOR_TOKEN_TYPE );
            }

            if ( _lookBusy.Contains( s ) )
            {
                // return a copy of an empty set; we may modify set inline
                return new LookaheadSet();
            }
            _lookBusy.Add( s );

            Transition transition0 = s.transition[0];
            if ( transition0 == null )
            {
                return null;
            }

            if ( transition0.label.IsAtom )
            {
                int atom = transition0.label.Atom;
                return new LookaheadSet( atom );
            }
            if ( transition0.label.IsSet )
            {
                IIntSet sl = transition0.label.Set;
                return new LookaheadSet( sl );
            }

            // compute FIRST of transition 0
            LookaheadSet tset = null;
            // if transition 0 is a rule call and we don't want FOLLOW, check cache
            if ( !chaseFollowTransitions && transition0 is RuleClosureTransition )
            {
                LookaheadSet prev = _firstCache.get( (NFAState)transition0.target );
                if ( prev != null )
                {
                    tset = new LookaheadSet( prev );
                }
            }

            // if not in cache, must compute
            if ( tset == null )
            {
                tset = FirstCore( (NFAState)transition0.target, chaseFollowTransitions );
                // save FIRST cache for transition 0 if rule call
                if ( !chaseFollowTransitions && transition0 is RuleClosureTransition )
                {
                    _firstCache[(NFAState)transition0.target] = tset;
                }
            }

            // did we fall off the end?
            if ( _grammar.type != GrammarType.Lexer && tset.Member( Label.EOR_TOKEN_TYPE ) )
            {
                if ( transition0 is RuleClosureTransition )
                {
                    // we called a rule that found the end of the rule.
                    // That means the rule is nullable and we need to
                    // keep looking at what follows the rule ref.  E.g.,
                    // a : b A ; where b is nullable means that LOOK(a)
                    // should include A.
                    RuleClosureTransition ruleInvocationTrans =
                        (RuleClosureTransition)transition0;
                    // remove the EOR and get what follows
                    //tset.remove(Label.EOR_TOKEN_TYPE);
                    NFAState following = (NFAState)ruleInvocationTrans.followState;
                    LookaheadSet fset = FirstCore( following, chaseFollowTransitions );
                    fset.OrInPlace( tset ); // tset cached; or into new set
                    fset.Remove( Label.EOR_TOKEN_TYPE );
                    tset = fset;
                }
            }

            Transition transition1 = s.transition[1];
            if ( transition1 != null )
            {
                LookaheadSet tset1 =
                    FirstCore( (NFAState)transition1.target, chaseFollowTransitions );
                tset1.OrInPlace( tset ); // tset cached; or into new set
                tset = tset1;
            }

            return tset;
        }
Пример #21
0
        /** Error recovery in ANTLR recognizers.
         *
         *  Based upon original ideas:
         *
         *  Algorithms + Data Structures = Programs by Niklaus Wirth
         *
         *  and
         *
         *  A note on error recovery in recursive descent parsers:
         *  http://portal.acm.org/citation.cfm?id=947902.947905
         *
         *  Later, Josef Grosch had some good ideas:
         *  Efficient and Comfortable Error Recovery in Recursive Descent Parsers:
         *  ftp://www.cocolab.com/products/cocktail/doca4.ps/ell.ps.zip
         *
         *  Like Grosch I implemented local FOLLOW sets that are combined at run-time
         *  upon error to avoid parsing overhead.
         */
        public virtual void GenerateLocalFollow( GrammarAST referencedElementNode,
                                        string referencedElementName,
                                        string enclosingRuleName,
                                        int elementIndex )
        {
            if (elementIndex < 0)
            {
                throw new ArgumentOutOfRangeException("elementIndex", "elementIndex cannot be less than zero.");
            }

            /*
            [email protected]("compute FOLLOW "+grammar.name+"."+referencedElementNode.toString()+
                             " for "+referencedElementName+"#"+elementIndex +" in "+
                             enclosingRuleName+
                             " line="+referencedElementNode.getLine());
                             */
            NFAState followingNFAState = referencedElementNode.followingNFAState;
            LookaheadSet follow = null;
            if ( followingNFAState != null )
            {
                // compute follow for this element and, as side-effect, track
                // the rule LOOK sensitivity.
                follow = grammar.First( followingNFAState );
            }

            if ( follow == null )
            {
                ErrorManager.InternalError( "no follow state or cannot compute follow" );
                follow = new LookaheadSet();
            }
            if ( follow.Member( Label.EOF ) )
            {
                // TODO: can we just remove?  Seems needed here:
                // compilation_unit : global_statement* EOF
                // Actually i guess we resync to EOF regardless
                follow.Remove( Label.EOF );
            }
            //[email protected](" "+follow);

            IList tokenTypeList = null;
            ulong[] words = null;
            if ( follow.tokenTypeSet == null )
            {
                words = new ulong[1];
                tokenTypeList = new List<object>();
            }
            else
            {
                BitSet bits = BitSet.Of( follow.tokenTypeSet );
                words = bits.ToPackedArray();
                tokenTypeList = follow.tokenTypeSet.ToList();
            }
            // use the target to convert to hex strings (typically)
            string[] wordStrings = new string[words.Length];
            for ( int j = 0; j < words.Length; j++ )
            {
                ulong w = words[j];
                wordStrings[j] = target.GetTarget64BitStringFromValue( w );
            }
            recognizerST.SetAttribute( "bitsets.{name,inName,bits,tokenTypes,tokenIndex}",
                    referencedElementName,
                    enclosingRuleName,
                    wordStrings,
                    tokenTypeList,
                    elementIndex );
            outputFileST.SetAttribute( "bitsets.{name,inName,bits,tokenTypes,tokenIndex}",
                    referencedElementName,
                    enclosingRuleName,
                    wordStrings,
                    tokenTypeList,
                    elementIndex );
            headerFileST.SetAttribute( "bitsets.{name,inName,bits,tokenTypes,tokenIndex}",
                    referencedElementName,
                    enclosingRuleName,
                    wordStrings,
                    tokenTypeList,
                    elementIndex );
        }