public virtual void ReportRecursionOverflow(DFAState d,
NFAConfiguration recursionNFAConfiguration)
{
// track the state number rather than the state as d will change
// out from underneath us; hash wouldn't return any value
// left-recursion is detected in start state. Since we can't
// call resolveNondeterminism() on the start state (it would
// not look k=1 to get min single token lookahead), we must
// prevent errors derived from this state. Avoid start state
if (d.stateNumber > 0)
{
int stateI = d.stateNumber;
_stateToRecursionOverflowConfigurationsMap.Map(stateI, recursionNFAConfiguration);
}
}
protected internal virtual IParseTree MatchImpl(IParseTree tree, IParseTree patternTree, MultiMap<string, IParseTree> labels)
{
if (tree == null)
{
throw new ArgumentException("tree cannot be null");
}
if (patternTree == null)
{
throw new ArgumentException("patternTree cannot be null");
}
// x and <ID>, x and y, or x and x; or could be mismatched types
if (tree is ITerminalNode && patternTree is ITerminalNode)
{
ITerminalNode t1 = (ITerminalNode)tree;
ITerminalNode t2 = (ITerminalNode)patternTree;
IParseTree mismatchedNode = null;
// both are tokens and they have same type
if (t1.Symbol.Type == t2.Symbol.Type)
{
if (t2.Symbol is TokenTagToken)
{
// x and <ID>
TokenTagToken tokenTagToken = (TokenTagToken)t2.Symbol;
// track label->list-of-nodes for both token name and label (if any)
labels.Map(tokenTagToken.TokenName, tree);
if (tokenTagToken.Label != null)
{
labels.Map(tokenTagToken.Label, tree);
}
}
else
{
if (t1.GetText().Equals(t2.GetText(), StringComparison.Ordinal))
{
}
else
{
// x and x
// x and y
if (mismatchedNode == null)
{
mismatchedNode = t1;
}
}
}
}
else
{
if (mismatchedNode == null)
{
mismatchedNode = t1;
}
}
return mismatchedNode;
}
if (tree is ParserRuleContext && patternTree is ParserRuleContext)
{
ParserRuleContext r1 = (ParserRuleContext)tree;
ParserRuleContext r2 = (ParserRuleContext)patternTree;
IParseTree mismatchedNode = null;
// (expr ...) and <expr>
RuleTagToken ruleTagToken = GetRuleTagToken(r2);
if (ruleTagToken != null)
{
if (r1.RuleIndex == r2.RuleIndex)
{
// track label->list-of-nodes for both rule name and label (if any)
labels.Map(ruleTagToken.RuleName, tree);
if (ruleTagToken.Label != null)
{
labels.Map(ruleTagToken.Label, tree);
}
}
else
{
if (mismatchedNode == null)
{
mismatchedNode = r1;
}
}
return mismatchedNode;
}
// (expr ...) and (expr ...)
if (r1.ChildCount != r2.ChildCount)
{
if (mismatchedNode == null)
{
mismatchedNode = r1;
}
return mismatchedNode;
}
int n = r1.ChildCount;
for (int i = 0; i < n; i++)
{
IParseTree childMatch = MatchImpl(r1.GetChild(i), patternTree.GetChild(i), labels);
if (childMatch != null)
{
return childMatch;
}
}
return mismatchedNode;
}
// if nodes aren't both tokens or both rule nodes, can't match
return tree;
}
public void edge(String source, String target)
{
edges.Map(source, target);
}
protected internal virtual IParseTree MatchImpl(IParseTree tree, IParseTree patternTree, MultiMap <string, IParseTree> labels)
{
if (tree == null)
{
throw new ArgumentException("tree cannot be null");
}
if (patternTree == null)
{
throw new ArgumentException("patternTree cannot be null");
}
// x and <ID>, x and y, or x and x; or could be mismatched types
if (tree is ITerminalNode && patternTree is ITerminalNode)
{
ITerminalNode t1 = (ITerminalNode)tree;
ITerminalNode t2 = (ITerminalNode)patternTree;
IParseTree mismatchedNode = null;
// both are tokens and they have same type
if (t1.Symbol.Type == t2.Symbol.Type)
{
if (t2.Symbol is TokenTagToken)
{
// x and <ID>
TokenTagToken tokenTagToken = (TokenTagToken)t2.Symbol;
// track label->list-of-nodes for both token name and label (if any)
labels.Map(tokenTagToken.TokenName, tree);
if (tokenTagToken.Label != null)
{
labels.Map(tokenTagToken.Label, tree);
}
}
else
{
if (t1.GetText().Equals(t2.GetText(), StringComparison.Ordinal))
{
}
else
{
// x and x
// x and y
if (mismatchedNode == null)
{
mismatchedNode = t1;
}
}
}
}
else
{
if (mismatchedNode == null)
{
mismatchedNode = t1;
}
}
return(mismatchedNode);
}
if (tree is ParserRuleContext && patternTree is ParserRuleContext)
{
ParserRuleContext r1 = (ParserRuleContext)tree;
ParserRuleContext r2 = (ParserRuleContext)patternTree;
IParseTree mismatchedNode = null;
// (expr ...) and <expr>
RuleTagToken ruleTagToken = GetRuleTagToken(r2);
if (ruleTagToken != null)
{
if (r1.RuleIndex == r2.RuleIndex)
{
// track label->list-of-nodes for both rule name and label (if any)
labels.Map(ruleTagToken.RuleName, tree);
if (ruleTagToken.Label != null)
{
labels.Map(ruleTagToken.Label, tree);
}
}
else
{
if (mismatchedNode == null)
{
mismatchedNode = r1;
}
}
return(mismatchedNode);
}
// (expr ...) and (expr ...)
if (r1.ChildCount != r2.ChildCount)
{
if (mismatchedNode == null)
{
mismatchedNode = r1;
}
return(mismatchedNode);
}
int n = r1.ChildCount;
for (int i = 0; i < n; i++)
{
IParseTree childMatch = MatchImpl(r1.GetChild(i), patternTree.GetChild(i), labels);
if (childMatch != null)
{
return(childMatch);
}
}
return(mismatchedNode);
}
// if nodes aren't both tokens or both rule nodes, can't match
return(tree);
}
/** Walk each NFA configuration in this DFA state looking for a conflict
* where (s|i|ctx) and (s|j|ctx) exist, indicating that state s with
* context conflicting ctx predicts alts i and j. Return an Integer set
* of the alternative numbers that conflict. Two contexts conflict if
* they are equal or one is a stack suffix of the other or one is
* the empty context.
*
* Use a hash table to record the lists of configs for each state
* as they are encountered. We need only consider states for which
* there is more than one configuration. The configurations' predicted
* alt must be different or must have different contexts to avoid a
* conflict.
*
* Don't report conflicts for DFA states that have conflicting Tokens
* rule NFA states; they will be resolved in favor of the first rule.
*/
protected virtual HashSet <int> GetConflictingAlts()
{
// TODO this is called multiple times: cache result?
//[email protected]("getNondetAlts for DFA state "+stateNumber);
HashSet <int> nondeterministicAlts = new HashSet <int>();
// If only 1 NFA conf then no way it can be nondeterministic;
// save the overhead. There are many o-a->o NFA transitions
// and so we save a hash map and iterator creation for each
// state.
int numConfigs = _nfaConfigurations.Count;
if (numConfigs <= 1)
{
return(null);
}
// First get a list of configurations for each state.
// Most of the time, each state will have one associated configuration.
MultiMap <int, NFAConfiguration> stateToConfigListMap =
new MultiMap <int, NFAConfiguration>();
for (int i = 0; i < numConfigs; i++)
{
NFAConfiguration configuration = (NFAConfiguration)_nfaConfigurations[i];
int stateI = configuration.State;
stateToConfigListMap.Map(stateI, configuration);
}
// potential conflicts are states with > 1 configuration and diff alts
ICollection <int> states = stateToConfigListMap.Keys.ToArray();
int numPotentialConflicts = 0;
foreach (int stateI in states)
{
bool thisStateHasPotentialProblem = false;
IList <NFAConfiguration> configsForState;
stateToConfigListMap.TryGetValue(stateI, out configsForState);
int alt = 0;
int numConfigsForState = configsForState.Count;
for (int i = 0; i < numConfigsForState && numConfigsForState > 1; i++)
{
NFAConfiguration c = (NFAConfiguration)configsForState[i];
if (alt == 0)
{
alt = c.Alt;
}
else if (c.Alt != alt)
{
/*
* [email protected]("potential conflict in state "+stateI+
* " configs: "+configsForState);
*/
// 11/28/2005: don't report closures that pinch back
// together in Tokens rule. We want to silently resolve
// to the first token definition ala lex/flex by ignoring
// these conflicts.
// Also this ensures that lexers look for more and more
// characters (longest match) before resorting to predicates.
// TestSemanticPredicates.testLexerMatchesLongestThenTestPred()
// for example would terminate at state s1 and test predicate
// meaning input "ab" would test preds to decide what to
// do but it should match rule C w/o testing preds.
if (dfa.Nfa.Grammar.type != GrammarType.Lexer ||
!dfa.NFADecisionStartState.enclosingRule.Name.Equals(Grammar.ArtificialTokensRuleName))
{
numPotentialConflicts++;
thisStateHasPotentialProblem = true;
}
}
}
if (!thisStateHasPotentialProblem)
{
// remove NFA state's configurations from
// further checking; no issues with it
// (can't remove as it's concurrent modification; set to null)
stateToConfigListMap[stateI] = null;
}
}
// a fast check for potential issues; most states have none
if (numPotentialConflicts == 0)
{
return(null);
}
// we have a potential problem, so now go through config lists again
// looking for different alts (only states with potential issues
// are left in the states set). Now we will check context.
// For example, the list of configs for NFA state 3 in some DFA
// state might be:
// [3|2|[28 18 $], 3|1|[28 $], 3|1, 3|2]
// I want to create a map from context to alts looking for overlap:
// [28 18 $] -> 2
// [28 $] -> 1
// [$] -> 1,2
// Indeed a conflict exists as same state 3, same context [$], predicts
// alts 1 and 2.
// walk each state with potential conflicting configurations
foreach (int stateI in states)
{
IList <NFAConfiguration> configsForState;
stateToConfigListMap.TryGetValue(stateI, out configsForState);
// compare each configuration pair s, t to ensure:
// s.ctx different than t.ctx if s.alt != t.alt
int numConfigsForState = 0;
if (configsForState != null)
{
numConfigsForState = configsForState.Count;
}
for (int i = 0; i < numConfigsForState; i++)
{
NFAConfiguration s = configsForState[i];
for (int j = i + 1; j < numConfigsForState; j++)
{
NFAConfiguration t = configsForState[j];
// conflicts means s.ctx==t.ctx or s.ctx is a stack
// suffix of t.ctx or vice versa (if alts differ).
// Also a conflict if s.ctx or t.ctx is empty
if (s.Alt != t.Alt && s.Context.ConflictsWith(t.Context))
{
nondeterministicAlts.Add(s.Alt);
nondeterministicAlts.Add(t.Alt);
}
}
}
}
if (nondeterministicAlts.Count == 0)
{
return(null);
}
return(nondeterministicAlts);
}
/** Walk each NFA configuration in this DFA state looking for a conflict
* where (s|i|ctx) and (s|j|ctx) exist, indicating that state s with
* context conflicting ctx predicts alts i and j. Return an Integer set
* of the alternative numbers that conflict. Two contexts conflict if
* they are equal or one is a stack suffix of the other or one is
* the empty context.
*
* Use a hash table to record the lists of configs for each state
* as they are encountered. We need only consider states for which
* there is more than one configuration. The configurations' predicted
* alt must be different or must have different contexts to avoid a
* conflict.
*
* Don't report conflicts for DFA states that have conflicting Tokens
* rule NFA states; they will be resolved in favor of the first rule.
*/
protected virtual HashSet<int> GetConflictingAlts()
{
// TODO this is called multiple times: cache result?
//[email protected]("getNondetAlts for DFA state "+stateNumber);
HashSet<int> nondeterministicAlts = new HashSet<int>();
// If only 1 NFA conf then no way it can be nondeterministic;
// save the overhead. There are many o-a->o NFA transitions
// and so we save a hash map and iterator creation for each
// state.
int numConfigs = nfaConfigurations.Size();
if ( numConfigs <= 1 )
{
return null;
}
// First get a list of configurations for each state.
// Most of the time, each state will have one associated configuration.
MultiMap<int, NFAConfiguration> stateToConfigListMap =
new MultiMap<int, NFAConfiguration>();
for ( int i = 0; i < numConfigs; i++ )
{
NFAConfiguration configuration = (NFAConfiguration)nfaConfigurations.Get( i );
int stateI = configuration.state;
stateToConfigListMap.Map( stateI, configuration );
}
// potential conflicts are states with > 1 configuration and diff alts
ICollection<int> states = stateToConfigListMap.Keys.ToArray();
int numPotentialConflicts = 0;
foreach ( int stateI in states )
{
bool thisStateHasPotentialProblem = false;
var configsForState = stateToConfigListMap.get( stateI );
int alt = 0;
int numConfigsForState = configsForState.Count;
for ( int i = 0; i < numConfigsForState && numConfigsForState > 1; i++ )
{
NFAConfiguration c = (NFAConfiguration)configsForState[i];
if ( alt == 0 )
{
alt = c.alt;
}
else if ( c.alt != alt )
{
/*
[email protected]("potential conflict in state "+stateI+
" configs: "+configsForState);
*/
// 11/28/2005: don't report closures that pinch back
// together in Tokens rule. We want to silently resolve
// to the first token definition ala lex/flex by ignoring
// these conflicts.
// Also this ensures that lexers look for more and more
// characters (longest match) before resorting to predicates.
// TestSemanticPredicates.testLexerMatchesLongestThenTestPred()
// for example would terminate at state s1 and test predicate
// meaning input "ab" would test preds to decide what to
// do but it should match rule C w/o testing preds.
if ( dfa.nfa.grammar.type != GrammarType.Lexer ||
!dfa.NFADecisionStartState.enclosingRule.Name.Equals( Grammar.ArtificialTokensRuleName ) )
{
numPotentialConflicts++;
thisStateHasPotentialProblem = true;
}
}
}
if ( !thisStateHasPotentialProblem )
{
// remove NFA state's configurations from
// further checking; no issues with it
// (can't remove as it's concurrent modification; set to null)
stateToConfigListMap[stateI] = null;
}
}
// a fast check for potential issues; most states have none
if ( numPotentialConflicts == 0 )
{
return null;
}
// we have a potential problem, so now go through config lists again
// looking for different alts (only states with potential issues
// are left in the states set). Now we will check context.
// For example, the list of configs for NFA state 3 in some DFA
// state might be:
// [3|2|[28 18 $], 3|1|[28 $], 3|1, 3|2]
// I want to create a map from context to alts looking for overlap:
// [28 18 $] -> 2
// [28 $] -> 1
// [$] -> 1,2
// Indeed a conflict exists as same state 3, same context [$], predicts
// alts 1 and 2.
// walk each state with potential conflicting configurations
foreach ( int stateI in states )
{
var configsForState = stateToConfigListMap.get( stateI );
// compare each configuration pair s, t to ensure:
// s.ctx different than t.ctx if s.alt != t.alt
int numConfigsForState = 0;
if ( configsForState != null )
{
numConfigsForState = configsForState.Count;
}
for ( int i = 0; i < numConfigsForState; i++ )
{
NFAConfiguration s = configsForState[i];
for ( int j = i + 1; j < numConfigsForState; j++ )
{
NFAConfiguration t = configsForState[j];
// conflicts means s.ctx==t.ctx or s.ctx is a stack
// suffix of t.ctx or vice versa (if alts differ).
// Also a conflict if s.ctx or t.ctx is empty
if ( s.alt != t.alt && s.context.ConflictsWith( t.context ) )
{
nondeterministicAlts.Add( s.alt );
nondeterministicAlts.Add( t.alt );
}
}
}
}
if ( nondeterministicAlts.Count == 0 )
{
return null;
}
return nondeterministicAlts;
}
public virtual DFA CreateLL_1_LookaheadDFA( int decision )
{
Decision d = GetDecision( decision );
string enclosingRule = d.startState.enclosingRule.Name;
Rule r = d.startState.enclosingRule;
NFAState decisionStartState = GetDecisionNFAStartState( decision );
if ( composite.WatchNFAConversion )
{
Console.Out.WriteLine( "--------------------\nattempting LL(1) DFA (d="
+ decisionStartState.DecisionNumber + ") for " +
decisionStartState.Description );
}
if ( r.IsSynPred && !synPredNamesUsedInDFA.Contains( enclosingRule ) )
{
return null;
}
// compute lookahead for each alt
int numAlts = GetNumberOfAltsForDecisionNFA( decisionStartState );
LookaheadSet[] altLook = new LookaheadSet[numAlts + 1];
for ( int alt = 1; alt <= numAlts; alt++ )
{
int walkAlt =
decisionStartState.TranslateDisplayAltToWalkAlt( alt );
NFAState altLeftEdge = GetNFAStateForAltOfDecision( decisionStartState, walkAlt );
NFAState altStartState = (NFAState)altLeftEdge.transition[0].Target;
//[email protected]("alt "+alt+" start state = "+altStartState.stateNumber);
altLook[alt] = ll1Analyzer.Look( altStartState );
//[email protected]("alt "+alt+": "+altLook[alt].toString(this));
}
// compare alt i with alt j for disjointness
bool decisionIsLL_1 = true;
for ( int i = 1; i <= numAlts; i++ )
{
for ( int j = i + 1; j <= numAlts; j++ )
{
/*
[email protected]("compare "+i+", "+j+": "+
altLook[i].toString(this)+" with "+
altLook[j].toString(this));
*/
LookaheadSet collision = altLook[i].Intersection( altLook[j] );
if ( !collision.IsNil )
{
//[email protected]("collision (non-LL(1)): "+collision.toString(this));
decisionIsLL_1 = false;
goto outer;
}
}
}
outer:
bool foundConfoundingPredicate =
ll1Analyzer.DetectConfoundingPredicates( decisionStartState );
if ( decisionIsLL_1 && !foundConfoundingPredicate )
{
// build an LL(1) optimized DFA with edge for each altLook[i]
if ( NFAToDFAConverter.debug )
{
Console.Out.WriteLine( "decision " + decision + " is simple LL(1)" );
}
DFA lookaheadDFA2 = new LL1DFA( decision, decisionStartState, altLook );
SetLookaheadDFA( decision, lookaheadDFA2 );
UpdateLineColumnToLookaheadDFAMap( lookaheadDFA2 );
return lookaheadDFA2;
}
// not LL(1) but perhaps we can solve with simplified predicate search
// even if k=1 set manually, only resolve here if we have preds; i.e.,
// don't resolve etc...
/*
SemanticContext visiblePredicates =
ll1Analyzer.getPredicates(decisionStartState);
boolean foundConfoundingPredicate =
ll1Analyzer.detectConfoundingPredicates(decisionStartState);
*/
// exit if not forced k=1 or we found a predicate situation we
// can't handle: predicates in rules invoked from this decision.
if ( GetUserMaxLookahead( decision ) != 1 || // not manually set to k=1
!GetAutoBacktrackMode( decision ) ||
foundConfoundingPredicate )
{
//[email protected]("trying LL(*)");
return null;
}
IList<IIntSet> edges = new List<IIntSet>();
for ( int i = 1; i < altLook.Length; i++ )
{
LookaheadSet s = altLook[i];
edges.Add( (IntervalSet)s.TokenTypeSet );
}
IList<IIntSet> disjoint = MakeEdgeSetsDisjoint( edges );
//[email protected]("disjoint="+disjoint);
MultiMap<IntervalSet, int> edgeMap = new MultiMap<IntervalSet, int>();
for ( int i = 0; i < disjoint.Count; i++ )
{
IntervalSet ds = (IntervalSet)disjoint[i];
for ( int alt = 1; alt < altLook.Length; alt++ )
{
LookaheadSet look = altLook[alt];
if ( !ds.And( look.TokenTypeSet ).IsNil )
{
edgeMap.Map( ds, alt );
}
}
}
//[email protected]("edge map: "+edgeMap);
// TODO: how do we know we covered stuff?
// build an LL(1) optimized DFA with edge for each altLook[i]
DFA lookaheadDFA = new LL1DFA( decision, decisionStartState, edgeMap );
SetLookaheadDFA( decision, lookaheadDFA );
// create map from line:col to decision DFA (for ANTLRWorks)
UpdateLineColumnToLookaheadDFAMap( lookaheadDFA );
return lookaheadDFA;
}
