private void ValidateGrammar()
{
var createAst = _grammar.FlagIsSet(LanguageFlags.CreateAst);
var missingAstTypeSet = new NonTerminalSet();
var invalidTransSet = new NonTerminalSet();
foreach (var nt in _grammarData.NonTerminals)
{
//Check that if CreateAst flag is set then AstNodeType or AstNodeCreator is assigned on all non-transient nodes.
if (createAst && nt.AstNodeCreator == null && nt.AstNodeType == null && !nt.FlagIsSet(TermFlags.NoAstNode))
{
missingAstTypeSet.Add(nt);
}
if (nt.FlagIsSet(TermFlags.IsTransient))
{
//List non-terminals cannot be marked transient - otherwise there may be some ambiguities and inconsistencies
if (nt.FlagIsSet(TermFlags.IsList))
{
_language.Errors.Add(GrammarErrorLevel.Error, null, Resources.ErrListCannotBeTransient, nt.Name);
}
//Count number of non-punctuation child nodes in each production
foreach (var prod in nt.Productions)
{
if (CountNonPunctuationTerms(prod) > 1)
{
invalidTransSet.Add(nt);
}
}
}//if transient
//Validate error productions
foreach (var prod in nt.Productions)
{
if (prod.IsSet(ProductionFlags.IsError))
{
var lastTerm = prod.RValues[prod.RValues.Count - 1];
if (!(lastTerm is Terminal) || lastTerm == _grammar.SyntaxError)
{
_language.Errors.Add(GrammarErrorLevel.Warning, null, Resources.ErrLastTermOfErrorProd, nt.Name);
}
// "The last term of error production must be a terminal. NonTerminal: {0}"
} //foreach prod
}
} //foreac nt
if (missingAstTypeSet.Count > 0)
{
_language.Errors.Add(GrammarErrorLevel.Warning, null, Resources.ErrNodeTypeNotSetOn, missingAstTypeSet.ToString());
}
if (invalidTransSet.Count > 0)
{
_language.Errors.Add(GrammarErrorLevel.Error, null, Resources.ErrTransientNtMustHaveOneTerm, invalidTransSet.ToString());
}
}//method
private static void ComputeNonTerminalsNullability(GrammarData data)
{
var undecided = data.NonTerminals;
while (undecided.Count > 0) {
var newUndecided = new NonTerminalSet();
foreach (NonTerminal nt in undecided)
if (!ComputeNullability(nt))
newUndecided.Add(nt);
if (undecided.Count == newUndecided.Count) return; //we didn't decide on any new, so we're done
undecided = newUndecided;
}//while
}
} //method
private static void ComputeNonTerminalsNullability(GrammarData data)
{
var undecided = data.NonTerminals;
while (undecided.Count > 0)
{
var newUndecided = new NonTerminalSet();
foreach (NonTerminal nt in undecided)
{
if (!ComputeNullability(nt))
{
newUndecided.Add(nt);
}
}
if (undecided.Count == newUndecided.Count)
{
return; //we didn't decide on any new, so we're done
}
undecided = newUndecided;
}//while
}
/*
Initial condition: we have state S with conflicts on lookaheads in Sc.Conflicts.
Each reduce item Ir in S has a set of lookahead sources Ir.ReducedLookaheadSources.
Our goal is to a create non-canonical state ncState with proper lookaheds, create jumps to this state
on jump lookaheads from S to ncState, remove these jump lookaheads from lookahead sets of reduce items
and replace them with non-canonical non-terminal lookaheads.
1. Compute all jump lookaheads jL and non-canonical lookaheads ncL for state Sc.
2. Collect relevant lookahead sources in lkhSources set. For each lookahead source lkhSrc in all reduce items,
if lkhSrc.Current.Firsts includes a jump lookahead in jL, include it into lkhSources.
3. Collect item cores for non-canonical state into ncCores. For each production Prod in productions
of current non-terminal of all items in lkhSources, if Prod.Firsts contains a jump lookahead, then add initial LR0 item of Prod
to ncCores.
4. Add to ncCores all shift items in state Sc that have Current term in jump lookaheads. We need to include
such shift items from original state into non-canonical state to allow proper resolution of shift-reduce
conflicts. We let shift items in current state "compete" with possible reductions to non-canonical lookaheads
inside non-canonical state.
5. Create (or find existing) non-canonical state Sn from ncCores.
6. Assign lookbacks to items in ncState. For each item Src in lkhSources, for each production Prod
in Src.Current.Productions, if Prod.DirectFirsts contains jump lookahead, then:
find LR item I in Sn with I.Core == Prod.LR0Items[0]; do the following: I.Lookbacks.Add(Src.Transition).
7. For state S for each reduce item I adjust I.Lookaheads: remove jump lookaheads from I.Lookaheads,
and add those non-canonical lookaheads that are in I.AllLookaheads
*/
//TODO: one thing to look at - see if all items in ncState should lead to reduce item in some state.
// There may be items (most likely top ones, expansions of original reduced lookahead) that never get
// to full reduce, because we switch back to canonical state on reduction of some "child" non-terminal and
// continue through canonical states from there.
// So we don't need to generate target transition states for these items (unreachable states).
// The main trouble is that unreachable state may introduce conflicts that in fact are never realized.
#endregion
private void SwitchStateToNLalrLookaheads(ParserState state) {
//1. Compute minimal (most expanded) non-canonical lookaheads that resolve all conflicts
ComputeStateNonCanonicalLookaheads(state);
var stateData = state.BuilderData;
//2. Collect reduced lookahead sources and non-terminal lookaheads
var lkhSources = new LRItemSet();
var ntSet = new NonTerminalSet(); //All non-terminals in current positions of lkhSources
foreach(var reduceItem in stateData.ReduceItems)
foreach (var lkhSource in reduceItem.ReducedLookaheadSources) {
var ntLkh = lkhSource.Core.Current as NonTerminal;
if (ntLkh == null) continue;
if (!ntLkh.Firsts.Overlaps(stateData.JumpLookaheads))continue;
lkhSources.Add(lkhSource);
ntSet.Add(ntLkh);
}
//2. Collect core set for non-canonical state
var ncCoreSet = new LR0ItemSet();
foreach(var ntLkh in ntSet)
foreach(var prod in ntLkh.Productions)
if (prod.Firsts.Overlaps(stateData.JumpLookaheads))
ncCoreSet.Add(prod.LR0Items[0]);
//4. Add shift items
foreach (var shiftItem in stateData.ShiftItems)
if (stateData.JumpLookaheads.Contains(shiftItem.Core.Current))
ncCoreSet.Add(shiftItem.Core);
//5. Find or create non-canonical state
var oldStateCount = Data.States.Count;
var ncState = FindOrCreateState(ncCoreSet, "SN"); //if not found, state is created and added to state list and state hash
bool ncStateIsNew = Data.States.Count > oldStateCount;
stateData.JumpTarget = ncState;
//6. Setup appropriate lookbacks in items in ncState;
// first set lookbacks for items originated from lookaheads of reduce items in original state.
foreach(var lkhSource in lkhSources) {
var ntLkh = lkhSource.Core.Current as NonTerminal;
foreach (var prod in ntLkh.Productions)
if (prod.Firsts.Overlaps(stateData.JumpLookaheads)) {
var ncItem = ncState.BuilderData.AllItems.FindByCore(prod.LR0Items[0]);
ncItem.Lookbacks.Add(lkhSource.Transition);
}//if
}//foreach lkhSource
//Now items orginated from shift items in original state in step 4 above
// just copy lookbacks
foreach (var shiftItem in stateData.ShiftItems)
if (stateData.JumpLookaheads.Contains(shiftItem.Core.Current)) {
var ncItem = ncState.BuilderData.ShiftItems.FindByCore(shiftItem.Core);
shiftItem.ShiftedItem = ncItem;
ncItem.Lookbacks.UnionWith(shiftItem.Lookbacks);
if (ncItem.Transition != null)
ncItem.Transition.Include(shiftItem.Transition.Includes);
}
PropagateLookbacksAndTransitionsThruShifts(ncState);
//7. Adjust reduce items lookaheads in original state
foreach (var reduceItem in stateData.ReduceItems) {
foreach(var jumpLkh in stateData.JumpLookaheads)
if (reduceItem.Lookaheads.Contains(jumpLkh))
reduceItem.Lookaheads.Remove(jumpLkh);
foreach (var ncLkh in stateData.NonCanonicalLookaheads)
if (reduceItem.AllLookaheads.Contains(ncLkh))
reduceItem.Lookaheads.Add(ncLkh);
}//foreach reduceItem
// 8. Create jump action to non-canonical state, remove shifts on jump lookaheads
state.JumpAction = ParserAction.CreateJump(ncState);
foreach (var jumpTerm in state.BuilderData.JumpLookaheads)
if (state.Actions.ContainsKey(jumpTerm))
state.Actions.Remove(jumpTerm);
//9. Complete generating states
state.BuilderData.Conflicts.ExceptWith(state.BuilderData.JumpLookaheads);
}//method
private void ValidateGrammar()
{
var createAst = _grammar.LanguageFlags.IsSet(LanguageFlags.CreateAst);
var missingAstTypeSet = new NonTerminalSet();
var invalidTransSet = new NonTerminalSet();
foreach(var nt in _grammarData.NonTerminals) {
//Check that if CreateAst flag is set then AstNodeType or AstNodeCreator is assigned on all non-transient nodes.
if(createAst && nt.AstNodeCreator == null && nt.AstNodeType == null && !nt.Flags.IsSet(TermFlags.NoAstNode))
missingAstTypeSet.Add(nt);
if(nt.Flags.IsSet(TermFlags.IsTransient)) {
//List non-terminals cannot be marked transient - otherwise there may be some ambiguities and inconsistencies
if (nt.Flags.IsSet(TermFlags.IsList))
_language.Errors.Add(GrammarErrorLevel.Error, null, Resources.ErrListCannotBeTransient, nt.Name);
//Count number of non-punctuation child nodes in each production
foreach(var prod in nt.Productions)
if (CountNonPunctuationTerms(prod) > 1) invalidTransSet.Add(nt);
}//if transient
//Validate error productions
foreach(var prod in nt.Productions)
if(prod.IsSet(ProductionFlags.IsError)) {
var lastTerm = prod.RValues[prod.RValues.Count -1];
if (!(lastTerm is Terminal) || lastTerm == _grammar.SyntaxError)
_language.Errors.Add(GrammarErrorLevel.Warning, null, Resources.ErrLastTermOfErrorProd, nt.Name);
// "The last term of error production must be a terminal. NonTerminal: {0}"
}//foreach prod
}//foreac nt
if (missingAstTypeSet.Count > 0)
_language.Errors.Add(GrammarErrorLevel.Warning, null, Resources.ErrNodeTypeNotSetOn, missingAstTypeSet.ToString());
if (invalidTransSet.Count > 0)
_language.Errors.Add(GrammarErrorLevel.Error, null, Resources.ErrTransientNtMustHaveOneTerm,invalidTransSet.ToString());
}
/*
* Initial condition: we have state S with conflicts on lookaheads in Sc.Conflicts.
* Each reduce item Ir in S has a set of lookahead sources Ir.ReducedLookaheadSources.
* Our goal is to a create non-canonical state ncState with proper lookaheds, create jumps to this state
* on jump lookaheads from S to ncState, remove these jump lookaheads from lookahead sets of reduce items
* and replace them with non-canonical non-terminal lookaheads.
* 1. Compute all jump lookaheads jL and non-canonical lookaheads ncL for state Sc.
* 2. Collect relevant lookahead sources in lkhSources set. For each lookahead source lkhSrc in all reduce items,
* if lkhSrc.Current.Firsts includes a jump lookahead in jL, include it into lkhSources.
* 3. Collect item cores for non-canonical state into ncCores. For each production Prod in productions
* of current non-terminal of all items in lkhSources, if Prod.Firsts contains a jump lookahead, then add initial LR0 item of Prod
* to ncCores.
* 4. Add to ncCores all shift items in state Sc that have Current term in jump lookaheads. We need to include
* such shift items from original state into non-canonical state to allow proper resolution of shift-reduce
* conflicts. We let shift items in current state "compete" with possible reductions to non-canonical lookaheads
* inside non-canonical state.
* 5. Create (or find existing) non-canonical state Sn from ncCores.
* 6. Assign lookbacks to items in ncState. For each item Src in lkhSources, for each production Prod
* in Src.Current.Productions, if Prod.DirectFirsts contains jump lookahead, then:
* find LR item I in Sn with I.Core == Prod.LR0Items[0]; do the following: I.Lookbacks.Add(Src.Transition).
* 7. For state S for each reduce item I adjust I.Lookaheads: remove jump lookaheads from I.Lookaheads,
* and add those non-canonical lookaheads that are in I.AllLookaheads
*/
//TODO: one thing to look at - see if all items in ncState should lead to reduce item in some state.
// There may be items (most likely top ones, expansions of original reduced lookahead) that never get
// to full reduce, because we switch back to canonical state on reduction of some "child" non-terminal and
// continue through canonical states from there.
// So we don't need to generate target transition states for these items (unreachable states).
// The main trouble is that unreachable state may introduce conflicts that in fact are never realized.
#endregion
private void SwitchStateToNLalrLookaheads(ParserState state)
{
//1. Compute minimal (most expanded) non-canonical lookaheads that resolve all conflicts
ComputeStateNonCanonicalLookaheads(state);
var stateData = state.BuilderData;
//2. Collect reduced lookahead sources and non-terminal lookaheads
var lkhSources = new LRItemSet();
var ntSet = new NonTerminalSet(); //All non-terminals in current positions of lkhSources
foreach (var reduceItem in stateData.ReduceItems)
{
foreach (var lkhSource in reduceItem.ReducedLookaheadSources)
{
var ntLkh = lkhSource.Core.Current as NonTerminal;
if (ntLkh == null)
{
continue;
}
if (!ntLkh.Firsts.Overlaps(stateData.JumpLookaheads))
{
continue;
}
lkhSources.Add(lkhSource);
ntSet.Add(ntLkh);
}
}
//2. Collect core set for non-canonical state
var ncCoreSet = new LR0ItemSet();
foreach (var ntLkh in ntSet)
{
foreach (var prod in ntLkh.Productions)
{
if (prod.Firsts.Overlaps(stateData.JumpLookaheads))
{
ncCoreSet.Add(prod.LR0Items[0]);
}
}
}
//4. Add shift items
foreach (var shiftItem in stateData.ShiftItems)
{
if (stateData.JumpLookaheads.Contains(shiftItem.Core.Current))
{
ncCoreSet.Add(shiftItem.Core);
}
}
//5. Find or create non-canonical state
var oldStateCount = Data.States.Count;
var ncState = FindOrCreateState(ncCoreSet, "SN"); //if not found, state is created and added to state list and state hash
bool ncStateIsNew = Data.States.Count > oldStateCount;
stateData.JumpTarget = ncState;
//6. Setup appropriate lookbacks in items in ncState;
// first set lookbacks for items originated from lookaheads of reduce items in original state.
foreach (var lkhSource in lkhSources)
{
var ntLkh = lkhSource.Core.Current as NonTerminal;
foreach (var prod in ntLkh.Productions)
{
if (prod.Firsts.Overlaps(stateData.JumpLookaheads))
{
var ncItem = ncState.BuilderData.AllItems.FindByCore(prod.LR0Items[0]);
ncItem.Lookbacks.Add(lkhSource.Transition);
} //if
}
} //foreach lkhSource
//Now items orginated from shift items in original state in step 4 above
// just copy lookbacks
foreach (var shiftItem in stateData.ShiftItems)
{
if (stateData.JumpLookaheads.Contains(shiftItem.Core.Current))
{
var ncItem = ncState.BuilderData.ShiftItems.FindByCore(shiftItem.Core);
shiftItem.ShiftedItem = ncItem;
ncItem.Lookbacks.UnionWith(shiftItem.Lookbacks);
if (ncItem.Transition != null)
{
ncItem.Transition.Include(shiftItem.Transition.Includes);
}
}
}
PropagateLookbacksAndTransitionsThruShifts(ncState);
//7. Adjust reduce items lookaheads in original state
foreach (var reduceItem in stateData.ReduceItems)
{
foreach (var jumpLkh in stateData.JumpLookaheads)
{
if (reduceItem.Lookaheads.Contains(jumpLkh))
{
reduceItem.Lookaheads.Remove(jumpLkh);
}
}
foreach (var ncLkh in stateData.NonCanonicalLookaheads)
{
if (reduceItem.AllLookaheads.Contains(ncLkh))
{
reduceItem.Lookaheads.Add(ncLkh);
}
}
}//foreach reduceItem
// 8. Create jump action to non-canonical state, remove shifts on jump lookaheads
state.JumpAction = ParserAction.CreateJump(ncState);
foreach (var jumpTerm in state.BuilderData.JumpLookaheads)
{
if (state.Actions.ContainsKey(jumpTerm))
{
state.Actions.Remove(jumpTerm);
}
}
//9. Complete generating states
state.BuilderData.Conflicts.ExceptWith(state.BuilderData.JumpLookaheads);
}//method