private void ComputeLookaheads(LRItemSet forItems) { foreach (var reduceItem in forItems) { // Find all source states - those that contribute lookaheads var sourceStates = new ParserStateSet(); foreach (var lookbackTrans in reduceItem.Lookbacks) { sourceStates.Add(lookbackTrans.ToState); sourceStates.UnionWith(lookbackTrans.ToState.BuilderData.ReadStateSet); foreach (var includeTrans in lookbackTrans.Includes) { sourceStates.Add(includeTrans.ToState); sourceStates.UnionWith(includeTrans.ToState.BuilderData.ReadStateSet); } //foreach includeTrans } //foreach lookbackTrans //Now merge all shift terminals from all source states foreach (var state in sourceStates) { reduceItem.Lookaheads.UnionWith(state.BuilderData.ShiftTerminals); } //Remove SyntaxError - it is pseudo terminal if (reduceItem.Lookaheads.Contains(_grammar.SyntaxError)) { reduceItem.Lookaheads.Remove(_grammar.SyntaxError); } //Sanity check if (reduceItem.Lookaheads.Count == 0) { _language.Errors.Add(GrammarErrorLevel.InternalError, reduceItem.State, "Reduce item '{0}' in state {1} has no lookaheads.", reduceItem.Core, reduceItem.State); } } //foreach reduceItem } //method
protected LRItemSet LR1Goto(LRItemSet items, Symbol s) { return(LR1Closure( new LRItemSet( items.Where(item => item.Marker < item.Length && item.Rule.Symbols[item.Marker].Equals(s)) .Select(item => new LRItem(item.Rule, item.Marker + 1))))); }
private TransitionList CreateLookbackTransitions(LRItemSet sourceItems) { var newTransitions = new TransitionList(); //Build set of initial cores - this is optimization for performance //We need to find all initial items in all states that shift into one of sourceItems // Each such initial item would have the core from the "initial" cores set that we build from source items. var iniCores = new LR0ItemSet(); foreach (var sourceItem in sourceItems) { iniCores.Add(sourceItem.Core.Production.LR0Items[0]); } //find foreach (var state in Data.States) { foreach (var iniItem in state.BuilderData.InitialItems) { if (!iniCores.Contains(iniItem.Core)) { continue; } var iniItemNt = iniItem.Core.Production.LValue; // iniItem's non-terminal (left side of production) Transition lookback = null; // local var for lookback - transition over iniItemNt var currItem = iniItem; // iniItem is initial item for all currItem's in the shift chain. while (currItem != null) { if (sourceItems.Contains(currItem)) { // We create transitions lazily, only when we actually need them. Check if we have iniItem's transition // in local variable; if not, get it from state's transitions table; if not found, create it. if (lookback == null && !state.BuilderData.Transitions.TryGetValue(iniItemNt, out lookback)) { lookback = new Transition(state, iniItemNt); newTransitions.Add(lookback); } //Now for currItem, either add trans to Lookbacks, or "include" it into currItem.Transition // We need lookbacks ONLY for final items; for non-Final items we need proper Include lists on transitions if (currItem.Core.IsFinal) { currItem.Lookbacks.Add(lookback); } else // if (currItem.Transition != null) // Note: looks like checking for currItem.Transition is redundant - currItem is either: // - Final - always the case for the first run of this method; // - it has a transition after the first run, due to the way we select sourceItems list // in SelectNewItemsThatNeedLookback (by transitions) { currItem.Transition.Include(lookback); } }//if //move to next item currItem = currItem.ShiftedItem; } //while } //foreach iniItem } //foreach state return(newTransitions); }
//We compute only transitions that are really needed to compute lookaheads in inadequate states. // We start with reduce items in inadequate state and find their lookbacks - this is initial list of transitions. // Then for each transition in the list we check if it has items with nullable tails; for those items we compute // lookbacks - these are new or already existing transitons - and so on, we repeat the operation until no new transitions // are created. private void ComputeTransitions(LRItemSet forItems) { var newItemsNeedLookbacks = forItems; while (newItemsNeedLookbacks.Count > 0) { var newTransitions = CreateLookbackTransitions(newItemsNeedLookbacks); newItemsNeedLookbacks = SelectNewItemsThatNeedLookback(newTransitions); } }
public LRItemSet SelectByLookahead(Terminal lookahead) { var result = new LRItemSet(); foreach (var item in this) { if (item.Lookaheads.Contains(lookahead)) { result.Add(item); } } return(result); }
public LRItemSet SelectByCurrent(BnfTerm current) { var result = new LRItemSet(); foreach (var item in this) { if (item.Core.Current == current) { result.Add(item); } } return(result); }
private LRItemSet GetReduceItemsInInadequateState() { var result = new LRItemSet(); foreach (var state in Data.States) { if (state.BuilderData.IsInadequate) { result.UnionWith(state.BuilderData.ReduceItems); } } return(result); }
private LRItemSet SelectNewItemsThatNeedLookback(TransitionList transitions) { //Select items with nullable tails that don't have lookbacks yet var items = new LRItemSet(); foreach (var trans in transitions) { foreach (var item in trans.Items) { if (item.Core.TailIsNullable && item.Lookbacks.Count == 0) //only if it does not have lookbacks yet { items.Add(item); } } } return(items); }
/// <summary> /// Computes the LR(1) closure, which includes lookaheads in the item sets. /// </summary> /// <param name="items">The list of items from which to complete the closure.</param> /// <returns>The LR(1) closure.</returns> // This is documented on page 261 of Compilers 2nd Ed. The difference is that each item // can have a set of lookaheads rather than duplicate items with single differing lookaheads. protected LRItemSet LR1Closure(LRItemSet items) { // Initialize the return set to the item set var newset = new LRItemSet(items); newset.IsClosed = true; // Keep looping until no more items were added in this iteration bool changed; do { changed = false; var toAdd = new List <LRItem>(); // For each item in the set with a marker before a nonterminal foreach (var item in newset.Where(i => i.Marker < i.Length && !i.Rule.Symbols[i.Marker].IsTerminal)) { var nonterminal = item.Rule.Symbols[item.Marker].Nonterminal; // Get all the possible lookaheads past this symbol var newLookaheads = new HashSet <Terminal_T>(); foreach (var lookahead in item.Lookaheads) { var followingSymbols = item.Rule.Symbols .Skip(item.Marker + 1) .Concat(new Symbol(lookahead).AsSingletonEnumerable()); newLookaheads.UnionWith(FirstOf(followingSymbols)); } if (!newLookaheads.Any()) { continue; } // For each rule of the production past the marker for this item toAdd.AddRange(Productions[nonterminal].Rules.Select(rule => new LRItem(rule, 0, newLookaheads))); } // Try to add the closure to the item set if (toAdd.Count > 0 && newset.Merge(toAdd)) { changed = true; } } while (changed); return(newset); }
/* Initial condition: we have state S with conflicts on lookaheads in Sc.Conflicts. Each reduce item Ir in S has a set of lookahead sources Ir.ReducedLookaheadSources. Our goal is to a create non-canonical state ncState with proper lookaheds, create jumps to this state on jump lookaheads from S to ncState, remove these jump lookaheads from lookahead sets of reduce items and replace them with non-canonical non-terminal lookaheads. 1. Compute all jump lookaheads jL and non-canonical lookaheads ncL for state Sc. 2. Collect relevant lookahead sources in lkhSources set. For each lookahead source lkhSrc in all reduce items, if lkhSrc.Current.Firsts includes a jump lookahead in jL, include it into lkhSources. 3. Collect item cores for non-canonical state into ncCores. For each production Prod in productions of current non-terminal of all items in lkhSources, if Prod.Firsts contains a jump lookahead, then add initial LR0 item of Prod to ncCores. 4. Add to ncCores all shift items in state Sc that have Current term in jump lookaheads. We need to include such shift items from original state into non-canonical state to allow proper resolution of shift-reduce conflicts. We let shift items in current state "compete" with possible reductions to non-canonical lookaheads inside non-canonical state. 5. Create (or find existing) non-canonical state Sn from ncCores. 6. Assign lookbacks to items in ncState. For each item Src in lkhSources, for each production Prod in Src.Current.Productions, if Prod.DirectFirsts contains jump lookahead, then: find LR item I in Sn with I.Core == Prod.LR0Items[0]; do the following: I.Lookbacks.Add(Src.Transition). 7. For state S for each reduce item I adjust I.Lookaheads: remove jump lookaheads from I.Lookaheads, and add those non-canonical lookaheads that are in I.AllLookaheads */ //TODO: one thing to look at - see if all items in ncState should lead to reduce item in some state. // There may be items (most likely top ones, expansions of original reduced lookahead) that never get // to full reduce, because we switch back to canonical state on reduction of some "child" non-terminal and // continue through canonical states from there. // So we don't need to generate target transition states for these items (unreachable states). // The main trouble is that unreachable state may introduce conflicts that in fact are never realized. #endregion private void SwitchStateToNLalrLookaheads(ParserState state) { //1. Compute minimal (most expanded) non-canonical lookaheads that resolve all conflicts ComputeStateNonCanonicalLookaheads(state); var stateData = state.BuilderData; //2. Collect reduced lookahead sources and non-terminal lookaheads var lkhSources = new LRItemSet(); var ntSet = new NonTerminalSet(); //All non-terminals in current positions of lkhSources foreach(var reduceItem in stateData.ReduceItems) foreach (var lkhSource in reduceItem.ReducedLookaheadSources) { var ntLkh = lkhSource.Core.Current as NonTerminal; if (ntLkh == null) continue; if (!ntLkh.Firsts.Overlaps(stateData.JumpLookaheads))continue; lkhSources.Add(lkhSource); ntSet.Add(ntLkh); } //2. Collect core set for non-canonical state var ncCoreSet = new LR0ItemSet(); foreach(var ntLkh in ntSet) foreach(var prod in ntLkh.Productions) if (prod.Firsts.Overlaps(stateData.JumpLookaheads)) ncCoreSet.Add(prod.LR0Items[0]); //4. Add shift items foreach (var shiftItem in stateData.ShiftItems) if (stateData.JumpLookaheads.Contains(shiftItem.Core.Current)) ncCoreSet.Add(shiftItem.Core); //5. Find or create non-canonical state var oldStateCount = Data.States.Count; var ncState = FindOrCreateState(ncCoreSet, "SN"); //if not found, state is created and added to state list and state hash bool ncStateIsNew = Data.States.Count > oldStateCount; stateData.JumpTarget = ncState; //6. Setup appropriate lookbacks in items in ncState; // first set lookbacks for items originated from lookaheads of reduce items in original state. foreach(var lkhSource in lkhSources) { var ntLkh = lkhSource.Core.Current as NonTerminal; foreach (var prod in ntLkh.Productions) if (prod.Firsts.Overlaps(stateData.JumpLookaheads)) { var ncItem = ncState.BuilderData.AllItems.FindByCore(prod.LR0Items[0]); ncItem.Lookbacks.Add(lkhSource.Transition); }//if }//foreach lkhSource //Now items orginated from shift items in original state in step 4 above // just copy lookbacks foreach (var shiftItem in stateData.ShiftItems) if (stateData.JumpLookaheads.Contains(shiftItem.Core.Current)) { var ncItem = ncState.BuilderData.ShiftItems.FindByCore(shiftItem.Core); shiftItem.ShiftedItem = ncItem; ncItem.Lookbacks.UnionWith(shiftItem.Lookbacks); if (ncItem.Transition != null) ncItem.Transition.Include(shiftItem.Transition.Includes); } PropagateLookbacksAndTransitionsThruShifts(ncState); //7. Adjust reduce items lookaheads in original state foreach (var reduceItem in stateData.ReduceItems) { foreach(var jumpLkh in stateData.JumpLookaheads) if (reduceItem.Lookaheads.Contains(jumpLkh)) reduceItem.Lookaheads.Remove(jumpLkh); foreach (var ncLkh in stateData.NonCanonicalLookaheads) if (reduceItem.AllLookaheads.Contains(ncLkh)) reduceItem.Lookaheads.Add(ncLkh); }//foreach reduceItem // 8. Create jump action to non-canonical state, remove shifts on jump lookaheads state.JumpAction = ParserAction.CreateJump(ncState); foreach (var jumpTerm in state.BuilderData.JumpLookaheads) if (state.Actions.ContainsKey(jumpTerm)) state.Actions.Remove(jumpTerm); //9. Complete generating states state.BuilderData.Conflicts.ExceptWith(state.BuilderData.JumpLookaheads); }//method
private LRItemSet SelectNewItemsThatNeedLookback(TransitionList transitions) { //Select items with nullable tails that don't have lookbacks yet var items = new LRItemSet(); foreach(var trans in transitions) foreach(var item in trans.Items) if (item.Core.TailIsNullable && item.Lookbacks.Count == 0) //only if it does not have lookbacks yet items.Add(item); return items; }
private LRItemSet GetReduceItemsInInadequateState() { var result = new LRItemSet(); foreach(var state in Data.States) { if (state.BuilderData.IsInadequate) result.UnionWith(state.BuilderData.ReduceItems); } return result; }
private TransitionList CreateLookbackTransitions(LRItemSet sourceItems) { var newTransitions = new TransitionList(); //Build set of initial cores - this is optimization for performance //We need to find all initial items in all states that shift into one of sourceItems // Each such initial item would have the core from the "initial" cores set that we build from source items. var iniCores = new LR0ItemSet(); foreach(var sourceItem in sourceItems) iniCores.Add(sourceItem.Core.Production.LR0Items[0]); //find foreach(var state in Data.States) { foreach(var iniItem in state.BuilderData.InitialItems) { if (!iniCores.Contains(iniItem.Core)) continue; var iniItemNt = iniItem.Core.Production.LValue; // iniItem's non-terminal (left side of production) Transition lookback = null; // local var for lookback - transition over iniItemNt var currItem = iniItem; // iniItem is initial item for all currItem's in the shift chain. while (currItem != null) { if(sourceItems.Contains(currItem)) { // We create transitions lazily, only when we actually need them. Check if we have iniItem's transition // in local variable; if not, get it from state's transitions table; if not found, create it. if(lookback == null && !state.BuilderData.Transitions.TryGetValue(iniItemNt, out lookback)) { lookback = new Transition(state, iniItemNt); newTransitions.Add(lookback); } //Now for currItem, either add trans to Lookbacks, or "include" it into currItem.Transition // We need lookbacks ONLY for final items; for non-Final items we need proper Include lists on transitions if (currItem.Core.IsFinal) currItem.Lookbacks.Add(lookback); else // if (currItem.Transition != null) // Note: looks like checking for currItem.Transition is redundant - currItem is either: // - Final - always the case for the first run of this method; // - it has a transition after the first run, due to the way we select sourceItems list // in SelectNewItemsThatNeedLookback (by transitions) currItem.Transition.Include(lookback); }//if //move to next item currItem = currItem.ShiftedItem; }//while }//foreach iniItem }//foreach state return newTransitions; }
//We compute only transitions that are really needed to compute lookaheads in inadequate states. // We start with reduce items in inadequate state and find their lookbacks - this is initial list of transitions. // Then for each transition in the list we check if it has items with nullable tails; for those items we compute // lookbacks - these are new or already existing transitons - and so on, we repeat the operation until no new transitions // are created. private void ComputeTransitions(LRItemSet forItems) { var newItemsNeedLookbacks = forItems; while(newItemsNeedLookbacks.Count > 0) { var newTransitions = CreateLookbackTransitions(newItemsNeedLookbacks); newItemsNeedLookbacks = SelectNewItemsThatNeedLookback(newTransitions); } }
private void ComputeLookaheads(LRItemSet forItems) { foreach(var reduceItem in forItems) { // Find all source states - those that contribute lookaheads var sourceStates = new ParserStateSet(); foreach(var lookbackTrans in reduceItem.Lookbacks) { sourceStates.Add(lookbackTrans.ToState); sourceStates.UnionWith(lookbackTrans.ToState.BuilderData.ReadStateSet); foreach(var includeTrans in lookbackTrans.Includes) { sourceStates.Add(includeTrans.ToState); sourceStates.UnionWith(includeTrans.ToState.BuilderData.ReadStateSet); }//foreach includeTrans }//foreach lookbackTrans //Now merge all shift terminals from all source states foreach(var state in sourceStates) reduceItem.Lookaheads.UnionWith(state.BuilderData.ShiftTerminals); //Remove SyntaxError - it is pseudo terminal if (reduceItem.Lookaheads.Contains(_grammar.SyntaxError)) reduceItem.Lookaheads.Remove(_grammar.SyntaxError); //Sanity check if (reduceItem.Lookaheads.Count == 0) _language.Errors.Add(GrammarErrorLevel.InternalError, reduceItem.State, "Reduce item '{0}' in state {1} has no lookaheads.", reduceItem.Core, reduceItem.State); }//foreach reduceItem }
private TransitionList CreateLookbackTransitions(LRItemSet sourceItems) { var newTransitions = new TransitionList(); //Build set of initial cores - this is optimization for performance //We need to find all initial items in all states that shift into one of sourceItems // Each such initial item would have the core from the "initial" cores set that we build from source items. var iniCores = new LR0ItemSet(); foreach(var sourceItem in sourceItems) iniCores.Add(sourceItem.Core.Production.LR0Items[0]); //find foreach(var state in Data.States) { foreach(var iniItem in state.BuilderData.InitialItems) { if (!iniCores.Contains(iniItem.Core)) continue; var currItem = iniItem; while(currItem != null) { if(sourceItems.Contains(currItem)) { //iniItem is initial item for currItem (one of source items) // check if transition for iniItem's non-terminal exists var ntLeft = iniItem.Core.Production.LValue; Transition trans; if(!state.BuilderData.Transitions.TryGetValue(ntLeft, out trans)) { trans = new Transition(iniItem.State, iniItem.Core.Production.LValue); newTransitions.Add(trans); } //Now for currItem, either add trans to Lookbackbacks, or "include" it into currItem.Transition if(currItem.Core.IsFinal) currItem.Lookbacks.Add(trans); else if(currItem.Transition != null) currItem.Transition.Include(trans); }//if //move to next items currItem = currItem.ShiftedItem; }//while }//foreach iniItem }//foreach state return newTransitions; }
//We compute only transitions that are really needed to compute lookaheads in inadequate states. // We start with reduce items in inadequate state and find their lookbacks - this is initial list of transitions. // Then for each transition in the list we check if it has items with nullable tails; for those items we compute // lookbacks - these are new or already existing transitons - and so on, we repeat the operation until no new transitions // are created. private void ComputeTransitions() { var newItemsNeedLookbacks = _itemsNeedLookaheads = GetReduceItemsInInadequateState(); while(newItemsNeedLookbacks.Count > 0) { var newTransitions = CreateLookbackTransitions(newItemsNeedLookbacks); newItemsNeedLookbacks = SelectNewItemsThatNeedLookback(newTransitions); } }
protected override LRItemSetCollection ComputeItemSetCollection() { var(collection, gotoSymbol) = ComputeLR0ItemSetKernelsCollectionAndGotoLookup(); // Determine propagation of lookaheads, and initialze lookaheads based on spontaneous generation // (page 270-275, dragon book 2nd ed.) collection.StartState.Single().Lookaheads.Add(Eof); var rulePropagations = new Dictionary <(int state, LRItem item), HashSet <(int state, LRItem item)> >(); // For all states foreach (var itemset in collection) { var gotosForState = gotoSymbol[itemset.Index]; // For every kernel item foreach (var kernelitem in itemset) { var itemKey = (state : itemset.Index, item : kernelitem); // Create an item set with a dummy lookahead. // This dummy item set is based on the current state (current item set). var dummyLookahead = Unknown.AsSingletonEnumerable(); var dummyItem = new LRItem(kernelitem.Rule, kernelitem.Marker, dummyLookahead); var dummyItemSet = new LRItemSet(dummyItem.AsSingletonEnumerable()); var j = LR1Closure(dummyItemSet); // For every symbol/state in the goto list foreach (var gotokvp in gotosForState) { // What would be the next state? var gotoItemSet = collection[gotokvp.Value]; var gotoItemLookup = gotoItemSet.ToDictionary(g => g, g => g); // Find the items in the dummy set with the marker before this symbol. foreach (var b in j.Where(bb => bb.Marker < bb.Length && bb.Rule.Symbols[bb.Marker].Equals(gotokvp.Key))) { // Get the item corresponding to the goto state with the marker advanced over the current symbol. var gotoItem = gotoItemLookup[new LRItem(b.Rule, b.Marker + 1)]; // Note if lookaheads are propagated to the next item if (b.Lookaheads.Any(l => l.CompareTo(Unknown) == 0)) { if (!rulePropagations.ContainsKey(itemKey)) { rulePropagations[itemKey] = new HashSet <(int, LRItem)>(); } rulePropagations[itemKey].Add((gotoItemSet.Index, gotoItem)); } gotoItem.Lookaheads.UnionWith( b.Lookaheads.Where(l => l.CompareTo(Unknown) != 0) ); } } } } bool changed; do { changed = false; foreach (var state in collection) { foreach (var item in state) { var itemKey = (state.Index, item); if (!rulePropagations.TryGetValue(itemKey, out var propagated)) { continue; } foreach (var key in propagated) { if (key.item.Lookaheads.TryUnionWith(item.Lookaheads)) { changed = true; } } } } } while (changed); // Close all the kernels for (var i = 0; i < collection.Count; i++) { collection[i] = LR1Closure(collection[i]); collection[i].Index = i; } return(collection); }
public LRItemSetCollection MockKernels() { //return new LRItemSetCollection() //{ // I0: S' -> . S var rule0 = new LRItemSet(new LRItem(_init.Rules.Single(), 0, isKernel: true).AsSingletonEnumerable()); // I1: S' -> S . var rule1 = new LRItemSet(new LRItem(_init.Rules.Single(), 1).AsSingletonEnumerable()); // I2: S -> L . = R // R -> L . var rule2 = new LRItemSet(new[] { new LRItem( _start.Rules.Single(r => r.Symbols.SequenceEqual( new [] { new Symbol(Access), new Symbol(Terminal.Equals), new Symbol(Expression) } ) ), 1 ), new LRItem( _expression.Rules.Single(r => r.Symbols.SequenceEqual( new [] { new Symbol(Access) } ) ), 1 ) }); // I3: S -> R . var rule3 = new LRItemSet(new[] { new LRItem( _start.Rules.Single(r => r.Symbols.SequenceEqual( new [] { new Symbol(Expression) } ) ), 1 ) }); // I4: L -> * . R var rule4 = new LRItemSet(new[] { new LRItem( _access.Rules.Single(r => r.Symbols.SequenceEqual( new [] { new Symbol(Star), new Symbol(Expression) } ) ), 1 ), }); // I5: L -> id . var rule5 = new LRItemSet(new[] { new LRItem( _access.Rules.Single(r => r.Symbols.SequenceEqual( new [] { new Symbol(Ident) } ) ), 1 ), }); // I6: S -> L = . R var rule6 = new LRItemSet(new[] { new LRItem( _start.Rules.Single(r => r.Symbols.SequenceEqual( new [] { new Symbol(Access), new Symbol(Terminal.Equals), new Symbol(Expression) } ) ), 2 ), }); // I7: L -> * R . var rule7 = new LRItemSet(new[] { new LRItem( _access.Rules.Single(r => r.Symbols.SequenceEqual( new [] { new Symbol(Star), new Symbol(Expression) } ) ), 2 ), }); // I8: R -> L . var rule8 = new LRItemSet(new[] { new LRItem( _expression.Rules.Single(r => r.Symbols.SequenceEqual( new [] { new Symbol(Access) } ) ), 1 ), }); // I9: S -> L = R . var rule9 = new LRItemSet(new[] { new LRItem( _start.Rules.Single(r => r.Symbols.SequenceEqual( new [] { new Symbol(Access), new Symbol(Terminal.Equals), new Symbol(Expression) } ) ), 3 ), }); return(new LRItemSetCollection() { rule0, rule1, rule2, rule3, rule4, rule5, rule6, rule7, rule8, rule9 }); }