public static void GetFirstCollection( this RegulationList grammar, out FIRSTCollection firstCollection, Dictionary <TreeNodeType, bool> nullableDict = null) { if (nullableDict == null) { GetNullableDict(grammar, out nullableDict); } FIRSTCollection firstList4Node; grammar.GetFirstCollection4Node(out firstList4Node, nullableDict); FIRSTCollection firstList4Regulation; grammar.GetFirstCollection4Regulation(out firstList4Regulation, nullableDict, firstList4Node); firstCollection = new FIRSTCollection(); foreach (var item in firstList4Node) { firstCollection.TryInsert(item); } foreach (var item in firstList4Regulation) { firstCollection.TryInsert(item); } }
/// <summary> /// 计算所有可能推导出null的结点。 /// </summary> /// <param name="grammar"></param> /// <returns></returns> public static void GetNullableDict( this RegulationList grammar, out Dictionary <TreeNodeType, bool> nullableDict) { nullableDict = new Dictionary <TreeNodeType, bool>(); // 初始化nullable List <TreeNodeType> allNodeTypes = grammar.GetAllTreeNodeTypes(); foreach (var item in allNodeTypes) { nullableDict.Add(item, false); } // 迭代到不动点 bool changed = false; do { changed = false; foreach (var regulation in grammar) { // 如果RightPart的结点全部可为null,就说明此regulation.Left是可推导出"null"的。 if (Nullable(regulation.RightPart, 0, regulation.RightPart.Count(), nullableDict)) { if (!nullableDict[regulation.Left]) { nullableDict[regulation.Left] = true; changed = true; } } } } while (changed); }
/// <summary> /// LR(0)的Closure操作。 /// 补全一个状态。 /// </summary> /// <param name="list"></param> /// <param name="state"></param> /// <returns></returns> static LR0State Closure(this RegulationList list, LR0State state) { Queue <LR0Item> queue = new Queue <LR0Item>(); foreach (var item in state) { queue.Enqueue(item); } while (queue.Count > 0) { LR0Item item = queue.Dequeue(); TreeNodeType node = item.GetNodeNext2Dot(); if (node == null) { continue; } foreach (var regulation in list) { if (regulation.Left == node) { var newItem = new LR0Item(regulation, 0); if (state.TryInsert(newItem)) { queue.Enqueue(newItem); } } } } return(state); }
/// <summary> /// 计算文法的所有单个的结点的FIRST /// </summary> /// <param name="grammar"></param> /// <param name="nullableDict"></param> /// <returns></returns> private static void GetFirstCollection4Node( this RegulationList grammar, out FIRSTCollection firstCollection, Dictionary <TreeNodeType, bool> nullableDict = null) { // 初始化FIRST firstCollection = new FIRSTCollection(); // 初始化非叶结点的FIRST foreach (var item in grammar.GetAllTreeNodeNonLeaveTypes()) { if (nullableDict[item]) { firstCollection.TryInsert(new FIRST(item, TreeNodeType.NullNode)); } else { firstCollection.TryInsert(new FIRST(item)); } } // 初始化叶结点的FIRST(叶结点的FIRST实际上已经完工) foreach (var item in grammar.GetAllTreeNodeLeaveTypes()) { firstCollection.TryInsert(new FIRST(item, item)); } bool changed = false; do { changed = false; foreach (var regulation in grammar) { FIRST first = FindFirst(firstCollection, regulation.Left); int rightPartCount = regulation.RightPart.Count(); for (int checkCount = 0; checkCount < rightPartCount; checkCount++) { // 如果前面checkCount个结点都可为null, // 就说明RightPart[checkCount]的FIRST是此regulation.Left的FIRST的一部分。 if (Nullable(regulation.RightPart, 0, checkCount, nullableDict)) { FIRST refFirst = FindFirst(firstCollection, regulation.RightNode(checkCount)); if (refFirst == null) { throw new Exception("algorithm error!"); } if (refFirst != first) { foreach (var value in refFirst.Values) { if (value != TreeNodeType.NullNode) { changed = first.TryInsert(value) || changed; } } } } } } } while (changed); }
/// <summary> /// 计算文法的FIRST和FOLLOW集 /// </summary> /// <param name="grammar"></param> /// <returns></returns> public static FirstListAndFollowList GetFirstListAndFollowList(this RegulationList grammar) { Dictionary <TreeNodeType, bool> nullableDict = new Dictionary <TreeNodeType, bool>(); GetNullableDict(grammar, out nullableDict); FIRSTCollection firstCollection4Node; grammar.GetFirstCollection4Node(out firstCollection4Node, nullableDict); FIRSTCollection firstCollection4Regulation; grammar.GetFirstCollection4Regulation(out firstCollection4Regulation, nullableDict, firstCollection4Node); FOLLOWCollection followCollection; grammar.GetFollowCollection(out followCollection, nullableDict, firstCollection4Node); var firstCollection = new FIRSTCollection(); foreach (var item in firstCollection4Node) { firstCollection.TryInsert(item); } foreach (var item in firstCollection4Regulation) { firstCollection.TryInsert(item); } return(new FirstListAndFollowList(firstCollection, followCollection)); }
public SyntaxTree Parse(TokenList tokenList) { LRParsingMap parsingMap = GetParsingMap(); RegulationList grammar = GetGrammar(); //TODO:这个convertor以后是可以配置的。 var tokenTypeConvertor = new TokenType2TreeNodeType(); var context = new ParsingContext(tokenList, grammar, parsingMap, tokenTypeConvertor); while (context.CurrentTokenIndex < context.TokenList.Count + 1) { PrintParsingProgress(context); TreeNodeType nodeType = context.CurrentNodeType(); int stateId = context.StateIdStack.Peek(); LRParsingAction action = parsingMap.GetAction(stateId, nodeType); int currentTokenIndex = action.Execute(context); context.CurrentTokenIndex = currentTokenIndex; } PrintLastState(context); if (context.TreeStack.Count > 0) { return(context.TreeStack.Peek()); } else { return(new SyntaxTree()); } }
public ParsingContext(TokenList tokenList, RegulationList grammar, LRParsingMap parsingMap, TokenType2TreeNodeType tokenTypeConvertor) { this.StateIdStack = new Stack <int>(); this.StateIdStack.Push(1); this.TreeStack = new Stack <SyntaxTree>(); this.TokenList = tokenList; this.Grammar = grammar; this.ParsingMap = parsingMap; this.TokenTypeConvertor = tokenTypeConvertor; }
/// <summary> /// LR(0)的Goto操作。 /// 将圆点移到所有LR(0)项中的符号<paramref name="x"/>之后。 /// </summary> /// <param name="list"></param> /// <param name="state"></param> /// <param name="x">一个文法符号,终结点或非终结点。</param> /// <returns></returns> static LR0State Goto(this RegulationList list, LR0State state, TreeNodeType x) { LR0State toState = new LR0State(); foreach (var item in state) { TreeNodeType nextNode = item.GetNodeNext2Dot(); if (nextNode == x) { var newItem = new LR0Item(item.Regulation, item.DotPosition + 1); toState.TryInsert(newItem); } } return(Closure(list, toState)); }
/// <summary> /// 计算文法的各个RightPart的FIRST集 /// </summary> /// <param name="grammar"></param> /// <param name="nullableDict"></param> /// <param name="firstList4Node"></param> /// <returns></returns> private static void GetFirstCollection4Regulation( this RegulationList grammar, out FIRSTCollection firstCollection, Dictionary <TreeNodeType, bool> nullableDict = null, FIRSTCollection firstList4Node = null) { // 初始化FIRST集 firstCollection = new FIRSTCollection(); foreach (var regulation in grammar) { firstCollection.TryInsert(new FIRST(regulation.RightPart)); } bool changed = false; do { changed = false; foreach (var first in firstCollection) { int count = first.Target.Count(); for (int i = 0; i < count; i++) { // 如果前i个结点都可为null,就说明下一个结点(first.Target[i])的FIRST是此RightPart的FIRST的一部分 if (Nullable(first.Target, 0, i, nullableDict)) { // 找到下一个结点的FIRST FIRST refFirst = FindFirst(firstList4Node, first.GetNode(i)); foreach (var value in refFirst.Values) { if (value != TreeNodeType.NullNode) { changed = first.TryInsert(value) || changed; } } } } { // 如果RightPart的全部结点都可为null,就说明此RightPart的FIRST包含"null"结点。 if (Nullable(first.Target, 0, first.Target.Count(), nullableDict)) { changed = first.TryInsert(TreeNodeType.NullNode) || changed; } } } } while (changed); }
/// <summary> /// LR(1)的Goto操作。 /// 将圆点移到所有LR(1)项中的符号<paramref name="x"/>之后。 /// </summary> /// <param name="list"></param> /// <param name="state"></param> /// <param name="x">一个文法符号,终结点或非终结点。</param> /// <param name="firstList"></param> /// <returns></returns> static SmallerLR1State Goto(this RegulationList list, SmallerLR1State state, TreeNodeType x, Dictionary <TreeNodeType, bool> nullableDict, FIRSTCollection firstList = null) { var toState = new SmallerLR1State(); foreach (var group in state.GetGroups()) { TreeNodeType nextNode = group.Item1.GetNodeNext2Dot(); if (nextNode == x) { toState.TryInsert( new LR0Item(group.Item1.Regulation, group.Item1.DotPosition + 1), group.Item2); } } return(Closure(list, toState, nullableDict, firstList)); }
/// <summary> /// LR(1)的Closure操作。 /// 补全一个状态。 /// </summary> /// <param name="grammar"></param> /// <param name="state"></param> /// <returns></returns> static SmallerLR1State Closure(this RegulationList grammar, SmallerLR1State state, Dictionary <TreeNodeType, bool> nullableDict = null, FIRSTCollection firstCollection = null) { if (nullableDict == null) { grammar.GetNullableDict(out nullableDict); } if (firstCollection == null) { grammar.GetFirstCollection(out firstCollection, nullableDict); } Queue <LR1Item> queue = new Queue <LR1Item>(); foreach (var item in state) { queue.Enqueue(item); } while (queue.Count > 0) { LR1Item item = queue.Dequeue(); TreeNodeType node = item.GetNodeNext2Dot(); if (node == null || node.IsLeave) { continue; } List <TreeNodeType> betaZ = item.GetBetaZ(); FIRST first = grammar.GetFirst(firstCollection, nullableDict, betaZ); List <Regulation> regulations = grammar.GetRegulations(node); foreach (var regulation in regulations) { foreach (var value in first.Values) { LR1Item newItem = new LR1Item(regulation, 0, value); if (state.TryInsert(newItem)) { queue.Enqueue(newItem); } } } } return(state); }
/// <summary> /// 根据文法和已经计算的所有结点的FIRST集,计算任意一个结点串的FIRST。 /// </summary> /// <param name="grammar"></param> /// <param name="firstList4Node"></param> /// <param name="target"></param> /// <returns></returns> private static FIRST GetFirst(this RegulationList grammar, FIRSTCollection firstList4Node, Dictionary <TreeNodeType, bool> nullableDict, List <TreeNodeType> target) { // 初始化FIRST集 FIRST first = new FIRST(target); bool changed = false; do { changed = false; int count = first.Target.Count(); for (int checkCount = 0; checkCount < count; checkCount++) { // 如果前i个结点都可为null,就说明下一个结点(first.Target[i])的FIRST是此RightPart的FIRST的一部分 if (Nullable(first.Target, 0, checkCount, nullableDict)) { // 找到下一个结点的FIRST FIRST refFirst = FindFirst(firstList4Node, first.GetNode(checkCount)); foreach (var value in refFirst.Values) { if (value != TreeNodeType.NullNode) { changed = first.TryInsert(value) || changed; } } } } { // 如果RightPart的全部结点都可为null,就说明此RightPart的FIRST包含"null"结点。 if (Nullable(first.Target, 0, first.Target.Count(), nullableDict)) { //throw new Exception("如果是在LR1分析器生成工具的设计时走到此处,说明代码有问题。"); if (!first.Values.Contains(TreeNodeType.NullNode)) { changed = first.TryInsert(TreeNodeType.NullNode) || changed; } } } } while (changed); return(first); }
public static void GetFollowCollection( this RegulationList grammar, out FOLLOWCollection followCollection, Dictionary <TreeNodeType, bool> nullableDict = null, FIRSTCollection firstCollection = null) { if (nullableDict == null) { grammar.GetNullableDict(out nullableDict); } if (firstCollection == null) { grammar.GetFirstCollection(out firstCollection, nullableDict); } FIRSTCollection firstList4Node; grammar.GetFirstCollection4Node(out firstList4Node, nullableDict); FIRSTCollection firstList4Regulation; grammar.GetFirstCollection4Regulation(out firstList4Regulation, nullableDict, firstList4Node); grammar.DoGetFollowList(out followCollection, nullableDict, firstList4Node); }
/// <summary> /// 计算文法的FOLLOW集 /// </summary> /// <param name="grammar"></param> /// <param name="nullableDict"></param> /// <param name="firstList4Node"></param> /// <returns></returns> private static void DoGetFollowList( this RegulationList grammar, out FOLLOWCollection followCollection, Dictionary <TreeNodeType, bool> nullableDict, FIRSTCollection firstList4Node) { // 初始化Follow list followCollection = new FOLLOWCollection(); foreach (var item in grammar.GetAllTreeNodeNonLeaveTypes()) { followCollection.TryInsert(new FOLLOW(item)); } // 迭代到不动点 bool changed = false; do { changed = false; foreach (var regulation in grammar) { int count = regulation.RightPart.Count(); for (int index = 0; index < count; index++) { // 准备为target添加follow元素 TreeNodeType target = regulation.RightNode(index); if (target.IsLeave) { continue; } // 叶结点没有follow FOLLOW follow = FindFollow(followCollection, target); // 找到follow对象 for (int checkCount = 0; checkCount < count - (index + 1); checkCount++) { if (Nullable(regulation.RightPart, index + 1, checkCount, nullableDict)) { // nullable之后的FIRST是target的follow的一部分 FIRST first = FindFirst( firstList4Node, regulation.RightNode(index + 1 + checkCount)); foreach (var value in first.Values) { if (value != TreeNodeType.NullNode) { changed = follow.TryInsert(value) || changed; } } } } { // 如果target之后的全部结点都是nullable,说明此regulation.Left的folow也是target的一部分。 if (Nullable(regulation.RightPart, index + 1, count - (index + 1), nullableDict)) { // 找到此regulation.Left的folow FOLLOW refFollow = FindFollow(followCollection, regulation.Left); if (refFollow != follow) { foreach (var item in refFollow.Values) { changed = follow.TryInsert(item) || changed; } } } } } } } while (changed); }
/// <summary> /// 用SLR分析法计算分析表 /// </summary> /// <param name="grammar"></param> /// <returns></returns> public static void GetSLRParsingMap(this RegulationList grammar, out LRParsingMap map, out LR0StateCollection stateCollection, out LR0EdgeCollection edgeCollection, TextWriter writer) { // 给文法添加一个辅助的开始产生式 S' -> S $ // 如何添加一个外来的结点类型?用Enum是无法做到的。 var decoratedS = new TreeNodeType("__S2", "S'", "<S'>"); var decoratedEnd = TreeNodeType.endOfTokenListNode; var decoratedRegulation = new Regulation( decoratedS, grammar[0].Left, decoratedEnd); var decoratedGrammar = new RegulationList(decoratedRegulation); decoratedGrammar.AddRange(grammar); // 初始化T为{ Closure(S' -> S $) } var firstItem = new LR0Item(decoratedGrammar[0], 0); var firstState = new LR0State(firstItem); firstState = decoratedGrammar.Closure(firstState); stateCollection = new LR0StateCollection(firstState); edgeCollection = new LR0EdgeCollection(stateCollection); var queue = new Queue <LR0State>(); queue.Enqueue(firstState); int lastOutputLength = 0; int stateListCount = 1; int queueCount = 1; while (queue.Count > 0) { LR0State fromState = queue.Dequeue(); queueCount--; int itemIndex = 0; int itemCount = fromState.Count(); foreach (var item in fromState) { { TextWriter currentWriter = Console.Out; if (Console.Out != writer) { Console.SetOut(writer); } for (int i = 0; i < lastOutputLength; i++) { Console.Write('\u0008'); } string output = string.Format("Calculating SLR State List: {0} <-- {1}, working on {2}/{3} ...", stateListCount, queueCount, 1 + itemIndex++, itemCount); Console.Write(output); lastOutputLength = output.Length; Console.SetOut(currentWriter); } TreeNodeType x = item.GetNodeNext2Dot(); if (x == null || x == decoratedEnd) { continue; } LR0State toState = decoratedGrammar.Goto(fromState, x); if (stateCollection.TryInsert(toState)) { queue.Enqueue(toState); stateListCount++; queueCount++; var edge = new LR0Edge(fromState, x, toState); edgeCollection.TryInsert(edge); } else { int index = stateCollection.IndexOf(toState); toState = stateCollection[index]; var edge = new LR0Edge(fromState, x, toState); edgeCollection.TryInsert(edge); } } } { TextWriter currentWriter = Console.Out; if (Console.Out != writer) { Console.SetOut(writer); } Console.WriteLine(); Console.SetOut(currentWriter); } map = new LRParsingMap(); foreach (var edge in edgeCollection) { if (edge.X.IsLeave) { int stateId = edge.From.ParsingMapIndex + 1; //stateCollection.IndexOf(edge.From) + 1 int gotoStateId = edge.To.ParsingMapIndex + 1; //stateCollection.IndexOf(edge.To) + 1 map.SetAction(stateId, edge.X, new LR1ShiftInAction(gotoStateId)); } else { int stateId = edge.From.ParsingMapIndex + 1; //stateCollection.IndexOf(edge.From) + 1 int gotoStateId = edge.To.ParsingMapIndex + 1; //stateCollection.IndexOf(edge.To) + 1 map.SetAction(stateId, edge.X, new LR1GotoAction(gotoStateId)); } } var endItem = new LR0Item(decoratedRegulation, 1); FOLLOWCollection followCollection; decoratedGrammar.GetFollowCollection(out followCollection); foreach (var state in stateCollection) { if (state.Contains(endItem)) { int stateId = state.ParsingMapIndex + 1;//stateCollection.IndexOf(state) + 1 map.SetAction(stateId, decoratedEnd, new LR1AceptAction()); } foreach (var lr0Item in state) { if (lr0Item.GetNodeNext2Dot() == null) { FOLLOW follow = FindFollow(followCollection, lr0Item.Regulation.Left); foreach (var value in follow.Values) { int stateId = state.ParsingMapIndex + 1;// stateCollection.IndexOf(state) + 1; int reductionId = decoratedGrammar.IndexOf(lr0Item.Regulation); var action = new LR1ReducitonAction(reductionId); map.SetAction(stateId, value, action); } } } } }
/// <summary> /// 用LR(1)分析法计算分析表 /// </summary> /// <param name="grammar"></param> /// <returns></returns> public static void GetLR1ParsingMap(this RegulationList grammar, out LRParsingMap map, out LR1StateCollection stateCollection, out LR1EdgeCollection edgeCollection, TextWriter originalOut) { // 给文法添加一个辅助的开始产生式 S' -> S $ // 如何添加一个外来的结点类型?用Enum是无法做到的。 var decoratedS = new TreeNodeType("__S2", "S'", "<S'>"); var decoratedEnd = TreeNodeType.endOfTokenListNode; var decoratedRegulation = new Regulation( decoratedS, grammar[0].Left, decoratedEnd); var decoratedGrammar = new RegulationList(decoratedRegulation); decoratedGrammar.AddRange(grammar); // 初始化T为{ Closure(S' -> S $) } var firstItem = new LR1Item(decoratedGrammar[0], 0, decoratedEnd); var firstState = new SmallerLR1State(firstItem); Dictionary <TreeNodeType, bool> nullableDict; decoratedGrammar.GetNullableDict(out nullableDict); FIRSTCollection firstCollection; decoratedGrammar.GetFirstCollection(out firstCollection, nullableDict); firstState = decoratedGrammar.Closure(firstState, nullableDict, firstCollection); stateCollection = new LR1StateCollection(firstState); edgeCollection = new LR1EdgeCollection(stateCollection); var queue = new Queue <SmallerLR1State>(); queue.Enqueue(firstState); int lastOutputLength = 0; int stateListCount = 1; int queueCount = 1; while (queue.Count > 0) { SmallerLR1State fromState = queue.Dequeue(); queueCount--; int itemIndex = 0; int groupCount = fromState.GroupCount; foreach (var group in fromState.GetGroups()) { { TextWriter currentWriter = Console.Out; if (Console.Out != originalOut) { Console.SetOut(originalOut); } for (int i = 0; i < lastOutputLength; i++) { Console.Write('\u0008'); } string output = string.Format("Calculating LR(1) State List: {0} <-- {1}, working on group {2}/{3} ...", stateListCount, queueCount, 1 + itemIndex++, groupCount); Console.Write(output); lastOutputLength = output.Length; Console.SetOut(currentWriter); } TreeNodeType x = group.Item1.GetNodeNext2Dot(); if (x == null || x == decoratedEnd) { continue; } SmallerLR1State toState = decoratedGrammar.Goto(fromState, x, nullableDict, firstCollection); if (stateCollection.TryInsert(toState))//融入组织之中吧 { int index = stateCollection.IndexOf(toState); toState = stateCollection[index]; queue.Enqueue(toState); stateListCount++; queueCount++; var edge = new LR1Edge(fromState, x, toState); edgeCollection.TryInsert(edge); } else { int index = stateCollection.IndexOf(toState); toState = stateCollection[index]; var edge = new LR1Edge(fromState, x, toState); edgeCollection.TryInsert(edge); } } } { TextWriter currentWriter = Console.Out; if (Console.Out != originalOut) { Console.SetOut(originalOut); } Console.WriteLine(); Console.SetOut(currentWriter); } map = new LRParsingMap(); foreach (var edge in edgeCollection) { if (edge.X.IsLeave) { int stateId = edge.From.ParsingMapIndex + 1; // stateCollection.IndexOf(edge.From) + 1; int gotoId = edge.To.ParsingMapIndex + 1; //stateCollection.IndexOf(edge.To) + 1 map.SetAction(stateId, edge.X, new LR1ShiftInAction(gotoId)); } else { int stateId = edge.From.ParsingMapIndex + 1; // stateCollection.IndexOf(edge.From) + 1; int gotoId = edge.To.ParsingMapIndex + 1; //stateCollection.IndexOf(edge.To) + 1 map.SetAction(stateId, edge.X, new LR1GotoAction(gotoId)); } } // TODO: not implemented var endItem = new LR1Item(decoratedRegulation, 1, decoratedEnd); foreach (var state in stateCollection) { if (state.Contains(endItem)) { int stateId = state.ParsingMapIndex + 1;// stateCollection.IndexOf(state) + 1; map.SetAction(stateId, decoratedEnd, new LR1AceptAction()); } foreach (var LR1Item in state) { if (LR1Item.GetNodeNext2Dot() == null) { int stateId = state.ParsingMapIndex + 1;// stateCollection.IndexOf(state) + 1; map.SetAction(stateId, LR1Item.LookAheadNodeType, new LR1ReducitonAction(decoratedGrammar.IndexOf(LR1Item.Regulation))); } } } }