private bool DoReduction(Stack <Token> tokenStack, LookAheadBuffer lookAheadBuffer, ref ushort lalrState, LRAction parseAction) { Token head = null; //Produce a reduction - remove as many tokens as members in the rule & push a nonterminal token Production prod = egtDataManager.GetProduction(parseAction.Value); bool reductionSkipped = false; //Skip reduction? if ((TrimReductions && prod.ConsistsOfOneNonTerminal()) || (SkipEmptyProductions && prod.Handle.Count == 0)) { head = tokenStack.Pop(); head.Symbol = prod.Head; reductionSkipped = true; } //Build a Reduction else { Reduction reduction = new Reduction(prod.Handle.Count, prod); //If a production has no handles, it has no location. //Set a would-be location instead. if (prod.Handle.Count == 0) { reduction.StartPosition = reduction.EndPosition = lookAheadBuffer.Position; } else { for (int i = prod.Handle.Count - 1; i >= 0; i--) { reduction[i] = tokenStack.Pop(); } } head = reduction; } //If n is -1 here, then we have an Internal Table Error!!! LRAction action = egtDataManager.GetLRActionList(tokenStack.Peek().State)[prod.Head]; if (action == null) { OnInternalError(); return(false); } lalrState = action.Value; head.State = lalrState; tokenStack.Push(head); if (!reductionSkipped) { OnReduction((Reduction)tokenStack.Peek()); } return(true); }
public bool Parse(TextReader reader) { if (egtDataManager == null) { OnNotLoadedError(); return false; } TokenQueueStack inputTokens = new TokenQueueStack(); Stack<Token> tokenStack = new Stack<Token>(); tokenStack.Push(new Token()); Stack<Token> groupStack = new Stack<Token>(); LookAheadBuffer lookAheadBuffer = new LookAheadBuffer(reader); ushort lalrState = 0; while (true) { if (inputTokens.Count == 0) { inputTokens.Push(ProduceToken(groupStack, lookAheadBuffer)); OnTokenRead(inputTokens.Peek()); } //Runaway group else if (groupStack.Count != 0) { OnGroupError(); return false; //Error; abort } else { Token read = inputTokens.Peek(); switch (read.Type) { case SymbolType.Noise: inputTokens.Pop(); break; case SymbolType.Error: OnLexicalError(read.EndPosition, read.Data); return false; //Error; abort default: LRActionList actionList = egtDataManager.GetLRActionList(lalrState); LRAction parseAction = actionList[read.Symbol]; if (parseAction == null) { HandleSyntaxError(actionList, read); return false; } switch (parseAction.Type) { case LRActionType.Accept: OnCompleted(tokenStack.Peek().Tag); return true; case LRActionType.Shift: lalrState = parseAction.Value; read.State = lalrState; tokenStack.Push(read); //It now exists on the Token-Stack and must be eliminated from the queue. inputTokens.Dequeue(); break; case LRActionType.Reduce: if (!DoReduction(tokenStack, lookAheadBuffer, ref lalrState, parseAction)) { return false; } break; } break; } } } }
private Token ProduceToken(Stack<Token> groupStack, LookAheadBuffer lookAheadBuffer) { while (true) { Token read = LookAheadDFA(lookAheadBuffer); if (read.Type == SymbolType.GroupStart && (groupStack.Count == 0 || groupStack.Peek().Group.Nesting.Contains(read.Group.TableIndex))) { lookAheadBuffer.Consume(read.Data.Length); read.EndPosition = lookAheadBuffer.Position; groupStack.Push(read); } else if (groupStack.Count == 0) { //The token is ready to be analyzed. lookAheadBuffer.Consume(read.Data.Length); read.EndPosition = lookAheadBuffer.Position; return read; } else if (Object.ReferenceEquals(groupStack.Peek().Group.End, read.Symbol)) { //End the current group Token pop = groupStack.Pop(); if (pop.Group.Ending == GroupEndingMode.Closed) { pop.Data += read.Data; lookAheadBuffer.Consume(read.Data.Length); read.EndPosition = lookAheadBuffer.Position; } //We are out of the group. Return pop'd token (which contains all the group text) if (groupStack.Count == 0) { //Change symbol to parent pop.Symbol = pop.Group.Container; return pop; } else { groupStack.Peek().Data += pop.Data; } } else if (read.Type == SymbolType.End) { return read; } else { //We are in a group, Append to the Token on the top of the stack. //Take into account the Token group mode Token top = groupStack.Peek(); if (top.Group.Advance == GroupAdvanceMode.Token) { // Append all text top.Data += read.Data; lookAheadBuffer.Consume(read.Data.Length); } else { // Append one character top.Data += read.Data[0].ToString(); lookAheadBuffer.Consume(1); } read.EndPosition = lookAheadBuffer.Position; } } }
//This function implements the DFA for th parser's lexer. //It generates a token which is used by the LALR state //machine. private Token LookAheadDFA(LookAheadBuffer lookAheadBuffer) { string ch = lookAheadBuffer.LookAhead(1); if (String.IsNullOrEmpty(ch) || ch[0] == Char.MaxValue) { return new Token(egtDataManager.GetFirstSymbolOfType(SymbolType.End), String.Empty) { StartPosition = lookAheadBuffer.Position, EndPosition = lookAheadBuffer.Position }; } int currentDFA = 0; int lookAheadPosition = 1; //Next byte in the input Stream int lastAcceptState = -1; //We have not yet accepted a character string int lastAcceptPosition = -1; while (true) { bool found = false; ch = lookAheadBuffer.LookAhead(lookAheadPosition); if (!String.IsNullOrEmpty(ch)) { foreach (FAEdge edge in egtDataManager.GetFAState(currentDFA).Edges) { //Look for character in the Character Set Table if (edge.Characters.Contains(ch[0])) { int target = edge.Target; if (egtDataManager.GetFAState(target).Accept != null) { lastAcceptState = target; lastAcceptPosition = lookAheadPosition; } currentDFA = target; lookAheadPosition++; found = true; break; } } } if (!found) { Symbol symbol = null; string data = String.Empty; // Lexer cannot recognize symbol if (lastAcceptState == -1) { symbol = egtDataManager.GetFirstSymbolOfType(SymbolType.Error); data = lookAheadBuffer.GetTextFromBuffer(1); } else { symbol = egtDataManager.GetFAState(lastAcceptState).Accept; data = lookAheadBuffer.GetTextFromBuffer(lastAcceptPosition); } return new Token(symbol, data) { StartPosition = lookAheadBuffer.Position }; } } }
private bool DoReduction(Stack<Token> tokenStack, LookAheadBuffer lookAheadBuffer, ref ushort lalrState, LRAction parseAction) { Token head = null; //Produce a reduction - remove as many tokens as members in the rule & push a nonterminal token Production prod = egtDataManager.GetProduction(parseAction.Value); bool reductionSkipped = false; //Skip reduction? if ((TrimReductions && prod.ConsistsOfOneNonTerminal()) || (SkipEmptyProductions && prod.Handle.Count == 0)) { head = tokenStack.Pop(); head.Symbol = prod.Head; reductionSkipped = true; } //Build a Reduction else { Reduction reduction = new Reduction(prod.Handle.Count, prod); //If a production has no handles, it has no location. //Set a would-be location instead. if (prod.Handle.Count == 0) { reduction.StartPosition = reduction.EndPosition = lookAheadBuffer.Position; } else { for (int i = prod.Handle.Count - 1; i >= 0; i--) { reduction[i] = tokenStack.Pop(); } } head = reduction; } //If n is -1 here, then we have an Internal Table Error!!! LRAction action = egtDataManager.GetLRActionList(tokenStack.Peek().State)[prod.Head]; if (action == null) { OnInternalError(); return false; } lalrState = action.Value; head.State = lalrState; tokenStack.Push(head); if (!reductionSkipped) { OnReduction((Reduction)tokenStack.Peek()); } return true; }
public bool Parse(TextReader reader) { if (egtDataManager == null) { OnNotLoadedError(); return(false); } TokenQueueStack inputTokens = new TokenQueueStack(); Stack <Token> tokenStack = new Stack <Token>(); tokenStack.Push(new Token()); Stack <Token> groupStack = new Stack <Token>(); LookAheadBuffer lookAheadBuffer = new LookAheadBuffer(reader); ushort lalrState = 0; while (true) { if (inputTokens.Count == 0) { inputTokens.Push(ProduceToken(groupStack, lookAheadBuffer)); OnTokenRead(inputTokens.Peek()); } //Runaway group else if (groupStack.Count != 0) { OnGroupError(); return(false); //Error; abort } else { Token read = inputTokens.Peek(); switch (read.Type) { case SymbolType.Noise: inputTokens.Pop(); break; case SymbolType.Error: OnLexicalError(read.EndPosition, read.Data); return(false); //Error; abort default: LRActionList actionList = egtDataManager.GetLRActionList(lalrState); LRAction parseAction = actionList[read.Symbol]; if (parseAction == null) { HandleSyntaxError(actionList, read); return(false); } switch (parseAction.Type) { case LRActionType.Accept: OnCompleted(tokenStack.Peek().Tag); return(true); case LRActionType.Shift: lalrState = parseAction.Value; read.State = lalrState; tokenStack.Push(read); //It now exists on the Token-Stack and must be eliminated from the queue. inputTokens.Dequeue(); break; case LRActionType.Reduce: if (!DoReduction(tokenStack, lookAheadBuffer, ref lalrState, parseAction)) { return(false); } break; } break; } } } }
private Token ProduceToken(Stack <Token> groupStack, LookAheadBuffer lookAheadBuffer) { while (true) { Token read = LookAheadDFA(lookAheadBuffer); if (read.Type == SymbolType.GroupStart && (groupStack.Count == 0 || groupStack.Peek().Group.Nesting.Contains(read.Group.TableIndex))) { lookAheadBuffer.Consume(read.Data.Length); read.EndPosition = lookAheadBuffer.Position; groupStack.Push(read); } else if (groupStack.Count == 0) { //The token is ready to be analyzed. lookAheadBuffer.Consume(read.Data.Length); read.EndPosition = lookAheadBuffer.Position; return(read); } else if (Object.ReferenceEquals(groupStack.Peek().Group.End, read.Symbol)) { //End the current group Token pop = groupStack.Pop(); if (pop.Group.Ending == GroupEndingMode.Closed) { pop.Data += read.Data; lookAheadBuffer.Consume(read.Data.Length); read.EndPosition = lookAheadBuffer.Position; } //We are out of the group. Return pop'd token (which contains all the group text) if (groupStack.Count == 0) { //Change symbol to parent pop.Symbol = pop.Group.Container; return(pop); } else { groupStack.Peek().Data += pop.Data; } } else if (read.Type == SymbolType.End) { return(read); } else { //We are in a group, Append to the Token on the top of the stack. //Take into account the Token group mode Token top = groupStack.Peek(); if (top.Group.Advance == GroupAdvanceMode.Token) { // Append all text top.Data += read.Data; lookAheadBuffer.Consume(read.Data.Length); } else { // Append one character top.Data += read.Data[0].ToString(); lookAheadBuffer.Consume(1); } read.EndPosition = lookAheadBuffer.Position; } } }
//This function implements the DFA for th parser's lexer. //It generates a token which is used by the LALR state //machine. private Token LookAheadDFA(LookAheadBuffer lookAheadBuffer) { string ch = lookAheadBuffer.LookAhead(1); if (String.IsNullOrEmpty(ch) || ch[0] == Char.MaxValue) { return(new Token(egtDataManager.GetFirstSymbolOfType(SymbolType.End), String.Empty) { StartPosition = lookAheadBuffer.Position, EndPosition = lookAheadBuffer.Position }); } int currentDFA = 0; int lookAheadPosition = 1; //Next byte in the input Stream int lastAcceptState = -1; //We have not yet accepted a character string int lastAcceptPosition = -1; while (true) { bool found = false; ch = lookAheadBuffer.LookAhead(lookAheadPosition); if (!String.IsNullOrEmpty(ch)) { foreach (FAEdge edge in egtDataManager.GetFAState(currentDFA).Edges) { //Look for character in the Character Set Table if (edge.Characters.Contains(ch[0])) { int target = edge.Target; if (egtDataManager.GetFAState(target).Accept != null) { lastAcceptState = target; lastAcceptPosition = lookAheadPosition; } currentDFA = target; lookAheadPosition++; found = true; break; } } } if (!found) { Symbol symbol = null; string data = String.Empty; // Lexer cannot recognize symbol if (lastAcceptState == -1) { symbol = egtDataManager.GetFirstSymbolOfType(SymbolType.Error); data = lookAheadBuffer.GetTextFromBuffer(1); } else { symbol = egtDataManager.GetFAState(lastAcceptState).Accept; data = lookAheadBuffer.GetTextFromBuffer(lastAcceptPosition); } return(new Token(symbol, data) { StartPosition = lookAheadBuffer.Position }); } } }