/// <summary> /// Retrieves a token from the input string. This method can be called multiple /// time to get tokens further on the input string. /// </summary> /// <returns>The token</returns> public TerminalToken RetrieveToken() { dfa.Reset(); Location startLocation = input.Location.Clone(); AcceptInfo acceptInfo = null; if (input.Position >= input.Text.Length) { return(new TerminalToken(SymbolCollection.EOF, SymbolCollection.EOF.Name, startLocation)); } State newState = dfa.GotoNext(input.ReadChar()); while (newState != null) { if (newState is EndState) { acceptInfo = new AcceptInfo((EndState)newState, input.Location.Clone()); } if (input.IsEof()) { newState = null; } else { newState = dfa.GotoNext(input.ReadChar()); } } if (acceptInfo == null) { int len = input.Location.Position - startLocation.Position; string text = input.Text.Substring(startLocation.Position, len); return(new TerminalToken(SymbolCollection.ERROR, text, startLocation)); } else { input.Location = acceptInfo.Location; int len = acceptInfo.Location.Position - startLocation.Position; string text = input.Text.Substring(startLocation.Position, len); return(new TerminalToken(acceptInfo.State.AcceptSymbol, text, startLocation)); } }
/// <summary> /// Retrieves a token from the input string. This method can be called multiple /// time to get tokens further on the input string. /// </summary> /// <returns>The token</returns> public TerminalToken RetrieveToken() { dfa.Reset(); Location startLocation = input.Location.Clone(); AcceptInfo acceptInfo = null; if (input.Position >= input.Text.Length) { return new TerminalToken(SymbolCollection.EOF, SymbolCollection.EOF.Name, startLocation); } State newState = dfa.GotoNext(input.ReadChar()); while (newState != null) { if (newState is EndState) { acceptInfo = new AcceptInfo((EndState)newState,input.Location.Clone()); } if (input.IsEof()) newState = null; else newState = dfa.GotoNext(input.ReadChar()); } if (acceptInfo == null) { int len = input.Location.Position - startLocation.Position; string text = input.Text.Substring(startLocation.Position,len); return new TerminalToken(SymbolCollection.ERROR,text,startLocation); } else { input.Location = acceptInfo.Location; int len = acceptInfo.Location.Position - startLocation.Position; string text = input.Text.Substring(startLocation.Position,len); return new TerminalToken(acceptInfo.State.AcceptSymbol,text,startLocation); } }