public TokenBuffer Tokenize(string input) { var buffer = new TokenBuffer(); foreach (var c in input) { var token = GetToken(c); if (token != null) buffer.Add(token); } buffer.Add(new EOF()); return buffer; }
public void Write(Token token) { TokenBuffer.Add(token); }