public Parser(SymbolTable symbols, Scanner scanner) { _symbols = symbols; _scanner = scanner; _lookahead = _scanner.ReadToken(); }
public Scanner(string filename, TextReader source, int tabsize) { // set up keyword map to TokenKind dictPop(); _source = source; _cursor = new Position(filename, 1, 0); // filename could be "(console)" if doing interactive interpretation _tabsize = tabsize; ReadChar(); // fetch initial lookahead character _nextToken = ScanToken(); // fetch initial lookahead token }
private Token ReadToken() { Token result = _lookahead; _lookahead = _scanner.ReadToken(); return result; }
/* ReadToken() scans a new token and returns the current lookahead token. */ public Token ReadToken() { Token result = _nextToken; _nextToken = ScanToken(); return result; }
private Token ScanToken() { Token result = new Token(_cursor); _text.Length = 0; char ch = ReadChar(); switch (ch) { // discard whitespace case ' ': case '\t': case '\n': case '\r': while (char.IsWhiteSpace(_nextChar)) ReadChar(); return ScanToken(); case ';': result.Kind = TokenKind.Semicolon; break; case ':': if (_nextChar == '=') { ReadChar(); result.Kind = TokenKind.Assignment; break; } result.Kind = TokenKind.Colon; break; case ',': result.Kind = TokenKind.Comma; break; case '~': result.Kind = TokenKind.Tilde; break; case '(': result.Kind = TokenKind.LeftParenthesis; break; case ')': result.Kind = TokenKind.RightParenthesis; break; case '+': result.Kind = TokenKind.Plus; break; case '-': result.Kind = TokenKind.Minus; break; case '*': result.Kind = TokenKind.Asterisk; break; case '/': result.Kind = TokenKind.Slash; break; case '<': result.Kind = TokenKind.LessThan; break; case '>': result.Kind = TokenKind.GreaterThan; break; case '=': result.Kind = TokenKind.Equal; break; case '\\': result.Kind = TokenKind.Backslash; break; case EOF: result.Kind = TokenKind.EndOfFile; break; case '"': while (_nextChar != '"') { if (_nextChar == '\n') throw new ScannerError(_cursor, "End of line in string literal"); else if (_nextChar == EOF) throw new ScannerError(_cursor, "End of file in string literal"); _text.Append(ReadChar()); } ReadChar(); // skip closing quote result.Kind = TokenKind.Literal_String; result.Text = _text.ToString(); break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': _text.Append(ch); while (_nextChar >= '0' && _nextChar <= '9') { _text.Append(_nextChar); ReadChar(); } result.Kind = TokenKind.Literal_Integer; result.Text = _text.ToString(); break; default: if (!Char.IsLetter(ch)) throw new ScannerError(_cursor, "Invalid character"); _text.Append(ch); while (char.IsLetterOrDigit(_nextChar)) { _text.Append(_nextChar); ReadChar(); } // check if the identifier is a keyword result.Text = _text.ToString(); if (_keywords.ContainsKey(result.Text)) result.Kind = _keywords[result.Text]; else result.Kind = TokenKind.Identifier; break; } return result; }