public void Expect(CDILToken token) { if (TokenType == token) { GetNextToken(); } else { throw BuildException("'" + token + "' expected"); } }
public void GetNextToken() { if (_tokenType == CDILToken.EOF) { throw BuildException("Cannot move past EOF in CDIL"); } SkipWhitespace(); int p = PeekChar(); if (p == -1) { _tokenType = CDILToken.EOF; return; } char ch = (char)p; if (ch == '\'') { ReadChar(); string text = ""; while ((p = ReadChar()) != -1) { ch = (char)p; if (ch == '\'') { break; } if (ch == '\\') { text += (char)ReadChar(); } else { text += ch; } } _tokenType = CDILToken.String; _tokenValue = text; return; } if (Char.IsNumber(ch) || ch == '-') { string text = ""; if (ch == '-') { text = "-"; ReadChar(); } while ((p = PeekChar()) != -1 && Char.IsNumber((char)p)) { ch = (char)p; ReadChar(); text += ch; } if (text == "-") { throw BuildException("Number expected after -"); } _tokenType = CDILToken.Integer; _tokenValue = Convert.ToInt32(text); return; } switch (ch) { case '(': ReadChar(); _tokenType = CDILToken.LeftParen; return; case ')': ReadChar(); _tokenType = CDILToken.RightParen; return; case '=': ReadChar(); _tokenType = CDILToken.Assign; return; case ',': ReadChar(); _tokenType = CDILToken.Comma; return; case '.': ReadChar(); _tokenType = CDILToken.Dot; return; case '$': ReadChar(); _tokenType = CDILToken.Dollar; return; case ';': ReadChar(); _tokenType = CDILToken.Semicolon; return; default: break; } if (Char.IsLetter(ch) || ch == '_') { string tokenName = ""; do { tokenName += (char)ReadChar(); ch = (char)PeekChar(); }while (Char.IsLetterOrDigit(ch) || ch == '_'); _tokenValue = tokenName; _tokenType = CDILToken.Keyword; return; } throw BuildException("Unrecognized character: " + ch); }
public CDILTokenizer() { _tokenType = CDILToken.BOF; }
public void GetNextToken() { if (_tokenType == CDILToken.EOF) { throw BuildException("Cannot move past EOF in CDIL"); } SkipWhitespace(); int p = PeekChar(); if (p == -1) { _tokenType = CDILToken.EOF; return; } char ch = (char)p; if (ch == '\'') { ReadChar(); string text = ""; while ((p = ReadChar()) != -1) { ch = (char)p; if (ch == '\'') break; if (ch == '\\') text += (char)ReadChar(); else text += ch; } _tokenType = CDILToken.String; _tokenValue = text; return; } if (Char.IsNumber(ch) || ch == '-') { string text = ""; if (ch == '-') { text = "-"; ReadChar(); } while ((p = PeekChar()) != -1 && Char.IsNumber((char)p)) { ch = (char)p; ReadChar(); text += ch; } if (text == "-") throw BuildException("Number expected after -"); _tokenType = CDILToken.Integer; _tokenValue = Convert.ToInt32(text); return; } switch (ch) { case '(': ReadChar(); _tokenType = CDILToken.LeftParen; return; case ')': ReadChar(); _tokenType = CDILToken.RightParen; return; case '=': ReadChar(); _tokenType = CDILToken.Assign; return; case ',': ReadChar(); _tokenType = CDILToken.Comma; return; case '.': ReadChar(); _tokenType = CDILToken.Dot; return; case '$': ReadChar(); _tokenType = CDILToken.Dollar; return; case ';': ReadChar(); _tokenType = CDILToken.Semicolon; return; default: break; } if (Char.IsLetter(ch) || ch == '_') { string tokenName = ""; do { tokenName += (char)ReadChar(); ch = (char)PeekChar(); } while (Char.IsLetterOrDigit(ch) || ch == '_'); _tokenValue = tokenName; _tokenType = CDILToken.Keyword; return; } throw BuildException("Unrecognized character: " + ch); }
public void Expect(CDILToken token) { if (TokenType == token) GetNextToken(); else throw BuildException("'" + token + "' expected"); }