public void Expect(CDILToken token)
{
if (TokenType == token)
{
GetNextToken();
}
else
{
throw BuildException("'" + token + "' expected");
}
}
public void GetNextToken()
{
if (_tokenType == CDILToken.EOF)
{
throw BuildException("Cannot move past EOF in CDIL");
}
SkipWhitespace();
int p = PeekChar();
if (p == -1)
{
_tokenType = CDILToken.EOF;
return;
}
char ch = (char)p;
if (ch == '\'')
{
ReadChar();
string text = "";
while ((p = ReadChar()) != -1)
{
ch = (char)p;
if (ch == '\'')
{
break;
}
if (ch == '\\')
{
text += (char)ReadChar();
}
else
{
text += ch;
}
}
_tokenType = CDILToken.String;
_tokenValue = text;
return;
}
if (Char.IsNumber(ch) || ch == '-')
{
string text = "";
if (ch == '-')
{
text = "-";
ReadChar();
}
while ((p = PeekChar()) != -1 && Char.IsNumber((char)p))
{
ch = (char)p;
ReadChar();
text += ch;
}
if (text == "-")
{
throw BuildException("Number expected after -");
}
_tokenType = CDILToken.Integer;
_tokenValue = Convert.ToInt32(text);
return;
}
switch (ch)
{
case '(':
ReadChar();
_tokenType = CDILToken.LeftParen;
return;
case ')':
ReadChar();
_tokenType = CDILToken.RightParen;
return;
case '=':
ReadChar();
_tokenType = CDILToken.Assign;
return;
case ',':
ReadChar();
_tokenType = CDILToken.Comma;
return;
case '.':
ReadChar();
_tokenType = CDILToken.Dot;
return;
case '$':
ReadChar();
_tokenType = CDILToken.Dollar;
return;
case ';':
ReadChar();
_tokenType = CDILToken.Semicolon;
return;
default:
break;
}
if (Char.IsLetter(ch) || ch == '_')
{
string tokenName = "";
do
{
tokenName += (char)ReadChar();
ch = (char)PeekChar();
}while (Char.IsLetterOrDigit(ch) || ch == '_');
_tokenValue = tokenName;
_tokenType = CDILToken.Keyword;
return;
}
throw BuildException("Unrecognized character: " + ch);
}
public CDILTokenizer()
{
_tokenType = CDILToken.BOF;
}
public void GetNextToken()
{
if (_tokenType == CDILToken.EOF)
{
throw BuildException("Cannot move past EOF in CDIL");
}
SkipWhitespace();
int p = PeekChar();
if (p == -1)
{
_tokenType = CDILToken.EOF;
return;
}
char ch = (char)p;
if (ch == '\'')
{
ReadChar();
string text = "";
while ((p = ReadChar()) != -1)
{
ch = (char)p;
if (ch == '\'')
break;
if (ch == '\\')
text += (char)ReadChar();
else
text += ch;
}
_tokenType = CDILToken.String;
_tokenValue = text;
return;
}
if (Char.IsNumber(ch) || ch == '-')
{
string text = "";
if (ch == '-')
{
text = "-";
ReadChar();
}
while ((p = PeekChar()) != -1 && Char.IsNumber((char)p))
{
ch = (char)p;
ReadChar();
text += ch;
}
if (text == "-") throw BuildException("Number expected after -");
_tokenType = CDILToken.Integer;
_tokenValue = Convert.ToInt32(text);
return;
}
switch (ch)
{
case '(':
ReadChar();
_tokenType = CDILToken.LeftParen;
return;
case ')':
ReadChar();
_tokenType = CDILToken.RightParen;
return;
case '=':
ReadChar();
_tokenType = CDILToken.Assign;
return;
case ',':
ReadChar();
_tokenType = CDILToken.Comma;
return;
case '.':
ReadChar();
_tokenType = CDILToken.Dot;
return;
case '$':
ReadChar();
_tokenType = CDILToken.Dollar;
return;
case ';':
ReadChar();
_tokenType = CDILToken.Semicolon;
return;
default:
break;
}
if (Char.IsLetter(ch) || ch == '_')
{
string tokenName = "";
do
{
tokenName += (char)ReadChar();
ch = (char)PeekChar();
}
while (Char.IsLetterOrDigit(ch) || ch == '_');
_tokenValue = tokenName;
_tokenType = CDILToken.Keyword;
return;
}
throw BuildException("Unrecognized character: " + ch);
}
public CDILTokenizer()
{
_tokenType = CDILToken.BOF;
}
public void Expect(CDILToken token)
{
if (TokenType == token)
GetNextToken();
else
throw BuildException("'" + token + "' expected");
}
