public PreprocessorKeywords(NameTable symbolTable) {
keywords = new Hashtable();
for (PreprocessorTokenType token = 0; token < PreprocessorTokenType.Identifier; token += 1) {
keywords.Add(symbolTable.Add(PreprocessorToken.TypeString(token)), token);
}
}
/// <summary>
/// Initializes Keywords object. Adds all keywords to _nameTable.
/// </summary>
public Keywords(NameTable nameTable) {
_keywords = new Hashtable((int)TokenType.Identifier);
_nameTable = nameTable;
for (TokenType token = 0; token < TokenType.Identifier; token += 1) {
_keywords.Add(_nameTable.Add(Token.GetString(token)), token);
}
}
public PreprocessorKeywords(NameTable symbolTable)
{
keywords = new Hashtable();
for (PreprocessorTokenType token = 0; token < PreprocessorTokenType.Identifier; token += 1)
{
keywords.Add(symbolTable.Add(PreprocessorToken.TypeString(token)), token);
}
}
/// <summary>
/// Initializes Keywords object. Adds all keywords to _nameTable.
/// </summary>
public Keywords(NameTable nameTable)
{
_keywords = new Hashtable((int)TokenType.Identifier);
_nameTable = nameTable;
for (TokenType token = 0; token < TokenType.Identifier; token += 1)
{
_keywords.Add(_nameTable.Add(Token.GetString(token)), token);
}
}
public PreprocessorToken NextToken(TextBuffer text)
{
_text = text;
SkipWhiteSpace();
BufferPosition position = text.Position;
char ch = PeekChar();
if (ch == '\0' || IsLineSeparator(ch))
{
return(NewPPToken(PreprocessorTokenType.EndOfLine, position));
}
ch = NextChar();
switch (ch)
{
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': {
int intValue = (ch - '0');
while (IsDigit(PeekChar()))
{
int value10 = intValue * 10;
if (value10 < intValue)
{
ReportError(LexError.NumericConstantOverflow);
}
else
{
intValue = value10 + (NextChar() - '0');
}
}
return(new PreprocessorIntToken(intValue, position));
}
case '=':
if (PeekChar() == '=')
{
NextChar();
return(NewPPToken(PreprocessorTokenType.EqualEqual, position));
}
break;
case '!':
if (PeekChar() == '=')
{
NextChar();
return(NewPPToken(PreprocessorTokenType.NotEqual, position));
}
else
{
return(NewPPToken(PreprocessorTokenType.Not, position));
}
case '&':
if (PeekChar() == '&')
{
NextChar();
return(NewPPToken(PreprocessorTokenType.And, position));
}
break;
case '|':
if (PeekChar() == '|')
{
NextChar();
return(NewPPToken(PreprocessorTokenType.Or, position));
}
break;
case '(':
return(NewPPToken(PreprocessorTokenType.OpenParen, position));
case ')':
return(NewPPToken(PreprocessorTokenType.CloseParen, position));
case '"':
_value.Length = 0;
while ((ch = PeekChar()) != '"')
{
if (EOF)
{
ReportError(LexError.UnexpectedEndOfFileString);
break;
}
else if (IsLineSeparator(ch))
{
ReportError(LexError.WhiteSpaceInConstant);
break;
}
_value.Append(ch);
NextChar();
}
NextChar();
return(new PreprocessorStringToken(_value.ToString(), position));
case '/':
if (PeekChar() == '/')
{
IgnoreRestOfLine();
return(NewPPToken(PreprocessorTokenType.EndOfLine, position));
}
break;
default:
if (IsLineSeparator(ch))
{
return(NewPPToken(PreprocessorTokenType.EndOfLine, position));
}
if (!IsIdentifierChar(ch))
{
break;
}
_value.Length = 0;
_value.Append(ch);
while (IsIdentifierChar(PeekChar()))
{
_value.Append(NextChar());
}
Name id = _nameTable.Add(_value);
PreprocessorTokenType type = _keywords.IsKeyword(id);
if (type != PreprocessorTokenType.Invalid)
{
return(NewPPToken(type, position));
}
else
{
return(new PreprocessorIdentifierToken(id, position));
}
}
return(NewPPToken(PreprocessorTokenType.Unknown, position));
}
public Parser(NameTable symbolTable, string path) {
this.symbolTable = symbolTable;
_path = path;
assemblyName = symbolTable.Add("assembly");
moduleName = symbolTable.Add("module");
unknownName = symbolTable.Add("__unknown");
getName = symbolTable.Add("get");
setName = symbolTable.Add("set");
addName = symbolTable.Add("add");
removeName = symbolTable.Add("remove");
partialName = symbolTable.Add("partial");
yieldName = symbolTable.Add("yield");
whereName = symbolTable.Add("where");
aliasName = symbolTable.Add("alias");
}
private Token NextToken()
{
SkipWhiteSpace();
StartToken();
bool atIdentifier = false;
char ch = PeekChar();
if (ch == '\0')
{
return(NewToken(TokenType.EOF));
}
ch = NextChar();
switch (ch)
{
case '\0':
Debug.Fail("Checked for EOF above");
return(null);
case '#':
if (_text.Line == _lastLine)
{
ReportError(LexError.UnexpectedCharacter, ch.ToString());
return(ErrorToken());
}
else
{
ClearPosition();
_text.Reverse();
return(null);
}
// operators
case '{': return(NewToken(TokenType.OpenCurly));
case '}': return(NewToken(TokenType.CloseCurly));
case '[': return(NewToken(TokenType.OpenSquare));
case ']': return(NewToken(TokenType.CloseSquare));
case '(': return(NewToken(TokenType.OpenParen));
case ')': return(NewToken(TokenType.CloseParen));
case ',': return(NewToken(TokenType.Comma));
case ':':
ch = PeekChar();
if (ch == ':')
{
NextChar();
return(NewToken(TokenType.ColonColon));
}
return(NewToken(TokenType.Colon));
case ';': return(NewToken(TokenType.Semicolon));
case '~': return(NewToken(TokenType.Tilde));
case '?':
ch = PeekChar();
if (ch == '?')
{
NextChar();
return(NewToken(TokenType.Coalesce));
}
return(NewToken(TokenType.Question));
case '+':
ch = PeekChar();
if (ch == '+')
{
NextChar();
return(NewToken(TokenType.PlusPlus));
}
else if (ch == '=')
{
NextChar();
return(NewToken(TokenType.PlusEqual));
}
return(NewToken(TokenType.Plus));
case '-':
ch = PeekChar();
if (ch == '-')
{
NextChar();
return(NewToken(TokenType.MinusMinus));
}
else if (ch == '=')
{
NextChar();
return(NewToken(TokenType.MinusEqual));
}
else if (ch == '>')
{
NextChar();
return(NewToken(TokenType.Arrow));
}
return(NewToken(TokenType.Minus));
case '*':
if (PeekChar() == '=')
{
NextChar();
return(NewToken(TokenType.StarEqual));
}
return(NewToken(TokenType.Star));
case '/':
ch = PeekChar();
if (ch == '=')
{
NextChar();
return(NewToken(TokenType.SlashEqual));
}
else if (ch == '/')
{
NextChar();
CommentTokenType commentType;
if (!EOF && PeekChar() == '/')
{
commentType = CommentTokenType.TripleSlash;
NextChar();
}
else
{
commentType = CommentTokenType.DoubleSlash;
}
_value.Length = 0;
while (!EOF && !IsLineSeparator(PeekChar()))
{
_value.Append(NextChar());
}
if (_includeComments)
{
return(new CommentToken(commentType, _value.ToString(), _path, TakePosition()));
}
else
{
TakePosition();
return(NextToken());
}
}
else if (ch == '*')
{
NextChar();
_value.Length = 0;
while (!EOF && (PeekChar() != '*' || PeekChar(1) != '/'))
{
_value.Append(NextChar());
}
if (EOF)
{
ReportError(LexError.UnexpectedEndOfFileStarSlash);
return(ErrorToken());
}
NextChar();
NextChar();
_lastLine = _text.Line;
if (_includeComments)
{
return(new CommentToken(CommentTokenType.SlashStar, _value.ToString(), _path, TakePosition()));
}
else
{
TakePosition();
return(NextToken());
}
}
return(NewToken(TokenType.Slash));
case '%':
if (PeekChar() == '=')
{
NextChar();
return(NewToken(TokenType.PercentEqual));
}
return(NewToken(TokenType.Percent));
case '&':
ch = PeekChar();
if (ch == '&')
{
NextChar();
return(NewToken(TokenType.LogAnd));
}
else if (ch == '=')
{
NextChar();
return(NewToken(TokenType.AndEqual));
}
return(NewToken(TokenType.Ampersand));
case '|':
ch = PeekChar();
if (ch == '|')
{
NextChar();
return(NewToken(TokenType.LogOr));
}
else if (ch == '=')
{
NextChar();
return(NewToken(TokenType.BarEqual));
}
return(NewToken(TokenType.Bar));
case '^':
if (PeekChar() == '=')
{
NextChar();
return(NewToken(TokenType.HatEqual));
}
return(NewToken(TokenType.Hat));
case '!':
if (PeekChar() == '=')
{
NextChar();
return(NewToken(TokenType.NotEqual));
}
return(NewToken(TokenType.Bang));
case '=':
if (PeekChar() == '=')
{
NextChar();
return(NewToken(TokenType.EqualEqual));
}
return(NewToken(TokenType.Equal));
case '<':
ch = PeekChar();
if (ch == '=')
{
NextChar();
return(NewToken(TokenType.LessEqual));
}
else if (ch == '<')
{
NextChar();
if (PeekChar() == '=')
{
NextChar();
return(NewToken(TokenType.ShiftLeftEqual));
}
return(NewToken(TokenType.ShiftLeft));
}
return(NewToken(TokenType.Less));
case '>':
ch = PeekChar();
if (ch == '=')
{
NextChar();
return(NewToken(TokenType.GreaterEqual));
}
return(NewToken(TokenType.Greater));
// literals
case '\'':
// char literal
{
char ch2;
if (ScanCharValue('\'', false, out ch, out ch2))
{
if (PeekChar() != '\'')
{
ReportError(LexError.BadCharConstant);
}
else
{
NextChar();
}
return(new CharToken(ch, _path, TakePosition()));
}
else
{
return(ErrorToken());
}
}
case '"':
// string literal
{
char ch2;
_value.Length = 0;
while (!EOF && ScanCharValue('"', true, out ch, out ch2))
{
_value.Append(ch);
if (ch2 != 0)
{
_value.Append(ch2);
}
}
if (EOF)
{
ReportError(LexError.UnexpectedEndOfFileString);
return(ErrorToken());
}
return(new StringToken(_value.ToString(), _path, TakePosition()));
}
case '@':
ch = PeekChar();
if (ch == '"')
{
// verbatim string literal
NextChar();
_value.Length = 0;
while (!EOF && (PeekChar() != '"' || PeekChar(1) == '"'))
{
// this is the one place where a CR/LF pair is significant
bool wasCRLF;
ch = NextChar(out wasCRLF);
_value.Append(ch);
if (wasCRLF)
{
_value.Append('\xA');
}
else if (ch == '"')
{
NextChar();
}
}
if (EOF)
{
ReportError(LexError.UnexpectedEndOfFileString);
return(ErrorToken());
}
NextChar();
return(new StringToken(_value.ToString(), _path, TakePosition()));
}
atIdentifier = true;
goto default;
case '0':
if (PeekChar() == 'x' || PeekChar() == 'X')
{
NextChar();
// hexadecimal constant
ulong value = 0;
while (IsHexDigit(PeekChar()))
{
if ((value & 0xF000000000000000) != 0)
{
ReportError(LexError.NumericConstantOverflow);
return(ErrorToken());
}
value = (value << 4) | (uint)HexValue(NextChar());
}
return(CreateIntegerConstant(value, ScanIntegerSuffixOpt()));
}
goto case '1';
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
DoNumber : {
bool foundDecimalPoint = (ch == '.');
bool foundExponent = false;
NumericSuffix suffix = NumericSuffix.None;
_value.Length = 0;
_value.Append(ch);
while (true)
{
ch = PeekChar();
if (ch == '.')
{
if (foundDecimalPoint || !IsDigit(PeekChar(1)))
{
break;
}
foundDecimalPoint = true;
}
else if (ch == 'e' || ch == 'E')
{
char nextChar = PeekChar(1);
if (IsDigit(nextChar) || ((nextChar == '+' || nextChar == '-') && IsDigit(PeekChar(2))))
{
foundExponent = true;
_value.Append(NextChar());
_value.Append(NextChar());
while (IsDigit(PeekChar()))
{
_value.Append(NextChar());
}
}
break;
}
else if (!IsDigit(ch))
{
break;
}
_value.Append(NextChar());
}
if (!foundDecimalPoint && !foundExponent)
{
suffix = ScanIntegerSuffixOpt();
}
if (suffix == NumericSuffix.None)
{
suffix = ScanRealSuffixOpt();
}
if (suffix < NumericSuffix.F && !foundDecimalPoint && !foundExponent)
{
// decimal integer constant
ulong numericValue = 0;
foreach (char digit in _value.ToString().ToCharArray())
{
ulong value10 = numericValue * 10;
if (value10 < numericValue)
{
ReportError(LexError.NumericConstantOverflow);
return(ErrorToken());
}
numericValue = value10 + (uint)(digit - '0');
}
return(CreateIntegerConstant(numericValue, suffix));
}
else
{
try {
// real constant
switch (suffix)
{
case NumericSuffix.F: {
float f = float.Parse(_value.ToString(), NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent, CultureInfo.InvariantCulture);
return(new FloatToken(f, _path, TakePosition()));
}
case NumericSuffix.D:
case NumericSuffix.None: {
double d = double.Parse(_value.ToString(), NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent, CultureInfo.InvariantCulture);
return(new DoubleToken(d, _path, TakePosition()));
}
case NumericSuffix.M: {
decimal dec = decimal.Parse(_value.ToString(), NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent, CultureInfo.InvariantCulture);
return(new DecimalToken(dec, _path, TakePosition()));
}
}
}
catch (Exception) {
// catch overflow exceptions from the numeric parse
}
ReportError(LexError.NumericConstantOverflow);
return(ErrorToken());
}
}
case '.':
if (IsDigit(PeekChar()))
{
goto DoNumber;
}
return(NewToken(TokenType.Dot));
default: {
_value.Length = 0;
if (!IsIdentifierChar(ch))
{
ReportError(LexError.UnexpectedCharacter, ch.ToString());
return(ErrorToken());
}
_value.Append(ch);
while (ScanIdentifierChar(out ch))
{
_value.Append(ch);
}
Debug.Assert(_value.Length > 0);
if (_value.Length > IdentifierToken.MaxIdentifierLength)
{
ReportError(LexError.IdentifierTooLong);
return(ErrorToken());
}
Name name = _nameTable.Add(_value);
if (!atIdentifier)
{
// check for keywords
TokenType keyword = _keywords.IsKeyword(name);
if (Token.IsKeyword(keyword))
{
return(new Token(keyword, _path, TakePosition()));
}
}
return(new IdentifierToken(name, atIdentifier, _path, TakePosition()));
}
}
}
