private Token NextToken()
{
SkipWhiteSpace();
StartToken();
bool atIdentifier = false;
char ch = PeekChar();
if (ch == '\0')
{
return(NewToken(TokenType.Eof));
}
ch = NextChar();
switch (ch)
{
case '\0':
Debug.Fail("Checked for EOF above");
return(null);
case '#':
if (text.Line == lastLine)
{
ReportError(LexError.UnexpectedCharacter, ch.ToString());
return(ErrorToken());
}
else
{
ClearPosition();
text.Reverse();
return(null);
}
// operators
case '{': return(NewToken(TokenType.OpenCurly));
case '}': return(NewToken(TokenType.CloseCurly));
case '[': return(NewToken(TokenType.OpenSquare));
case ']': return(NewToken(TokenType.CloseSquare));
case '(': return(NewToken(TokenType.OpenParen));
case ')': return(NewToken(TokenType.CloseParen));
case ',': return(NewToken(TokenType.Comma));
case ':':
ch = PeekChar();
if (ch == ':')
{
NextChar();
return(NewToken(TokenType.ColonColon));
}
return(NewToken(TokenType.Colon));
case ';': return(NewToken(TokenType.Semicolon));
case '~': return(NewToken(TokenType.Tilde));
case '?':
ch = PeekChar();
if (ch == '?')
{
NextChar();
return(NewToken(TokenType.Coalesce));
}
return(NewToken(TokenType.Question));
case '+':
ch = PeekChar();
if (ch == '+')
{
NextChar();
return(NewToken(TokenType.PlusPlus));
}
else if (ch == '=')
{
NextChar();
return(NewToken(TokenType.PlusEqual));
}
return(NewToken(TokenType.Plus));
case '-':
ch = PeekChar();
if (ch == '-')
{
NextChar();
return(NewToken(TokenType.MinusMinus));
}
else if (ch == '=')
{
NextChar();
return(NewToken(TokenType.MinusEqual));
}
else if (ch == '>')
{
NextChar();
return(NewToken(TokenType.Arrow));
}
return(NewToken(TokenType.Minus));
case '*':
if (PeekChar() == '=')
{
NextChar();
return(NewToken(TokenType.StarEqual));
}
return(NewToken(TokenType.Star));
case '/':
ch = PeekChar();
if (ch == '=')
{
NextChar();
return(NewToken(TokenType.SlashEqual));
}
else if (ch == '/')
{
NextChar();
CommentTokenType commentType;
if (!Eof && PeekChar() == '/')
{
commentType = CommentTokenType.TripleSlash;
NextChar();
}
else
{
commentType = CommentTokenType.DoubleSlash;
}
value.Length = 0;
while (!Eof && !IsLineSeparator(PeekChar()))
{
value.Append(NextChar());
}
if (includeComments)
{
return(new CommentToken(commentType, value.ToString(), path, TakePosition()));
}
TakePosition();
return(NextToken());
}
else if (ch == '*')
{
NextChar();
value.Length = 0;
while (!Eof && (PeekChar() != '*' || PeekChar(1) != '/'))
{
value.Append(NextChar());
}
if (Eof)
{
ReportError(LexError.UnexpectedEndOfFileStarSlash);
return(ErrorToken());
}
NextChar();
NextChar();
lastLine = text.Line;
if (includeComments)
{
return(new CommentToken(CommentTokenType.SlashStar, value.ToString(), path, TakePosition()));
}
TakePosition();
return(NextToken());
}
return(NewToken(TokenType.Slash));
case '%':
if (PeekChar() == '=')
{
NextChar();
return(NewToken(TokenType.PercentEqual));
}
return(NewToken(TokenType.Percent));
case '&':
ch = PeekChar();
if (ch == '&')
{
NextChar();
return(NewToken(TokenType.LogAnd));
}
else if (ch == '=')
{
NextChar();
return(NewToken(TokenType.AndEqual));
}
return(NewToken(TokenType.Ampersand));
case '|':
ch = PeekChar();
if (ch == '|')
{
NextChar();
return(NewToken(TokenType.LogOr));
}
else if (ch == '=')
{
NextChar();
return(NewToken(TokenType.BarEqual));
}
return(NewToken(TokenType.Bar));
case '^':
if (PeekChar() == '=')
{
NextChar();
return(NewToken(TokenType.HatEqual));
}
return(NewToken(TokenType.Hat));
case '!':
if (PeekChar() == '=')
{
NextChar();
return(NewToken(TokenType.NotEqual));
}
return(NewToken(TokenType.Bang));
case '=':
if (PeekChar() == '=')
{
NextChar();
return(NewToken(TokenType.EqualEqual));
}
return(NewToken(TokenType.Equal));
case '<':
ch = PeekChar();
if (ch == '=')
{
NextChar();
return(NewToken(TokenType.LessEqual));
}
else if (ch == '<')
{
NextChar();
if (PeekChar() == '=')
{
NextChar();
return(NewToken(TokenType.ShiftLeftEqual));
}
return(NewToken(TokenType.ShiftLeft));
}
return(NewToken(TokenType.Less));
case '>':
ch = PeekChar();
if (ch == '=')
{
NextChar();
return(NewToken(TokenType.GreaterEqual));
}
return(NewToken(TokenType.Greater));
// literals
case '\'':
// char literal
{
char ch2;
if (ScanCharValue('\'', false, out ch, out ch2))
{
if (PeekChar() != '\'')
{
ReportError(LexError.BadCharConstant);
}
else
{
NextChar();
}
return(new CharToken(ch, path, TakePosition()));
}
return(ErrorToken());
}
case '"':
// string literal
{
value.Length = 0;
while (!Eof && ScanCharValue('"', true, out ch, out char ch2))
{
value.Append(ch);
if (ch2 != 0)
{
value.Append(ch2);
}
}
if (Eof)
{
ReportError(LexError.UnexpectedEndOfFileString);
return(ErrorToken());
}
return(new StringToken(value.ToString(), path, TakePosition()));
}
case '@':
ch = PeekChar();
if (ch == '"')
{
// verbatim string literal
NextChar();
value.Length = 0;
while (!Eof && (PeekChar() != '"' || PeekChar(1) == '"'))
{
// this is the one place where a CR/LF pair is significant
ch = NextChar(out bool wasCrlf);
value.Append(ch);
if (wasCrlf)
{
value.Append('\xA');
}
else if (ch == '"')
{
NextChar();
}
}
if (Eof)
{
ReportError(LexError.UnexpectedEndOfFileString);
return(ErrorToken());
}
NextChar();
return(new StringToken(value.ToString(), path, TakePosition()));
}
atIdentifier = true;
goto default;
case '0':
if (PeekChar() == 'x' || PeekChar() == 'X')
{
NextChar();
// hexadecimal constant
ulong value = 0;
while (IsHexDigit(PeekChar()))
{
if ((value & 0xF000000000000000) != 0)
{
ReportError(LexError.NumericConstantOverflow);
return(ErrorToken());
}
value = (value << 4) | (uint)HexValue(NextChar());
}
return(CreateIntegerConstant(value, ScanIntegerSuffixOpt()));
}
goto case '1';
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
DoNumber:
{
bool foundDecimalPoint = ch == '.';
bool foundExponent = false;
NumericSuffix suffix = NumericSuffix.None;
value.Length = 0;
value.Append(ch);
while (true)
{
ch = PeekChar();
if (ch == '.')
{
if (foundDecimalPoint || !IsDigit(PeekChar(1)))
{
break;
}
foundDecimalPoint = true;
}
else if (ch == 'e' || ch == 'E')
{
char nextChar = PeekChar(1);
if (IsDigit(nextChar) || (nextChar == '+' || nextChar == '-') && IsDigit(PeekChar(2)))
{
foundExponent = true;
value.Append(NextChar());
value.Append(NextChar());
while (IsDigit(PeekChar()))
{
value.Append(NextChar());
}
}
break;
}
else if (!IsDigit(ch))
{
break;
}
value.Append(NextChar());
}
if (!foundDecimalPoint && !foundExponent)
{
suffix = ScanIntegerSuffixOpt();
}
if (suffix == NumericSuffix.None)
{
suffix = ScanRealSuffixOpt();
}
if (suffix < NumericSuffix.F && !foundDecimalPoint && !foundExponent)
{
// decimal integer constant
ulong numericValue = 0;
foreach (char digit in value.ToString())
{
ulong value10 = numericValue * 10;
if (value10 < numericValue)
{
ReportError(LexError.NumericConstantOverflow);
return(ErrorToken());
}
numericValue = value10 + (uint)(digit - '0');
}
return(CreateIntegerConstant(numericValue, suffix));
}
try
{
// real constant
switch (suffix)
{
case NumericSuffix.F:
{
float f = float.Parse(value.ToString(),
NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent,
CultureInfo.InvariantCulture);
return(new FloatToken(f, path, TakePosition()));
}
case NumericSuffix.D:
case NumericSuffix.None:
{
double d = double.Parse(value.ToString(),
NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent,
CultureInfo.InvariantCulture);
return(new DoubleToken(d, path, TakePosition()));
}
case NumericSuffix.M:
{
decimal dec = decimal.Parse(value.ToString(),
NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent,
CultureInfo.InvariantCulture);
return(new DecimalToken(dec, path, TakePosition()));
}
}
}
catch (Exception)
{
// catch overflow exceptions from the numeric parse
}
ReportError(LexError.NumericConstantOverflow);
return(ErrorToken());
}
case '.':
if (IsDigit(PeekChar()))
{
goto DoNumber;
}
return(NewToken(TokenType.Dot));
default:
{
value.Length = 0;
if (!IsIdentifierChar(ch))
{
ReportError(LexError.UnexpectedCharacter, ch.ToString());
return(ErrorToken());
}
value.Append(ch);
while (ScanIdentifierChar(out ch))
{
value.Append(ch);
}
Debug.Assert(value.Length > 0);
if (value.Length > IdentifierToken.MAX_IDENTIFIER_LENGTH)
{
ReportError(LexError.IdentifierTooLong);
return(ErrorToken());
}
Name name = nameTable.Add(value);
if (!atIdentifier)
{
// check for keywords
TokenType keyword = keywords.IsKeyword(name);
if (Token.IsKeyword(keyword))
{
return(new Token(keyword, path, TakePosition()));
}
}
return(new IdentifierToken(name, atIdentifier, path, TakePosition()));
}
}
}
