protected override bool LexNext(State hiddenState)
{
CppLexerState state = (CppLexerState)hiddenState;
bool allowWhitespaces = !state.IsInsidePreprocessor;
if (allowWhitespaces)
{
Buffer.SkipAllWhitespaces();
}
if (Buffer.IsEOF)
{
return(false);
}
int line = Buffer.TextPosition.Line;
char first = Buffer.Peek();
char second = Buffer.Peek(1);
char third = Buffer.Peek(2);
Buffer.StartLexeme();
LexResult lexRes = new LexResult(CppTokenKind.Unknown, true);
switch (first)
{
case '&':
{
if (second == '&')
{
lexRes.Kind = CppTokenKind.LogicalAndOp;
Buffer.AdvanceColumns(2);
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.AndAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.AndOp;
Buffer.AdvanceColumn();
}
}
break;
case '|':
{
if (second == '|')
{
lexRes.Kind = CppTokenKind.LogicalOrOp;
Buffer.AdvanceColumns(2);
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.OrAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.OrOp;
Buffer.AdvanceColumn();
}
}
break;
case '=':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.LogicalEqualsOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.EqOp;
Buffer.AdvanceColumn();
}
}
break;
case '!':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.LogicalNotEqualsOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.ExclationMark;
Buffer.AdvanceColumn();
}
}
break;
case '<':
{
if (second == '<')
{
if (third == '=')
{
lexRes.Kind = CppTokenKind.LeftShiftAssign;
Buffer.AdvanceColumns(3);
}
else
{
lexRes.Kind = CppTokenKind.LeftShiftOp;
Buffer.AdvanceColumns(2);
}
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.LessOrEqualOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.LessThanOp;
Buffer.AdvanceColumn();
}
}
break;
case '>':
{
if (second == '>')
{
if (third == '=')
{
lexRes.Kind = CppTokenKind.RightShiftAssign;
Buffer.AdvanceColumns(3);
}
else
{
lexRes.Kind = CppTokenKind.RightShiftOp;
Buffer.AdvanceColumns(2);
}
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.GreaterOrEqualOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.GreaterThanOp;
Buffer.AdvanceColumn();
}
}
break;
case '+':
{
if (second == '+')
{
lexRes.Kind = CppTokenKind.IncOp;
Buffer.AdvanceColumns(2);
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.AddAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.AddOp;
Buffer.AdvanceColumn();
}
}
break;
case '-':
{
if (second == '-')
{
lexRes.Kind = CppTokenKind.DecOp;
Buffer.AdvanceColumns(2);
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.SubAssign;
Buffer.AdvanceColumns(2);
}
else if (second == '>')
{
lexRes.Kind = CppTokenKind.PtrOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.SubOp;
Buffer.AdvanceColumn();
}
}
break;
case '/':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.DivAssign;
Buffer.AdvanceColumns(2);
}
else if (second == '/')
{
lexRes = LexSingleLineComment(Buffer, true);
if (!lexRes.IsComplete)
{
AddError(Buffer.LexemeStart, $"Unterminated single-line comment, expect '\n' or '\r' but found '{Buffer.Peek()}'", lexRes.Kind.ToString());
}
}
else if (second == '*')
{
lexRes = LexMultiLineComment(Buffer, true);
if (!lexRes.IsComplete)
{
AddError(Buffer.LexemeStart, $"Unterminated single-line comment, expect '*/' but found '{Buffer.Peek()}'", lexRes.Kind.ToString());
}
}
else
{
Buffer.AdvanceColumn();
lexRes.Kind = CppTokenKind.DivOp;
}
}
break;
case '*':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.MulAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.MulOp;
Buffer.AdvanceColumn();
}
}
break;
case '%':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.ModAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.ModOp;
Buffer.AdvanceColumn();
}
}
break;
case '.':
{
if (second == '.' && third == '.')
{
lexRes.Kind = CppTokenKind.Ellipsis;
Buffer.AdvanceColumns(3);
}
else if (SyntaxUtils.IsNumeric(second))
{
lexRes = LexNumber();
}
else
{
lexRes.Kind = CppTokenKind.Dot;
Buffer.AdvanceColumn();
}
}
break;
case '^':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.XorAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.XorOp;
Buffer.AdvanceColumn();
}
}
break;
case '#':
return(LexPreprocessor(state));
case '"':
lexRes = LexString("string");
break;
case '\'':
lexRes = LexString("char");
break;
case '~':
lexRes.Kind = CppTokenKind.Tilde;
Buffer.AdvanceColumn();
break;
case '\\':
lexRes.Kind = CppTokenKind.Backslash;
Buffer.AdvanceColumn();
break;
case ',':
lexRes.Kind = CppTokenKind.Comma;
Buffer.AdvanceColumn();
break;
case ';':
lexRes.Kind = CppTokenKind.Semicolon;
Buffer.AdvanceColumn();
break;
case ':':
lexRes.Kind = CppTokenKind.Colon;
Buffer.AdvanceColumn();
break;
case '?':
lexRes.Kind = CppTokenKind.QuestionMark;
Buffer.AdvanceColumn();
break;
case '{':
lexRes.Kind = CppTokenKind.LeftBrace;
Buffer.AdvanceColumn();
break;
case '}':
lexRes.Kind = CppTokenKind.RightBrace;
Buffer.AdvanceColumn();
break;
case '[':
lexRes.Kind = CppTokenKind.LeftBracket;
Buffer.AdvanceColumn();
break;
case ']':
lexRes.Kind = CppTokenKind.RightBracket;
Buffer.AdvanceColumn();
break;
case '(':
lexRes.Kind = CppTokenKind.LeftParen;
Buffer.AdvanceColumn();
break;
case ')':
lexRes.Kind = CppTokenKind.RightParen;
Buffer.AdvanceColumn();
break;
default:
{
if (SyntaxUtils.IsLineBreak(first) && allowWhitespaces)
{
lexRes.Kind = CppTokenKind.EndOfLine;
int nb = SyntaxUtils.GetLineBreakChars(first, second);
Buffer.AdvanceLine(nb);
}
else if (first == '\t' && allowWhitespaces)
{
lexRes.Kind = CppTokenKind.Spacings;
while (!Buffer.IsEOF)
{
if (Buffer.Peek() != '\t')
{
break;
}
Buffer.AdvanceTab();
}
}
else if (SyntaxUtils.IsSpacing(first) && allowWhitespaces)
{
lexRes.Kind = CppTokenKind.Spacings;
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsSpacing);
}
else if (SyntaxUtils.IsIdentStart(first))
{
Debug.Assert(!state.IsInsidePreprocessor);
lexRes = LexIdent(false);
}
else if (SyntaxUtils.IsNumeric(first))
{
lexRes = LexNumber();
}
else
{
AddError(Buffer.TextPosition, $"Unexpected character '{first}'", "Character");
return(false);
}
}
break;
}
return(PushToken(CppTokenPool.Make(lexRes.Kind, Buffer.LexemeRange, lexRes.IsComplete)));
}
private LexResult LexNumber()
{
Debug.Assert(SyntaxUtils.IsNumeric(Buffer.Peek()) || Buffer.Peek() == '.');
CppTokenKind kind;
char first = Buffer.Peek(0);
char second = Buffer.Peek(1);
bool dotSeen = false;
if (first == '0')
{
if (second == 'x' || second == 'X')
{
// Hex
kind = CppTokenKind.HexLiteral;
Buffer.AdvanceColumns(2); // Skip 0[xX]
}
else if (second == 'b' || second == 'B')
{
// Binary
kind = CppTokenKind.BinaryLiteral;
Buffer.AdvanceColumns(2); // Skip 0[bB]
}
else
{
// Octal
kind = CppTokenKind.OctalLiteral;
}
}
else if (first == '.')
{
Debug.Assert(SyntaxUtils.IsNumeric(second));
kind = CppTokenKind.IntegerFloatLiteral;
Buffer.AdvanceColumn();
dotSeen = true;
}
else
{
Debug.Assert(SyntaxUtils.IsNumeric(first));
kind = CppTokenKind.IntegerLiteral;
}
// @NOTE(final): We never set the DecimalHexLiteral kind initially,
// as every hex decimal always starts as a normal hex literal!
Debug.Assert(kind != CppTokenKind.HexadecimalFloatLiteral);
// First number part
int firstLiteralPos = Buffer.TextPosition.Index;
bool readNextLiteral = false;
do
{
readNextLiteral = false;
int s = Buffer.TextPosition.Index;
switch (kind)
{
case CppTokenKind.IntegerLiteral:
case CppTokenKind.IntegerFloatLiteral:
if (SyntaxUtils.IsNumeric(Buffer.Peek()))
{
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsNumeric);
}
else
{
AddError(Buffer.TextPosition, $"Expect integer literal, but got '{Buffer.Peek()}'", kind.ToString());
}
break;
case CppTokenKind.OctalLiteral:
if (SyntaxUtils.IsOctal(Buffer.Peek()))
{
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsOctal);
}
else
{
AddError(Buffer.TextPosition, $"Expect octal literal, but got '{Buffer.Peek()}'", kind.ToString());
}
break;
case CppTokenKind.HexLiteral:
if (SyntaxUtils.IsHex(Buffer.Peek()))
{
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsHex);
}
else
{
AddError(Buffer.TextPosition, $"Expect hex literal, but got '{Buffer.Peek()}'", kind.ToString());
}
break;
case CppTokenKind.BinaryLiteral:
if (SyntaxUtils.IsBinary(Buffer.Peek()))
{
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsBinary);
}
else
{
AddError(Buffer.TextPosition, $"Expect binary literal, but got '{Buffer.Peek()}'", kind.ToString());
}
break;
default:
AddError(Buffer.TextPosition, $"Unsupported token kind '{kind}' for integer literal on {Buffer}", kind.ToString());
break;
}
bool hadIntegerLiteral = Buffer.TextPosition.Index > s;
if (kind != CppTokenKind.IntegerFloatLiteral && kind != CppTokenKind.HexadecimalFloatLiteral)
{
// @NOTE(final): Single quotes (') are allowed as separators for any non-decimal literal
char check0 = Buffer.Peek();
if (check0 == '\'')
{
if (!hadIntegerLiteral)
{
AddError(Buffer.TextPosition, $"Too many single quote escape in integer literal, expect any integer literal but got '{Buffer.Peek()}'", kind.ToString());
return(new LexResult(kind, false));
}
Buffer.AdvanceColumn();
readNextLiteral = true;
}
}
} while (!Buffer.IsEOF && readNextLiteral);
// Validate any literal after starting dot
if (dotSeen)
{
if (firstLiteralPos == Buffer.TextPosition.Index)
{
AddError(Buffer.TextPosition, $"Expect any integer literal after starting dot, but got '{Buffer.Peek()}'", kind.ToString());
return(new LexResult(kind, false));
}
}
// Dot separator
if ((!dotSeen) &&
((kind == CppTokenKind.IntegerLiteral) ||
(kind == CppTokenKind.HexLiteral) ||
(kind == CppTokenKind.OctalLiteral)
))
{
char check0 = Buffer.Peek();
if (check0 == '.')
{
dotSeen = true;
Buffer.AdvanceColumn();
if (kind == CppTokenKind.IntegerLiteral || kind == CppTokenKind.OctalLiteral)
{
kind = CppTokenKind.IntegerFloatLiteral;
}
else
{
Debug.Assert(kind == CppTokenKind.HexLiteral);
kind = CppTokenKind.HexadecimalFloatLiteral;
}
}
else if (SyntaxUtils.IsExponentPrefix(check0))
{
if (kind == CppTokenKind.IntegerLiteral || kind == CppTokenKind.OctalLiteral)
{
kind = CppTokenKind.IntegerFloatLiteral;
}
else
{
Debug.Assert(kind == CppTokenKind.HexLiteral);
kind = CppTokenKind.HexadecimalFloatLiteral;
}
}
}
// Decimal after dot separator
if ((kind != CppTokenKind.IntegerFloatLiteral) &&
(kind != CppTokenKind.HexadecimalFloatLiteral))
{
// Integer suffix
if (SyntaxUtils.IsIntegerSuffix(Buffer.Peek()))
{
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsIntegerSuffix, 3);
}
}
else
{
if (kind == CppTokenKind.IntegerFloatLiteral)
{
// Float decimal
if (SyntaxUtils.IsNumeric(Buffer.Peek()))
{
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsNumeric);
}
if (Buffer.Peek() == 'e' || Buffer.Peek() == 'E')
{
AdvanceExponent('e');
}
}
else
{
// Hex decimal
Debug.Assert(kind == CppTokenKind.HexadecimalFloatLiteral);
if (SyntaxUtils.IsHex(Buffer.Peek()))
{
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsHex);
}
if (Buffer.Peek() == 'p' || Buffer.Peek() == 'P')
{
AdvanceExponent('e');
}
}
// Float suffix
if (SyntaxUtils.IsFloatSuffix(Buffer.Peek()))
{
Buffer.AdvanceColumn();
}
}
return(new LexResult(kind, true));
}
public static CToken PeekTokenRaw(TextStream stream)
{
TextPosition startPos = stream.TextPosition;
if (stream.IsEOF)
{
return(new CToken(CppTokenKind.Eof, startPos, stream.TextPosition));
}
char first = stream.Peek(0);
char second = stream.Peek(1);
char third = stream.Peek(2);
CppTokenKind kind;
switch (first)
{
case '&':
{
if (second == '&')
{
kind = CppTokenKind.LogicalAndOp;
stream.AdvanceColumns(2);
}
else if (second == '=')
{
kind = CppTokenKind.AndAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.AndOp;
stream.AdvanceColumn();
}
}
break;
case '|':
{
if (second == '|')
{
kind = CppTokenKind.LogicalOrOp;
stream.AdvanceColumns(2);
}
else if (second == '=')
{
kind = CppTokenKind.OrAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.OrOp;
stream.AdvanceColumn();
}
}
break;
case '=':
{
if (second == '=')
{
kind = CppTokenKind.LogicalEqualsOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.EqOp;
stream.AdvanceColumn();
}
}
break;
case '!':
{
if (second == '=')
{
kind = CppTokenKind.LogicalNotEqualsOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.ExclationMark;
stream.AdvanceColumn();
}
}
break;
case '<':
{
if (second == '<')
{
if (third == '=')
{
kind = CppTokenKind.LeftShiftAssign;
stream.AdvanceColumns(3);
}
else
{
kind = CppTokenKind.LeftShiftOp;
stream.AdvanceColumns(2);
}
}
else if (second == '=')
{
kind = CppTokenKind.LessOrEqualOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.LessThanOp;
stream.AdvanceColumn();
}
}
break;
case '>':
{
if (second == '>')
{
if (third == '=')
{
kind = CppTokenKind.RightShiftAssign;
stream.AdvanceColumns(3);
}
else
{
kind = CppTokenKind.RightShiftOp;
stream.AdvanceColumns(2);
}
}
else if (second == '=')
{
kind = CppTokenKind.GreaterOrEqualOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.GreaterThanOp;
stream.AdvanceColumn();
}
}
break;
case '+':
{
if (second == '+')
{
kind = CppTokenKind.IncOp;
stream.AdvanceColumns(2);
}
else if (second == '=')
{
kind = CppTokenKind.AddAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.AddOp;
stream.AdvanceColumn();
}
}
break;
case '-':
{
if (second == '-')
{
kind = CppTokenKind.DecOp;
stream.AdvanceColumns(2);
}
else if (second == '=')
{
kind = CppTokenKind.SubAssign;
stream.AdvanceColumns(2);
}
else if (second == '>')
{
kind = CppTokenKind.PtrOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.SubOp;
stream.AdvanceColumn();
}
}
break;
case '/':
{
if (second == '=')
{
kind = CppTokenKind.DivAssign;
stream.AdvanceColumns(2);
}
else if (second == '/')
{
kind = CppTokenKind.SingleLineComment;
stream.AdvanceColumns(2);
char specialChar = stream.Peek();
if (DoxygenSyntax.SingleLineDocChars.Contains(specialChar))
{
kind = CppTokenKind.SingleLineCommentDoc;
stream.AdvanceColumn();
}
while (!stream.IsEOF)
{
if (SyntaxUtils.IsLineBreak(stream.Peek()))
{
break;
}
stream.AdvanceColumn();
}
}
else if (second == '*')
{
kind = CppTokenKind.MultiLineComment;
stream.AdvanceColumns(2);
char specialChar = stream.Peek();
if (DoxygenSyntax.MultiLineDocChars.Contains(specialChar))
{
kind = CppTokenKind.MultiLineCommentDoc;
stream.AdvanceColumn();
}
while (!stream.IsEOF)
{
char n0 = stream.Peek();
char n1 = stream.Peek(1);
if (n0 == '*' && n1 == '/')
{
stream.AdvanceColumns(2);
break;
}
else
{
stream.AdvanceManual(n0, n1);
}
}
}
else
{
stream.AdvanceColumn();
kind = CppTokenKind.DivOp;
}
}
break;
case '*':
{
if (second == '=')
{
kind = CppTokenKind.MulAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.MulOp;
stream.AdvanceColumn();
}
}
break;
case '%':
{
if (second == '=')
{
kind = CppTokenKind.ModAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.ModOp;
stream.AdvanceColumn();
}
}
break;
case '.':
{
if (second == '.' && third == '.')
{
kind = CppTokenKind.Ellipsis;
stream.AdvanceColumns(3);
}
else if (SyntaxUtils.IsNumeric(second))
{
stream.AdvanceColumn();
kind = ReadNumberLiteral(stream);
}
else
{
kind = CppTokenKind.Dot;
stream.AdvanceColumn();
}
}
break;
case '^':
{
if (second == '=')
{
kind = CppTokenKind.XorAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.XorOp;
stream.AdvanceColumn();
}
}
break;
case '"':
case '\'':
{
stream.AdvanceColumn();
if (first == '"')
{
kind = ReadStringLiteral(stream);
}
else
{
kind = ReadCharacterLiteral(stream);
}
}
break;
case '~':
kind = CppTokenKind.Tilde;
stream.AdvanceColumn();
break;
case '\\':
kind = CppTokenKind.Backslash;
stream.AdvanceColumn();
break;
case '#':
kind = CppTokenKind.PreprocessorStart;
stream.AdvanceColumn();
break;
case ',':
kind = CppTokenKind.Comma;
stream.AdvanceColumn();
break;
case ';':
kind = CppTokenKind.Semicolon;
stream.AdvanceColumn();
break;
case ':':
kind = CppTokenKind.Colon;
stream.AdvanceColumn();
break;
case '?':
kind = CppTokenKind.QuestionMark;
stream.AdvanceColumn();
break;
case '{':
kind = CppTokenKind.LeftBrace;
stream.AdvanceColumn();
break;
case '}':
kind = CppTokenKind.RightBrace;
stream.AdvanceColumn();
break;
case '[':
kind = CppTokenKind.LeftBracket;
stream.AdvanceColumn();
break;
case ']':
kind = CppTokenKind.RightBracket;
stream.AdvanceColumn();
break;
case '(':
kind = CppTokenKind.LeftParen;
stream.AdvanceColumn();
break;
case ')':
kind = CppTokenKind.RightParen;
stream.AdvanceColumn();
break;
default:
{
if (SyntaxUtils.IsLineBreak(first))
{
kind = CppTokenKind.EndOfLine;
int nb = SyntaxUtils.GetLineBreakChars(first, second);
stream.AdvanceLine(nb);
}
else if (first == '\t')
{
kind = CppTokenKind.Spacings;
while (!stream.IsEOF)
{
if (stream.Peek() != '\t')
{
break;
}
stream.AdvanceTab();
}
}
else if (SyntaxUtils.IsSpacing(first))
{
kind = CppTokenKind.Spacings;
stream.AdvanceColumnsWhile(SyntaxUtils.IsSpacing);
}
else if (SyntaxUtils.IsIdentStart(first))
{
kind = CppTokenKind.IdentLiteral;
stream.AdvanceColumnsWhile(SyntaxUtils.IsIdentPart);
}
else if (SyntaxUtils.IsNumeric(first))
{
kind = ReadNumberLiteral(stream);
}
else
{
kind = CppTokenKind.Unknown;
}
}
break;
}
CToken result = MakeToken(kind, stream, startPos, stream.TextPosition);
return(result);
}
private static CppTokenKind ReadNumberLiteral(TextStream stream)
{
// D = [0-9]
// H = [a-fA-F0-9]
// IS = ((u|U)|(u|U)?(l|L|ll|LL)|(l|L|ll|LL)(u|U))
// FS = (f|F|l|L)
// E = ([Ee][+-]?{D}+)
// P = ([Pp][+-]?{D}+)
// Hex integer = 0[xX]{H}+{IS}?
// Octal integer = 0[0-7]*{IS}?
// Integer = [1-9]{D}*{IS}?
// Decimal = {D}+{E}{FS}?
// Decimal = {D}*"."{D}+{E}?{FS}?
// Decimal = {D}+"."{D}*{E}?{FS}?
// Decimal = 0[xX]{H}+{P}{FS}?
// Decimal = 0[xX]{H}*"."{H}+{P}?{FS}?
// Decimal = 0[xX]{H}+"."{H}*{P}?{FS}?
// Examples: 42
// Examples: 427.893
// Examples: 0x10.1p0
// Examples: 0x1.2p3
// Examples: .1E4f
// Examples: 58.
// Examples: 123.456e-67
// Examples: 4e2
CppTokenKind result;
char first = stream.Peek(0);
char second = stream.Peek(1);
bool dotSeen = false;
if (first == '0')
{
if (second == 'x' || second == 'X')
{
// Hex
stream.AdvanceColumns(2); // Skip 0[xX]
result = CppTokenKind.HexLiteral;
}
else if (second == '.')
{
// 0.123456f decimal
result = CppTokenKind.IntegerFloatLiteral;
}
else
{
// Octal
result = CppTokenKind.OctalLiteral;
}
}
else if (first == '.')
{
// .123488E+4 decimal
Debug.Assert(SyntaxUtils.IsNumeric(second));
result = CppTokenKind.IntegerFloatLiteral;
stream.AdvanceColumn();
dotSeen = true;
}
else
{
Debug.Assert(SyntaxUtils.IsNumeric(first));
result = CppTokenKind.IntegerLiteral;
}
// First number part
switch (result)
{
case CppTokenKind.IntegerLiteral:
stream.AdvanceColumnsWhile(SyntaxUtils.IsNumeric);
break;
case CppTokenKind.HexLiteral:
stream.AdvanceColumnsWhile(SyntaxUtils.IsHex);
break;
case CppTokenKind.OctalLiteral:
stream.AdvanceColumnsWhile(SyntaxUtils.IsOctal);
break;
case CppTokenKind.IntegerFloatLiteral:
{
// .12345f number without a number start
}
break;
default:
throw new Exception($"Unsupport number literal token '{result}' as number literal on {stream}");
}
if (stream.Peek() == '.')
{
stream.AdvanceColumn();
if (!dotSeen)
{
result = result == CppTokenKind.HexLiteral ? CppTokenKind.HexadecimalFloatLiteral : CppTokenKind.IntegerFloatLiteral;
dotSeen = true;
}
else
{
throw new Exception($"Invalid decimal literal for token '{result}' as number literal on {stream}");
}
}
if (result == CppTokenKind.IntegerLiteral)
{
char n = stream.Peek();
if (n == 'e' || n == 'E')
{
result = CppTokenKind.IntegerFloatLiteral;
}
}
if (result == CppTokenKind.IntegerLiteral || result == CppTokenKind.OctalLiteral || result == CppTokenKind.HexLiteral)
{
// Integer type [uUlL]{1,3}
stream.AdvanceColumnsWhile((c) => c == 'u' || c == 'U' || c == 'l' || c == 'L', 3);
}
else
{
// Expontial ([eE][+-]?[0-9]+)|([pP][+-]?[0-9]+)
if (result == CppTokenKind.HexadecimalFloatLiteral)
{
stream.AdvanceColumnsWhile(SyntaxUtils.IsHex);
AdvanceExponent(stream, 'p');
}
else
{
stream.AdvanceColumnsWhile(SyntaxUtils.IsNumeric);
AdvanceExponent(stream, 'e');
}
// Double type [fFlL]{0,1}
char n = stream.Peek();
if (n == 'f' || n == 'F' || n == 'l' || n == 'L')
{
stream.AdvanceColumn();
}
}
return(result);
}
private static CppTokenKind ReadCharacterLiteral(TextStream stream)
{
char n = stream.Peek();
if (n == '\\')
{
stream.AdvanceColumn(); // Skip over backslash
char escapeChar = stream.Peek();
switch (escapeChar)
{
case '\'':
case '"':
case '?':
case '\\':
case 'a':
case 'b':
case 'f':
case 'n':
case 'r':
case 't':
case 'v':
stream.AdvanceColumn();
break;
case 'u':
case 'U':
{
int maxCount = escapeChar == 'u' ? 4 : 8;
stream.AdvanceColumn();
if (SyntaxUtils.IsNumeric(stream.Peek()))
{
stream.AdvanceColumnsWhile(SyntaxUtils.IsNumeric, maxCount);
}
}
break;
case 'X':
{
// Hex escape
stream.AdvanceColumn();
if (SyntaxUtils.IsHex(stream.Peek()))
{
stream.AdvanceColumnsWhile(SyntaxUtils.IsHex, 2);
}
}
break;
default:
{
// Octal escape
if (SyntaxUtils.IsOctal(escapeChar))
{
stream.AdvanceColumnsWhile(SyntaxUtils.IsOctal, 3);
}
}
break;
}
}
else
{
stream.AdvanceColumn();
}
if (stream.Peek() == '\'')
{
stream.AdvanceColumn();
}
return(CppTokenKind.CharLiteral);
}
