protected override bool LexNext(State hiddenState)
{
CppLexerState state = (CppLexerState)hiddenState;
bool allowWhitespaces = !state.IsInsidePreprocessor;
if (allowWhitespaces)
{
Buffer.SkipAllWhitespaces();
}
if (Buffer.IsEOF)
{
return(false);
}
int line = Buffer.TextPosition.Line;
char first = Buffer.Peek();
char second = Buffer.Peek(1);
char third = Buffer.Peek(2);
Buffer.StartLexeme();
LexResult lexRes = new LexResult(CppTokenKind.Unknown, true);
switch (first)
{
case '&':
{
if (second == '&')
{
lexRes.Kind = CppTokenKind.LogicalAndOp;
Buffer.AdvanceColumns(2);
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.AndAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.AndOp;
Buffer.AdvanceColumn();
}
}
break;
case '|':
{
if (second == '|')
{
lexRes.Kind = CppTokenKind.LogicalOrOp;
Buffer.AdvanceColumns(2);
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.OrAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.OrOp;
Buffer.AdvanceColumn();
}
}
break;
case '=':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.LogicalEqualsOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.EqOp;
Buffer.AdvanceColumn();
}
}
break;
case '!':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.LogicalNotEqualsOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.ExclationMark;
Buffer.AdvanceColumn();
}
}
break;
case '<':
{
if (second == '<')
{
if (third == '=')
{
lexRes.Kind = CppTokenKind.LeftShiftAssign;
Buffer.AdvanceColumns(3);
}
else
{
lexRes.Kind = CppTokenKind.LeftShiftOp;
Buffer.AdvanceColumns(2);
}
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.LessOrEqualOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.LessThanOp;
Buffer.AdvanceColumn();
}
}
break;
case '>':
{
if (second == '>')
{
if (third == '=')
{
lexRes.Kind = CppTokenKind.RightShiftAssign;
Buffer.AdvanceColumns(3);
}
else
{
lexRes.Kind = CppTokenKind.RightShiftOp;
Buffer.AdvanceColumns(2);
}
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.GreaterOrEqualOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.GreaterThanOp;
Buffer.AdvanceColumn();
}
}
break;
case '+':
{
if (second == '+')
{
lexRes.Kind = CppTokenKind.IncOp;
Buffer.AdvanceColumns(2);
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.AddAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.AddOp;
Buffer.AdvanceColumn();
}
}
break;
case '-':
{
if (second == '-')
{
lexRes.Kind = CppTokenKind.DecOp;
Buffer.AdvanceColumns(2);
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.SubAssign;
Buffer.AdvanceColumns(2);
}
else if (second == '>')
{
lexRes.Kind = CppTokenKind.PtrOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.SubOp;
Buffer.AdvanceColumn();
}
}
break;
case '/':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.DivAssign;
Buffer.AdvanceColumns(2);
}
else if (second == '/')
{
lexRes = LexSingleLineComment(Buffer, true);
if (!lexRes.IsComplete)
{
AddError(Buffer.LexemeStart, $"Unterminated single-line comment, expect '\n' or '\r' but found '{Buffer.Peek()}'", lexRes.Kind.ToString());
}
}
else if (second == '*')
{
lexRes = LexMultiLineComment(Buffer, true);
if (!lexRes.IsComplete)
{
AddError(Buffer.LexemeStart, $"Unterminated single-line comment, expect '*/' but found '{Buffer.Peek()}'", lexRes.Kind.ToString());
}
}
else
{
Buffer.AdvanceColumn();
lexRes.Kind = CppTokenKind.DivOp;
}
}
break;
case '*':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.MulAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.MulOp;
Buffer.AdvanceColumn();
}
}
break;
case '%':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.ModAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.ModOp;
Buffer.AdvanceColumn();
}
}
break;
case '.':
{
if (second == '.' && third == '.')
{
lexRes.Kind = CppTokenKind.Ellipsis;
Buffer.AdvanceColumns(3);
}
else if (SyntaxUtils.IsNumeric(second))
{
lexRes = LexNumber();
}
else
{
lexRes.Kind = CppTokenKind.Dot;
Buffer.AdvanceColumn();
}
}
break;
case '^':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.XorAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.XorOp;
Buffer.AdvanceColumn();
}
}
break;
case '#':
return(LexPreprocessor(state));
case '"':
lexRes = LexString("string");
break;
case '\'':
lexRes = LexString("char");
break;
case '~':
lexRes.Kind = CppTokenKind.Tilde;
Buffer.AdvanceColumn();
break;
case '\\':
lexRes.Kind = CppTokenKind.Backslash;
Buffer.AdvanceColumn();
break;
case ',':
lexRes.Kind = CppTokenKind.Comma;
Buffer.AdvanceColumn();
break;
case ';':
lexRes.Kind = CppTokenKind.Semicolon;
Buffer.AdvanceColumn();
break;
case ':':
lexRes.Kind = CppTokenKind.Colon;
Buffer.AdvanceColumn();
break;
case '?':
lexRes.Kind = CppTokenKind.QuestionMark;
Buffer.AdvanceColumn();
break;
case '{':
lexRes.Kind = CppTokenKind.LeftBrace;
Buffer.AdvanceColumn();
break;
case '}':
lexRes.Kind = CppTokenKind.RightBrace;
Buffer.AdvanceColumn();
break;
case '[':
lexRes.Kind = CppTokenKind.LeftBracket;
Buffer.AdvanceColumn();
break;
case ']':
lexRes.Kind = CppTokenKind.RightBracket;
Buffer.AdvanceColumn();
break;
case '(':
lexRes.Kind = CppTokenKind.LeftParen;
Buffer.AdvanceColumn();
break;
case ')':
lexRes.Kind = CppTokenKind.RightParen;
Buffer.AdvanceColumn();
break;
default:
{
if (SyntaxUtils.IsLineBreak(first) && allowWhitespaces)
{
lexRes.Kind = CppTokenKind.EndOfLine;
int nb = SyntaxUtils.GetLineBreakChars(first, second);
Buffer.AdvanceLine(nb);
}
else if (first == '\t' && allowWhitespaces)
{
lexRes.Kind = CppTokenKind.Spacings;
while (!Buffer.IsEOF)
{
if (Buffer.Peek() != '\t')
{
break;
}
Buffer.AdvanceTab();
}
}
else if (SyntaxUtils.IsSpacing(first) && allowWhitespaces)
{
lexRes.Kind = CppTokenKind.Spacings;
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsSpacing);
}
else if (SyntaxUtils.IsIdentStart(first))
{
Debug.Assert(!state.IsInsidePreprocessor);
lexRes = LexIdent(false);
}
else if (SyntaxUtils.IsNumeric(first))
{
lexRes = LexNumber();
}
else
{
AddError(Buffer.TextPosition, $"Unexpected character '{first}'", "Character");
return(false);
}
}
break;
}
return(PushToken(CppTokenPool.Make(lexRes.Kind, Buffer.LexemeRange, lexRes.IsComplete)));
}
private bool LexPreprocessor(CppLexerState state)
{
Debug.Assert(Buffer.Peek() == '#');
state.StartPreprocessor();
// Preprocessor start
Buffer.StartLexeme();
Buffer.AdvanceColumn();
PushToken(CppTokenPool.Make(CppTokenKind.PreprocessorStart, Buffer.LexemeRange, true));
do
{
Buffer.SkipSpacings(TextStream.SkipType.All);
Buffer.StartLexeme();
char first = Buffer.Peek();
char second = Buffer.Peek(1);
char third = Buffer.Peek(2);
if (first == '\\')
{
if (SyntaxUtils.IsLineBreak(second))
{
Buffer.AdvanceColumn();
int lb = SyntaxUtils.GetLineBreakChars(second, third);
Buffer.AdvanceLine(lb);
continue;
}
else
{
AddError(Buffer.TextPosition, $"Unterminated preprocessor next-line, expect linebreak after '\' but got '{second}'", "Preprocessor");
return(false);
}
}
else if (first == '#')
{
Buffer.AdvanceColumn();
PushToken(CppTokenPool.Make(CppTokenKind.PreprocessorOperator, Buffer.LexemeRange, true));
}
else if (SyntaxUtils.IsLineBreak(first))
{
int lb = SyntaxUtils.GetLineBreakChars(first, second);
Buffer.AdvanceLine(lb);
PushToken(CppTokenPool.Make(CppTokenKind.EndOfLine, Buffer.LexemeRange, true));
break;
}
else if (SyntaxUtils.IsIdentStart(first))
{
LexResult identResult = LexIdent(true);
CppToken identToken = CppTokenPool.Make(identResult.Kind, Buffer.LexemeRange, identResult.IsComplete);
PushToken(identToken);
Buffer.SkipSpacings(TextStream.SkipType.All);
Buffer.StartLexeme();
if (identToken.Kind == CppTokenKind.PreprocessorKeyword)
{
switch (identToken.Value)
{
case "define":
{
if (!SyntaxUtils.IsIdentStart(Buffer.Peek()))
{
AddError(Buffer.TextPosition, $"Expect identifier for define, but got '{Buffer.Peek()}'", "Preprocessor");
return(false);
}
LexResult defineValueResult = LexIdent(false);
CppToken defineValueToken = CppTokenPool.Make(CppTokenKind.PreprocessorDefineSource, Buffer.LexemeRange, defineValueResult.IsComplete);
PushToken(defineValueToken);
}
break;
case "defined":
{
if (Buffer.Peek() == '(')
{
Buffer.AdvanceColumn();
if (!SyntaxUtils.IsIdentStart(Buffer.Peek()))
{
AddError(Buffer.TextPosition, $"Expect identifier for defined, but got '{Buffer.Peek()}'", "Preprocessor");
return(false);
}
LexResult definedValueResult = LexIdent(false);
CppToken definedValueToken = CppTokenPool.Make(CppTokenKind.PreprocessorDefineTarget, Buffer.LexemeRange, definedValueResult.IsComplete);
PushToken(definedValueToken);
Buffer.SkipSpacings(TextStream.SkipType.All);
if (Buffer.Peek() != ')')
{
AddError(Buffer.TextPosition, $"Unterminated defined token, expect ')' but got '{Buffer.Peek()}'", "Preprocessor");
return(false);
}
}
}
break;
case "include":
{
char n = Buffer.Peek();
if (n == '<' || n == '"')
{
bool isComplete = false;
Buffer.AdvanceColumn();
char quote = (n == '<') ? '>' : n;
while (!Buffer.IsEOF)
{
if (Buffer.Peek() == quote)
{
isComplete = true;
Buffer.AdvanceColumn();
break;
}
Buffer.AdvanceColumn();
}
CppToken includeToken = CppTokenPool.Make(CppTokenKind.PreprocessorInclude, Buffer.LexemeRange, isComplete);
PushToken(includeToken);
}
else
{
return(false);
}
}
break;
case "pragma":
{
// @NOTE(final): Just skip until end-of-line
while (!Buffer.IsEOF)
{
char c = Buffer.Peek();
if (SyntaxUtils.IsLineBreak(c))
{
break;
}
if (c == '\t')
{
Buffer.AdvanceTab();
}
else
{
Buffer.AdvanceColumn();
}
}
}
break;
}
}
}
else
{
if (!LexNext(state))
{
break;
}
}
} while (!Buffer.IsEOF);
state.EndPreprocessor();
PushToken(CppTokenPool.Make(CppTokenKind.PreprocessorEnd, new TextRange(Buffer.TextPosition, 0), true));
return(true);
}