public void SkipAllWhitespaces()
{
do
{
char c0 = Peek();
char c1 = Peek(1);
if (c0 == InvalidCharacter)
{
break;
}
else if (c0 == '\t')
{
AdvanceTab();
}
else if (SyntaxUtils.IsLineBreak(c0))
{
int nb = SyntaxUtils.GetLineBreakChars(c0, c1);
AdvanceLine(nb);
}
else if (char.IsWhiteSpace(c0))
{
AdvanceColumn();
}
else
{
break;
}
} while (!IsEOF);
}
public int AdvanceAuto(int numChars = 1)
{
// @NOTE(final): This is super slow, so only use it when needed
Debug.Assert(numChars >= 1);
TextPosition p = TextPosition;
int result = 0;
while (result < numChars)
{
char c0 = Peek();
char c1 = Peek(1);
if (SyntaxUtils.IsLineBreak(c0))
{
int lb = SyntaxUtils.GetLineBreakChars(c0, c1);
p.Line++;
p.Column = 1;
p.Index += lb;
result += lb;
}
if (c0 == '\t')
{
p.Column += ColumnsPerTab;
p.Index++;
result++;
}
else
{
p.Column++;
p.Index++;
result++;
}
}
TextPosition = p;
return(result);
}
public void AdvanceManual(char first, char second)
{
if (first == '\t')
{
AdvanceTab();
}
else if (SyntaxUtils.IsLineBreak(first))
{
int nb = SyntaxUtils.GetLineBreakChars(first, second);
AdvanceLine(nb);
}
else
{
AdvanceColumn();
}
}
private bool LexUntilCodeEnd(CommandResult commandResult)
{
// Special case, we dont want to parse doxygen stuff inside a code section
// so we wait until a @endcode follows
bool isComplete = false;
while (!Buffer.IsEOF)
{
char c0 = Buffer.Peek();
char c1 = Buffer.Peek(1);
if ((c0 == '@' || c0 == '\\') && SyntaxUtils.IsIdentStart(c1))
{
Buffer.StartLexeme();
Buffer.AdvanceColumn();
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsIdentPart);
string ident = Buffer.GetSourceText(Buffer.LexemeStart.Index + 1, Buffer.LexemeWidth - 1);
if ("endcode".Equals(ident))
{
PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.CommandEnd, Buffer.LexemeRange, true));
isComplete = true;
break;
}
}
else if (SyntaxUtils.IsLineBreak(c0))
{
int lb = SyntaxUtils.GetLineBreakChars(c0, c1);
Buffer.AdvanceLine(lb);
}
else if ('\t'.Equals(c0))
{
Buffer.AdvanceTab();
}
else
{
Buffer.AdvanceColumn();
}
}
if (!isComplete)
{
AddError(commandResult.StartPos, $"Unterminated code-block, expect '@endcode' or '\\endcode'", "Code", commandResult.CommandName);
return(false);
}
return(true);
}
public void SkipLineBreaks(SkipType type)
{
do
{
char c0 = Peek();
char c1 = Peek(1);
if (c0 == InvalidCharacter)
{
break;
}
else if (SyntaxUtils.IsLineBreak(c0))
{
int lb = SyntaxUtils.GetLineBreakChars(c0, c1);
AdvanceLine(lb);
}
else
{
break;
}
} while (!IsEOF && type == SkipType.All);
}
protected override bool LexNext(State hiddenState)
{
CppLexerState state = (CppLexerState)hiddenState;
bool allowWhitespaces = !state.IsInsidePreprocessor;
if (allowWhitespaces)
{
Buffer.SkipAllWhitespaces();
}
if (Buffer.IsEOF)
{
return(false);
}
int line = Buffer.TextPosition.Line;
char first = Buffer.Peek();
char second = Buffer.Peek(1);
char third = Buffer.Peek(2);
Buffer.StartLexeme();
LexResult lexRes = new LexResult(CppTokenKind.Unknown, true);
switch (first)
{
case '&':
{
if (second == '&')
{
lexRes.Kind = CppTokenKind.LogicalAndOp;
Buffer.AdvanceColumns(2);
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.AndAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.AndOp;
Buffer.AdvanceColumn();
}
}
break;
case '|':
{
if (second == '|')
{
lexRes.Kind = CppTokenKind.LogicalOrOp;
Buffer.AdvanceColumns(2);
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.OrAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.OrOp;
Buffer.AdvanceColumn();
}
}
break;
case '=':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.LogicalEqualsOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.EqOp;
Buffer.AdvanceColumn();
}
}
break;
case '!':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.LogicalNotEqualsOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.ExclationMark;
Buffer.AdvanceColumn();
}
}
break;
case '<':
{
if (second == '<')
{
if (third == '=')
{
lexRes.Kind = CppTokenKind.LeftShiftAssign;
Buffer.AdvanceColumns(3);
}
else
{
lexRes.Kind = CppTokenKind.LeftShiftOp;
Buffer.AdvanceColumns(2);
}
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.LessOrEqualOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.LessThanOp;
Buffer.AdvanceColumn();
}
}
break;
case '>':
{
if (second == '>')
{
if (third == '=')
{
lexRes.Kind = CppTokenKind.RightShiftAssign;
Buffer.AdvanceColumns(3);
}
else
{
lexRes.Kind = CppTokenKind.RightShiftOp;
Buffer.AdvanceColumns(2);
}
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.GreaterOrEqualOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.GreaterThanOp;
Buffer.AdvanceColumn();
}
}
break;
case '+':
{
if (second == '+')
{
lexRes.Kind = CppTokenKind.IncOp;
Buffer.AdvanceColumns(2);
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.AddAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.AddOp;
Buffer.AdvanceColumn();
}
}
break;
case '-':
{
if (second == '-')
{
lexRes.Kind = CppTokenKind.DecOp;
Buffer.AdvanceColumns(2);
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.SubAssign;
Buffer.AdvanceColumns(2);
}
else if (second == '>')
{
lexRes.Kind = CppTokenKind.PtrOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.SubOp;
Buffer.AdvanceColumn();
}
}
break;
case '/':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.DivAssign;
Buffer.AdvanceColumns(2);
}
else if (second == '/')
{
lexRes = LexSingleLineComment(Buffer, true);
if (!lexRes.IsComplete)
{
AddError(Buffer.LexemeStart, $"Unterminated single-line comment, expect '\n' or '\r' but found '{Buffer.Peek()}'", lexRes.Kind.ToString());
}
}
else if (second == '*')
{
lexRes = LexMultiLineComment(Buffer, true);
if (!lexRes.IsComplete)
{
AddError(Buffer.LexemeStart, $"Unterminated single-line comment, expect '*/' but found '{Buffer.Peek()}'", lexRes.Kind.ToString());
}
}
else
{
Buffer.AdvanceColumn();
lexRes.Kind = CppTokenKind.DivOp;
}
}
break;
case '*':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.MulAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.MulOp;
Buffer.AdvanceColumn();
}
}
break;
case '%':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.ModAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.ModOp;
Buffer.AdvanceColumn();
}
}
break;
case '.':
{
if (second == '.' && third == '.')
{
lexRes.Kind = CppTokenKind.Ellipsis;
Buffer.AdvanceColumns(3);
}
else if (SyntaxUtils.IsNumeric(second))
{
lexRes = LexNumber();
}
else
{
lexRes.Kind = CppTokenKind.Dot;
Buffer.AdvanceColumn();
}
}
break;
case '^':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.XorAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.XorOp;
Buffer.AdvanceColumn();
}
}
break;
case '#':
return(LexPreprocessor(state));
case '"':
lexRes = LexString("string");
break;
case '\'':
lexRes = LexString("char");
break;
case '~':
lexRes.Kind = CppTokenKind.Tilde;
Buffer.AdvanceColumn();
break;
case '\\':
lexRes.Kind = CppTokenKind.Backslash;
Buffer.AdvanceColumn();
break;
case ',':
lexRes.Kind = CppTokenKind.Comma;
Buffer.AdvanceColumn();
break;
case ';':
lexRes.Kind = CppTokenKind.Semicolon;
Buffer.AdvanceColumn();
break;
case ':':
lexRes.Kind = CppTokenKind.Colon;
Buffer.AdvanceColumn();
break;
case '?':
lexRes.Kind = CppTokenKind.QuestionMark;
Buffer.AdvanceColumn();
break;
case '{':
lexRes.Kind = CppTokenKind.LeftBrace;
Buffer.AdvanceColumn();
break;
case '}':
lexRes.Kind = CppTokenKind.RightBrace;
Buffer.AdvanceColumn();
break;
case '[':
lexRes.Kind = CppTokenKind.LeftBracket;
Buffer.AdvanceColumn();
break;
case ']':
lexRes.Kind = CppTokenKind.RightBracket;
Buffer.AdvanceColumn();
break;
case '(':
lexRes.Kind = CppTokenKind.LeftParen;
Buffer.AdvanceColumn();
break;
case ')':
lexRes.Kind = CppTokenKind.RightParen;
Buffer.AdvanceColumn();
break;
default:
{
if (SyntaxUtils.IsLineBreak(first) && allowWhitespaces)
{
lexRes.Kind = CppTokenKind.EndOfLine;
int nb = SyntaxUtils.GetLineBreakChars(first, second);
Buffer.AdvanceLine(nb);
}
else if (first == '\t' && allowWhitespaces)
{
lexRes.Kind = CppTokenKind.Spacings;
while (!Buffer.IsEOF)
{
if (Buffer.Peek() != '\t')
{
break;
}
Buffer.AdvanceTab();
}
}
else if (SyntaxUtils.IsSpacing(first) && allowWhitespaces)
{
lexRes.Kind = CppTokenKind.Spacings;
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsSpacing);
}
else if (SyntaxUtils.IsIdentStart(first))
{
Debug.Assert(!state.IsInsidePreprocessor);
lexRes = LexIdent(false);
}
else if (SyntaxUtils.IsNumeric(first))
{
lexRes = LexNumber();
}
else
{
AddError(Buffer.TextPosition, $"Unexpected character '{first}'", "Character");
return(false);
}
}
break;
}
return(PushToken(CppTokenPool.Make(lexRes.Kind, Buffer.LexemeRange, lexRes.IsComplete)));
}
private bool LexPreprocessor(CppLexerState state)
{
Debug.Assert(Buffer.Peek() == '#');
state.StartPreprocessor();
// Preprocessor start
Buffer.StartLexeme();
Buffer.AdvanceColumn();
PushToken(CppTokenPool.Make(CppTokenKind.PreprocessorStart, Buffer.LexemeRange, true));
do
{
Buffer.SkipSpacings(TextStream.SkipType.All);
Buffer.StartLexeme();
char first = Buffer.Peek();
char second = Buffer.Peek(1);
char third = Buffer.Peek(2);
if (first == '\\')
{
if (SyntaxUtils.IsLineBreak(second))
{
Buffer.AdvanceColumn();
int lb = SyntaxUtils.GetLineBreakChars(second, third);
Buffer.AdvanceLine(lb);
continue;
}
else
{
AddError(Buffer.TextPosition, $"Unterminated preprocessor next-line, expect linebreak after '\' but got '{second}'", "Preprocessor");
return(false);
}
}
else if (first == '#')
{
Buffer.AdvanceColumn();
PushToken(CppTokenPool.Make(CppTokenKind.PreprocessorOperator, Buffer.LexemeRange, true));
}
else if (SyntaxUtils.IsLineBreak(first))
{
int lb = SyntaxUtils.GetLineBreakChars(first, second);
Buffer.AdvanceLine(lb);
PushToken(CppTokenPool.Make(CppTokenKind.EndOfLine, Buffer.LexemeRange, true));
break;
}
else if (SyntaxUtils.IsIdentStart(first))
{
LexResult identResult = LexIdent(true);
CppToken identToken = CppTokenPool.Make(identResult.Kind, Buffer.LexemeRange, identResult.IsComplete);
PushToken(identToken);
Buffer.SkipSpacings(TextStream.SkipType.All);
Buffer.StartLexeme();
if (identToken.Kind == CppTokenKind.PreprocessorKeyword)
{
switch (identToken.Value)
{
case "define":
{
if (!SyntaxUtils.IsIdentStart(Buffer.Peek()))
{
AddError(Buffer.TextPosition, $"Expect identifier for define, but got '{Buffer.Peek()}'", "Preprocessor");
return(false);
}
LexResult defineValueResult = LexIdent(false);
CppToken defineValueToken = CppTokenPool.Make(CppTokenKind.PreprocessorDefineSource, Buffer.LexemeRange, defineValueResult.IsComplete);
PushToken(defineValueToken);
}
break;
case "defined":
{
if (Buffer.Peek() == '(')
{
Buffer.AdvanceColumn();
if (!SyntaxUtils.IsIdentStart(Buffer.Peek()))
{
AddError(Buffer.TextPosition, $"Expect identifier for defined, but got '{Buffer.Peek()}'", "Preprocessor");
return(false);
}
LexResult definedValueResult = LexIdent(false);
CppToken definedValueToken = CppTokenPool.Make(CppTokenKind.PreprocessorDefineTarget, Buffer.LexemeRange, definedValueResult.IsComplete);
PushToken(definedValueToken);
Buffer.SkipSpacings(TextStream.SkipType.All);
if (Buffer.Peek() != ')')
{
AddError(Buffer.TextPosition, $"Unterminated defined token, expect ')' but got '{Buffer.Peek()}'", "Preprocessor");
return(false);
}
}
}
break;
case "include":
{
char n = Buffer.Peek();
if (n == '<' || n == '"')
{
bool isComplete = false;
Buffer.AdvanceColumn();
char quote = (n == '<') ? '>' : n;
while (!Buffer.IsEOF)
{
if (Buffer.Peek() == quote)
{
isComplete = true;
Buffer.AdvanceColumn();
break;
}
Buffer.AdvanceColumn();
}
CppToken includeToken = CppTokenPool.Make(CppTokenKind.PreprocessorInclude, Buffer.LexemeRange, isComplete);
PushToken(includeToken);
}
else
{
return(false);
}
}
break;
case "pragma":
{
// @NOTE(final): Just skip until end-of-line
while (!Buffer.IsEOF)
{
char c = Buffer.Peek();
if (SyntaxUtils.IsLineBreak(c))
{
break;
}
if (c == '\t')
{
Buffer.AdvanceTab();
}
else
{
Buffer.AdvanceColumn();
}
}
}
break;
}
}
}
else
{
if (!LexNext(state))
{
break;
}
}
} while (!Buffer.IsEOF);
state.EndPreprocessor();
PushToken(CppTokenPool.Make(CppTokenKind.PreprocessorEnd, new TextRange(Buffer.TextPosition, 0), true));
return(true);
}
public static CToken PeekTokenRaw(TextStream stream)
{
TextPosition startPos = stream.TextPosition;
if (stream.IsEOF)
{
return(new CToken(CppTokenKind.Eof, startPos, stream.TextPosition));
}
char first = stream.Peek(0);
char second = stream.Peek(1);
char third = stream.Peek(2);
CppTokenKind kind;
switch (first)
{
case '&':
{
if (second == '&')
{
kind = CppTokenKind.LogicalAndOp;
stream.AdvanceColumns(2);
}
else if (second == '=')
{
kind = CppTokenKind.AndAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.AndOp;
stream.AdvanceColumn();
}
}
break;
case '|':
{
if (second == '|')
{
kind = CppTokenKind.LogicalOrOp;
stream.AdvanceColumns(2);
}
else if (second == '=')
{
kind = CppTokenKind.OrAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.OrOp;
stream.AdvanceColumn();
}
}
break;
case '=':
{
if (second == '=')
{
kind = CppTokenKind.LogicalEqualsOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.EqOp;
stream.AdvanceColumn();
}
}
break;
case '!':
{
if (second == '=')
{
kind = CppTokenKind.LogicalNotEqualsOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.ExclationMark;
stream.AdvanceColumn();
}
}
break;
case '<':
{
if (second == '<')
{
if (third == '=')
{
kind = CppTokenKind.LeftShiftAssign;
stream.AdvanceColumns(3);
}
else
{
kind = CppTokenKind.LeftShiftOp;
stream.AdvanceColumns(2);
}
}
else if (second == '=')
{
kind = CppTokenKind.LessOrEqualOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.LessThanOp;
stream.AdvanceColumn();
}
}
break;
case '>':
{
if (second == '>')
{
if (third == '=')
{
kind = CppTokenKind.RightShiftAssign;
stream.AdvanceColumns(3);
}
else
{
kind = CppTokenKind.RightShiftOp;
stream.AdvanceColumns(2);
}
}
else if (second == '=')
{
kind = CppTokenKind.GreaterOrEqualOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.GreaterThanOp;
stream.AdvanceColumn();
}
}
break;
case '+':
{
if (second == '+')
{
kind = CppTokenKind.IncOp;
stream.AdvanceColumns(2);
}
else if (second == '=')
{
kind = CppTokenKind.AddAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.AddOp;
stream.AdvanceColumn();
}
}
break;
case '-':
{
if (second == '-')
{
kind = CppTokenKind.DecOp;
stream.AdvanceColumns(2);
}
else if (second == '=')
{
kind = CppTokenKind.SubAssign;
stream.AdvanceColumns(2);
}
else if (second == '>')
{
kind = CppTokenKind.PtrOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.SubOp;
stream.AdvanceColumn();
}
}
break;
case '/':
{
if (second == '=')
{
kind = CppTokenKind.DivAssign;
stream.AdvanceColumns(2);
}
else if (second == '/')
{
kind = CppTokenKind.SingleLineComment;
stream.AdvanceColumns(2);
char specialChar = stream.Peek();
if (DoxygenSyntax.SingleLineDocChars.Contains(specialChar))
{
kind = CppTokenKind.SingleLineCommentDoc;
stream.AdvanceColumn();
}
while (!stream.IsEOF)
{
if (SyntaxUtils.IsLineBreak(stream.Peek()))
{
break;
}
stream.AdvanceColumn();
}
}
else if (second == '*')
{
kind = CppTokenKind.MultiLineComment;
stream.AdvanceColumns(2);
char specialChar = stream.Peek();
if (DoxygenSyntax.MultiLineDocChars.Contains(specialChar))
{
kind = CppTokenKind.MultiLineCommentDoc;
stream.AdvanceColumn();
}
while (!stream.IsEOF)
{
char n0 = stream.Peek();
char n1 = stream.Peek(1);
if (n0 == '*' && n1 == '/')
{
stream.AdvanceColumns(2);
break;
}
else
{
stream.AdvanceManual(n0, n1);
}
}
}
else
{
stream.AdvanceColumn();
kind = CppTokenKind.DivOp;
}
}
break;
case '*':
{
if (second == '=')
{
kind = CppTokenKind.MulAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.MulOp;
stream.AdvanceColumn();
}
}
break;
case '%':
{
if (second == '=')
{
kind = CppTokenKind.ModAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.ModOp;
stream.AdvanceColumn();
}
}
break;
case '.':
{
if (second == '.' && third == '.')
{
kind = CppTokenKind.Ellipsis;
stream.AdvanceColumns(3);
}
else if (SyntaxUtils.IsNumeric(second))
{
stream.AdvanceColumn();
kind = ReadNumberLiteral(stream);
}
else
{
kind = CppTokenKind.Dot;
stream.AdvanceColumn();
}
}
break;
case '^':
{
if (second == '=')
{
kind = CppTokenKind.XorAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.XorOp;
stream.AdvanceColumn();
}
}
break;
case '"':
case '\'':
{
stream.AdvanceColumn();
if (first == '"')
{
kind = ReadStringLiteral(stream);
}
else
{
kind = ReadCharacterLiteral(stream);
}
}
break;
case '~':
kind = CppTokenKind.Tilde;
stream.AdvanceColumn();
break;
case '\\':
kind = CppTokenKind.Backslash;
stream.AdvanceColumn();
break;
case '#':
kind = CppTokenKind.PreprocessorStart;
stream.AdvanceColumn();
break;
case ',':
kind = CppTokenKind.Comma;
stream.AdvanceColumn();
break;
case ';':
kind = CppTokenKind.Semicolon;
stream.AdvanceColumn();
break;
case ':':
kind = CppTokenKind.Colon;
stream.AdvanceColumn();
break;
case '?':
kind = CppTokenKind.QuestionMark;
stream.AdvanceColumn();
break;
case '{':
kind = CppTokenKind.LeftBrace;
stream.AdvanceColumn();
break;
case '}':
kind = CppTokenKind.RightBrace;
stream.AdvanceColumn();
break;
case '[':
kind = CppTokenKind.LeftBracket;
stream.AdvanceColumn();
break;
case ']':
kind = CppTokenKind.RightBracket;
stream.AdvanceColumn();
break;
case '(':
kind = CppTokenKind.LeftParen;
stream.AdvanceColumn();
break;
case ')':
kind = CppTokenKind.RightParen;
stream.AdvanceColumn();
break;
default:
{
if (SyntaxUtils.IsLineBreak(first))
{
kind = CppTokenKind.EndOfLine;
int nb = SyntaxUtils.GetLineBreakChars(first, second);
stream.AdvanceLine(nb);
}
else if (first == '\t')
{
kind = CppTokenKind.Spacings;
while (!stream.IsEOF)
{
if (stream.Peek() != '\t')
{
break;
}
stream.AdvanceTab();
}
}
else if (SyntaxUtils.IsSpacing(first))
{
kind = CppTokenKind.Spacings;
stream.AdvanceColumnsWhile(SyntaxUtils.IsSpacing);
}
else if (SyntaxUtils.IsIdentStart(first))
{
kind = CppTokenKind.IdentLiteral;
stream.AdvanceColumnsWhile(SyntaxUtils.IsIdentPart);
}
else if (SyntaxUtils.IsNumeric(first))
{
kind = ReadNumberLiteral(stream);
}
else
{
kind = CppTokenKind.Unknown;
}
}
break;
}
CToken result = MakeToken(kind, stream, startPos, stream.TextPosition);
return(result);
}
