public static CToken GetToken(TextStream cursor)
{
CToken result = PeekToken(cursor);
if (result.Kind != CppTokenKind.Eof)
{
Debug.Assert(result.Length > 0);
cursor.Seek(result.End);
}
return(result);
}
private static CToken MakeToken(CppTokenKind kind, TextStream stream, TextPosition startPos, TextPosition endPos)
{
int length = endPos.Index - startPos.Index;
string value = stream.GetSourceText(startPos.Index, length);
if (kind == CppTokenKind.IdentLiteral)
{
if (CppLexer.ReservedKeywords.Contains(value))
{
kind = CppTokenKind.ReservedKeyword;
}
else if (CppLexer.TypeKeywords.Contains(value) || CppLexer.GlobalClassKeywords.Contains(value))
{
kind = CppTokenKind.TypeKeyword;
}
}
CToken result = new CToken(kind, startPos, endPos, value);
return(result);
}
public static CToken PeekToken(TextStream cursor)
{
TextPosition startPos = cursor.TextPosition;
CToken token = PeekTokenRaw(cursor);
do
{
if (token.Kind == CppTokenKind.Eof)
{
break;
}
if (!(token.Kind == CppTokenKind.Spacings || token.Kind == CppTokenKind.EndOfLine))
{
break;
}
Debug.Assert(token.Length > 0);
cursor.Seek(token.End);
token = PeekTokenRaw(cursor);
} while (!cursor.IsEOF);
cursor.Seek(startPos);
return(token);
}
public static CToken PeekTokenRaw(TextStream stream)
{
TextPosition startPos = stream.TextPosition;
if (stream.IsEOF)
{
return(new CToken(CppTokenKind.Eof, startPos, stream.TextPosition));
}
char first = stream.Peek(0);
char second = stream.Peek(1);
char third = stream.Peek(2);
CppTokenKind kind;
switch (first)
{
case '&':
{
if (second == '&')
{
kind = CppTokenKind.LogicalAndOp;
stream.AdvanceColumns(2);
}
else if (second == '=')
{
kind = CppTokenKind.AndAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.AndOp;
stream.AdvanceColumn();
}
}
break;
case '|':
{
if (second == '|')
{
kind = CppTokenKind.LogicalOrOp;
stream.AdvanceColumns(2);
}
else if (second == '=')
{
kind = CppTokenKind.OrAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.OrOp;
stream.AdvanceColumn();
}
}
break;
case '=':
{
if (second == '=')
{
kind = CppTokenKind.LogicalEqualsOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.EqOp;
stream.AdvanceColumn();
}
}
break;
case '!':
{
if (second == '=')
{
kind = CppTokenKind.LogicalNotEqualsOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.ExclationMark;
stream.AdvanceColumn();
}
}
break;
case '<':
{
if (second == '<')
{
if (third == '=')
{
kind = CppTokenKind.LeftShiftAssign;
stream.AdvanceColumns(3);
}
else
{
kind = CppTokenKind.LeftShiftOp;
stream.AdvanceColumns(2);
}
}
else if (second == '=')
{
kind = CppTokenKind.LessOrEqualOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.LessThanOp;
stream.AdvanceColumn();
}
}
break;
case '>':
{
if (second == '>')
{
if (third == '=')
{
kind = CppTokenKind.RightShiftAssign;
stream.AdvanceColumns(3);
}
else
{
kind = CppTokenKind.RightShiftOp;
stream.AdvanceColumns(2);
}
}
else if (second == '=')
{
kind = CppTokenKind.GreaterOrEqualOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.GreaterThanOp;
stream.AdvanceColumn();
}
}
break;
case '+':
{
if (second == '+')
{
kind = CppTokenKind.IncOp;
stream.AdvanceColumns(2);
}
else if (second == '=')
{
kind = CppTokenKind.AddAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.AddOp;
stream.AdvanceColumn();
}
}
break;
case '-':
{
if (second == '-')
{
kind = CppTokenKind.DecOp;
stream.AdvanceColumns(2);
}
else if (second == '=')
{
kind = CppTokenKind.SubAssign;
stream.AdvanceColumns(2);
}
else if (second == '>')
{
kind = CppTokenKind.PtrOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.SubOp;
stream.AdvanceColumn();
}
}
break;
case '/':
{
if (second == '=')
{
kind = CppTokenKind.DivAssign;
stream.AdvanceColumns(2);
}
else if (second == '/')
{
kind = CppTokenKind.SingleLineComment;
stream.AdvanceColumns(2);
char specialChar = stream.Peek();
if (DoxygenSyntax.SingleLineDocChars.Contains(specialChar))
{
kind = CppTokenKind.SingleLineCommentDoc;
stream.AdvanceColumn();
}
while (!stream.IsEOF)
{
if (SyntaxUtils.IsLineBreak(stream.Peek()))
{
break;
}
stream.AdvanceColumn();
}
}
else if (second == '*')
{
kind = CppTokenKind.MultiLineComment;
stream.AdvanceColumns(2);
char specialChar = stream.Peek();
if (DoxygenSyntax.MultiLineDocChars.Contains(specialChar))
{
kind = CppTokenKind.MultiLineCommentDoc;
stream.AdvanceColumn();
}
while (!stream.IsEOF)
{
char n0 = stream.Peek();
char n1 = stream.Peek(1);
if (n0 == '*' && n1 == '/')
{
stream.AdvanceColumns(2);
break;
}
else
{
stream.AdvanceManual(n0, n1);
}
}
}
else
{
stream.AdvanceColumn();
kind = CppTokenKind.DivOp;
}
}
break;
case '*':
{
if (second == '=')
{
kind = CppTokenKind.MulAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.MulOp;
stream.AdvanceColumn();
}
}
break;
case '%':
{
if (second == '=')
{
kind = CppTokenKind.ModAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.ModOp;
stream.AdvanceColumn();
}
}
break;
case '.':
{
if (second == '.' && third == '.')
{
kind = CppTokenKind.Ellipsis;
stream.AdvanceColumns(3);
}
else if (SyntaxUtils.IsNumeric(second))
{
stream.AdvanceColumn();
kind = ReadNumberLiteral(stream);
}
else
{
kind = CppTokenKind.Dot;
stream.AdvanceColumn();
}
}
break;
case '^':
{
if (second == '=')
{
kind = CppTokenKind.XorAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.XorOp;
stream.AdvanceColumn();
}
}
break;
case '"':
case '\'':
{
stream.AdvanceColumn();
if (first == '"')
{
kind = ReadStringLiteral(stream);
}
else
{
kind = ReadCharacterLiteral(stream);
}
}
break;
case '~':
kind = CppTokenKind.Tilde;
stream.AdvanceColumn();
break;
case '\\':
kind = CppTokenKind.Backslash;
stream.AdvanceColumn();
break;
case '#':
kind = CppTokenKind.PreprocessorStart;
stream.AdvanceColumn();
break;
case ',':
kind = CppTokenKind.Comma;
stream.AdvanceColumn();
break;
case ';':
kind = CppTokenKind.Semicolon;
stream.AdvanceColumn();
break;
case ':':
kind = CppTokenKind.Colon;
stream.AdvanceColumn();
break;
case '?':
kind = CppTokenKind.QuestionMark;
stream.AdvanceColumn();
break;
case '{':
kind = CppTokenKind.LeftBrace;
stream.AdvanceColumn();
break;
case '}':
kind = CppTokenKind.RightBrace;
stream.AdvanceColumn();
break;
case '[':
kind = CppTokenKind.LeftBracket;
stream.AdvanceColumn();
break;
case ']':
kind = CppTokenKind.RightBracket;
stream.AdvanceColumn();
break;
case '(':
kind = CppTokenKind.LeftParen;
stream.AdvanceColumn();
break;
case ')':
kind = CppTokenKind.RightParen;
stream.AdvanceColumn();
break;
default:
{
if (SyntaxUtils.IsLineBreak(first))
{
kind = CppTokenKind.EndOfLine;
int nb = SyntaxUtils.GetLineBreakChars(first, second);
stream.AdvanceLine(nb);
}
else if (first == '\t')
{
kind = CppTokenKind.Spacings;
while (!stream.IsEOF)
{
if (stream.Peek() != '\t')
{
break;
}
stream.AdvanceTab();
}
}
else if (SyntaxUtils.IsSpacing(first))
{
kind = CppTokenKind.Spacings;
stream.AdvanceColumnsWhile(SyntaxUtils.IsSpacing);
}
else if (SyntaxUtils.IsIdentStart(first))
{
kind = CppTokenKind.IdentLiteral;
stream.AdvanceColumnsWhile(SyntaxUtils.IsIdentPart);
}
else if (SyntaxUtils.IsNumeric(first))
{
kind = ReadNumberLiteral(stream);
}
else
{
kind = CppTokenKind.Unknown;
}
}
break;
}
CToken result = MakeToken(kind, stream, startPos, stream.TextPosition);
return(result);
}
