public CToken(CppTokenKind kind, TextPosition start, TextPosition end, string value = null)
{
Kind = kind;
Start = start;
End = end;
Value = value;
}
public static string TokensToString(IEnumerable <CppToken> tokens)
{
var builder = new StringBuilder();
CppTokenKind previousKind = 0;
foreach (var token in tokens)
{
if (token.Kind == CppTokenKind.Comment)
{
continue;
}
// If previous token and new token are identifiers/keyword, we need a space between them
if (previousKind.IsIdentifierOrKeyword() &&
token.Kind.IsIdentifierOrKeyword())
{
builder.Append(" ");
}
builder.Append(token.Text);
previousKind = token.Kind;
}
return(builder.ToString());
}
public static CppToken Make(CppTokenKind kind, TextRange range, bool isComplete)
{
if (_pool == null)
{
_pool = new ObjectPool <CppToken>(() => new CppToken());
}
CppToken result = _pool.Aquire();
result.Set(kind, range, isComplete);
return(result);
}
public static LexResult LexMultiLineComment(TextStream stream, bool init)
{
CppTokenKind kind = CppTokenKind.MultiLineComment;
if (init)
{
Debug.Assert(stream.Peek(0) == '/');
Debug.Assert(stream.Peek(1) == '*');
stream.AdvanceColumns(2);
char n = stream.Peek();
if (DoxygenSyntax.MultiLineDocChars.Contains(n))
{
stream.AdvanceColumn();
kind = CppTokenKind.MultiLineCommentDoc;
if (n == '*' && stream.Peek() == '/')
{
stream.AdvanceColumn();
return(new LexResult(kind, true));
}
}
}
bool isComplete = false;
while (!stream.IsEOF)
{
char c0 = stream.Peek();
if (c0 == '*')
{
char c1 = stream.Peek(1);
if (c1 == '/')
{
stream.AdvanceColumns(2);
isComplete = true;
break;
}
else
{
stream.AdvanceColumn();
}
}
else if (char.IsWhiteSpace(c0))
{
stream.SkipAllWhitespaces();
}
else
{
stream.AdvanceColumn();
}
}
return(new LexResult(kind, isComplete));
}
private static CToken MakeToken(CppTokenKind kind, TextStream stream, TextPosition startPos, TextPosition endPos)
{
int length = endPos.Index - startPos.Index;
string value = stream.GetSourceText(startPos.Index, length);
if (kind == CppTokenKind.IdentLiteral)
{
if (CppLexer.ReservedKeywords.Contains(value))
{
kind = CppTokenKind.ReservedKeyword;
}
else if (CppLexer.TypeKeywords.Contains(value) || CppLexer.GlobalClassKeywords.Contains(value))
{
kind = CppTokenKind.TypeKeyword;
}
}
CToken result = new CToken(kind, startPos, endPos, value);
return(result);
}
private LexResult LexIdent(bool isPreprocessor)
{
Debug.Assert(SyntaxUtils.IsIdentStart(Buffer.Peek()));
StringBuilder identBuffer = new StringBuilder();
while (!Buffer.IsEOF)
{
char c = Buffer.Peek();
if (SyntaxUtils.IsIdentPart(c))
{
identBuffer.Append(c);
Buffer.AdvanceColumn();
}
else
{
break;
}
}
CppTokenKind kind = CppTokenKind.IdentLiteral;
TextPosition identStart = Buffer.LexemeStart;
int identLength = Buffer.LexemeWidth;
string identString = identBuffer.ToString();
if (isPreprocessor && PreProcessorKeywords.Contains(identString))
{
kind = CppTokenKind.PreprocessorKeyword;
}
else if (ReservedKeywords.Contains(identString))
{
kind = CppTokenKind.ReservedKeyword;
}
else if (TypeKeywords.Contains(identString) || GlobalClassKeywords.Contains(identString))
{
kind = CppTokenKind.TypeKeyword;
}
else
{
kind = CppTokenKind.IdentLiteral;
}
return(new LexResult(kind, true));
}
public static LexResult LexSingleLineComment(TextStream stream, bool init)
{
CppTokenKind kind = CppTokenKind.SingleLineComment;
if (init)
{
Debug.Assert(stream.Peek(0) == '/');
Debug.Assert(stream.Peek(1) == '/');
stream.AdvanceColumns(2);
if (DoxygenSyntax.SingleLineDocChars.Contains(stream.Peek()))
{
stream.AdvanceColumn();
kind = CppTokenKind.SingleLineCommentDoc;
}
}
while (!stream.IsEOF)
{
char c0 = stream.Peek();
char c1 = stream.Peek(1);
if (c0 == TextStream.InvalidCharacter)
{
break;
}
else if (SyntaxUtils.IsLineBreak(c0))
{
break;
}
else if (c0 == '\t')
{
stream.AdvanceTab();
}
else
{
stream.AdvanceColumn();
}
}
bool isComplete = stream.IsEOF || SyntaxUtils.IsLineBreak(stream.Peek());
return(new LexResult(kind, isComplete));
}
/// <summary>
/// Gets a boolean indicating whether this token kind is an identifier or keyword
/// </summary>
/// <param name="kind">The token kind</param>
/// <returns><c>true</c> if the token is an identifier or keyword, <c>false</c> otherwise</returns>
public static bool IsIdentifierOrKeyword(this CppTokenKind kind)
{
return(kind == CppTokenKind.Identifier || kind == CppTokenKind.Keyword);
}
/// <summary>
/// Creates a new instance of a C++ token.
/// </summary>
/// <param name="kind">Kind of this token</param>
/// <param name="text">Text of this token</param>
public CppToken(CppTokenKind kind, string text)
{
Kind = kind;
Text = text;
}
public void Set(CppTokenKind kind, TextRange range, bool isComplete)
{
Set(range, isComplete);
Kind = kind;
}
private CppToken(CppTokenKind kind, TextRange range, bool isComplete) : base(range, isComplete)
{
Kind = kind;
}
private LexResult LexString(string typeName)
{
Debug.Assert(Buffer.Peek(0) == '"' || Buffer.Peek(0) == '\'');
char quoteChar = Buffer.Peek();
Buffer.AdvanceColumn();
bool isComplete = false;
CppTokenKind kind = quoteChar == '\'' ? CppTokenKind.CharLiteral : CppTokenKind.StringLiteral;
int maxCount = (kind == CppTokenKind.CharLiteral) ? 1 : -1;
int minCount = (kind == CppTokenKind.CharLiteral) ? 1 : 0;
int count = 0;
while (!Buffer.IsEOF)
{
char first = Buffer.Peek();
char second = Buffer.Peek(1);
if (first == quoteChar)
{
isComplete = true;
break;
}
else if (first == '\\')
{
switch (second)
{
case '\'':
case '"':
case '?':
case '\\':
case 'a':
case 'b':
case 'f':
case 'n':
case 'e':
case 'r':
case 't':
case 'v':
{
Buffer.AdvanceColumns(2);
++count;
continue;
}
case 'x':
case 'X':
case 'u':
case 'U':
{
Buffer.AdvanceColumns(2);
if (SyntaxUtils.IsHex(Buffer.Peek()))
{
int len = 0;
while (!Buffer.IsEOF)
{
if (!SyntaxUtils.IsHex(Buffer.Peek()))
{
break;
}
else
{
++len;
Buffer.AdvanceColumn();
}
}
}
else
{
AddError(Buffer.TextPosition, $"Unsupported hex escape character '{Buffer.Peek()}'!", typeName);
break;
}
++count;
continue;
}
default:
if (SyntaxUtils.IsOctal(second))
{
Buffer.AdvanceColumn();
while (!Buffer.IsEOF)
{
if (!SyntaxUtils.IsOctal(Buffer.Peek()))
{
break;
}
else
{
Buffer.AdvanceColumn();
}
}
++count;
continue;
}
else
{
AddError(Buffer.TextPosition, $"Not supported escape character '{Buffer.Peek()}'!", typeName);
break;
}
}
}
else if (SyntaxUtils.IsLineBreak(first))
{
break;
}
else if (char.IsWhiteSpace(first))
{
Buffer.AdvanceManual(first, second);
}
else
{
Buffer.AdvanceColumn();
}
++count;
}
// Skip over quote char
if (isComplete)
{
Debug.Assert(Buffer.Peek() == quoteChar);
Buffer.AdvanceColumn();
}
if (!isComplete)
{
AddError(Buffer.LexemeStart, $"Unterminated {typeName} literal!", typeName);
}
else
{
if (minCount > 0 && count < minCount)
{
AddError(Buffer.LexemeStart, $"Not enough characters for {typeName} literal, expect {minCount} but got {count}!", typeName);
}
else if (maxCount > -1 && (count > maxCount))
{
AddError(Buffer.LexemeStart, $"Too many characters for {typeName} literal, expect {maxCount} but got {count}!", typeName);
}
}
return(new LexResult(kind, isComplete));
}
public LexResult(CppTokenKind kind, bool isComplete)
{
Kind = kind;
IsComplete = isComplete;
}
