/// <summary>
/// Tokenize the source code described by the given range into raw lexical tokens.
/// </summary>
/// <param name="range">The range to tokenize.</param>
/// <returns>A list of raw lexical tokens.</returns>
public IEnumerable <Token> Tokenize(SourceRange range, TokenKind filter = TokenKind.All)
{
IntPtr tokenPtr;
uint numTokens;
Interop.clang_tokenize(Native, range.Native, out tokenPtr, out numTokens);
var tokens = new Interop.Token[numTokens];
try {
unsafe {
for (uint i = 0; i < numTokens; ++i)
{
tokens[i] = *(Interop.Token *)(tokenPtr + (int)(i * sizeof(Interop.Token)));
}
return(tokens
.Where(t => filter == TokenKind.All || Interop.clang_getTokenKind(t) == filter)
.Select(token => new Token(token, this)));
}
} finally {
Interop.clang_disposeTokens(Native, tokenPtr, numTokens);
}
}
