/// <summary> /// Tokenize the source code described by the given range into raw lexical tokens. /// </summary> /// <param name="range">The range to tokenize.</param> /// <returns>A list of raw lexical tokens.</returns> public IEnumerable <Token> Tokenize(SourceRange range, TokenKind filter = TokenKind.All) { IntPtr tokenPtr; uint numTokens; Interop.clang_tokenize(Native, range.Native, out tokenPtr, out numTokens); var tokens = new Interop.Token[numTokens]; try { unsafe { for (uint i = 0; i < numTokens; ++i) { tokens[i] = *(Interop.Token *)(tokenPtr + (int)(i * sizeof(Interop.Token))); } return(tokens .Where(t => filter == TokenKind.All || Interop.clang_getTokenKind(t) == filter) .Select(token => new Token(token, this))); } } finally { Interop.clang_disposeTokens(Native, tokenPtr, numTokens); } }
/// <summary> /// Tokenize the source code described by the given range into raw lexical tokens. /// </summary> /// <param name="range">The range to tokenize.</param> /// <returns>A list of raw lexical tokens.</returns> public IEnumerable<Token> Tokenize(SourceRange range, TokenKind filter = TokenKind.All) { IntPtr tokenPtr; uint numTokens; Interop.clang_tokenize(Native, range.Native, out tokenPtr, out numTokens); var tokens = new Interop.Token[numTokens]; try { unsafe { for (uint i = 0; i < numTokens; ++i) { tokens[i] = *(Interop.Token*)(tokenPtr + (int)(i * sizeof(Interop.Token))); } return tokens .Where(t => filter == TokenKind.All || Interop.clang_getTokenKind(t) == filter) .Select(token => new Token(token, this)); } } finally { Interop.clang_disposeTokens(Native, tokenPtr, numTokens); } }