private int EnsureToken(int tokenIndex)
{
_cancellationToken.ThrowIfCancellationRequested();
if (_tokens.Any() && _tokens.Last().Kind == SyntaxKind.EndOfFileToken && tokenIndex == _tokens.Count - 1)
{
return(_tokens.Count - 1);
}
List <SyntaxToken> badTokens = null;
while (tokenIndex >= _tokens.Count)
{
var token = _lexer.Lex();
// Skip any bad tokens.
if (token.Kind == SyntaxKind.BadToken)
{
if (badTokens == null)
{
badTokens = new List <SyntaxToken>();
}
badTokens.Clear();
while (token.Kind == SyntaxKind.BadToken)
{
badTokens.Add(token);
token = _lexer.Lex();
}
}
if (badTokens != null && badTokens.Count > 0)
{
var trivia = ImmutableArray.Create(CreateSkippedTokensTrivia(badTokens))
.Concat(token.LeadingTrivia).ToImmutableArray();
token = token.WithLeadingTrivia(trivia);
}
_tokens.Add(token);
if (token.Kind == SyntaxKind.EndOfFileToken)
{
break;
}
}
return(Math.Min(tokenIndex, _tokens.Count - 1));
}
public static IReadOnlyList<SyntaxToken> ParseAllTokens(SourceText sourceText)
{
var tokens = new List<SyntaxToken>();
var lexer = new UnityLexer(sourceText);
SyntaxToken token;
do
{
tokens.Add(token = lexer.Lex());
} while (token.Kind != SyntaxKind.EndOfFileToken);
return tokens;
}