Lex() 공개 메소드

public Lex ( ) : SyntaxToken
리턴 SyntaxToken
예제 #1
0
        private int EnsureToken(int tokenIndex)
        {
            _cancellationToken.ThrowIfCancellationRequested();

            if (_tokens.Any() && _tokens.Last().Kind == SyntaxKind.EndOfFileToken && tokenIndex == _tokens.Count - 1)
            {
                return(_tokens.Count - 1);
            }

            List <SyntaxToken> badTokens = null;

            while (tokenIndex >= _tokens.Count)
            {
                var token = _lexer.Lex();

                // Skip any bad tokens.
                if (token.Kind == SyntaxKind.BadToken)
                {
                    if (badTokens == null)
                    {
                        badTokens = new List <SyntaxToken>();
                    }
                    badTokens.Clear();
                    while (token.Kind == SyntaxKind.BadToken)
                    {
                        badTokens.Add(token);
                        token = _lexer.Lex();
                    }
                }

                if (badTokens != null && badTokens.Count > 0)
                {
                    var trivia = ImmutableArray.Create(CreateSkippedTokensTrivia(badTokens))
                                 .Concat(token.LeadingTrivia).ToImmutableArray();
                    token = token.WithLeadingTrivia(trivia);
                }

                _tokens.Add(token);

                if (token.Kind == SyntaxKind.EndOfFileToken)
                {
                    break;
                }
            }

            return(Math.Min(tokenIndex, _tokens.Count - 1));
        }
예제 #2
0
        public static IReadOnlyList<SyntaxToken> ParseAllTokens(SourceText sourceText)
        {
            var tokens = new List<SyntaxToken>();

            var lexer = new UnityLexer(sourceText);
            SyntaxToken token;
            do
            {
                tokens.Add(token = lexer.Lex());
            } while (token.Kind != SyntaxKind.EndOfFileToken);

            return tokens;
        }