public IEnumerable<SpannedToken> Run(IEnumerable<string> segments, int offset) { var lexer = new RustLexer.RustLexer(new UnbufferedCharStream(new TextSegmentsCharStream(segments))); while (true) { IToken current = lexer.NextToken(); if (current.Type == RustLexer.RustLexer.Eof) break; yield return new SpannedToken(current.Type, new Span(current.StartIndex + offset, current.StopIndex - current.StartIndex + 1)); } }
public IEnumerable <SpannedToken> Run(IEnumerable <string> segments, int offset) { var lexer = new RustLexer.RustLexer(new UnbufferedCharStream(new TextSegmentsCharStream(segments))); while (true) { IToken current = lexer.NextToken(); if (current.Type == RustLexer.RustLexer.Eof) { break; } yield return(new SpannedToken(current.Type, new Span(current.StartIndex + offset, current.StopIndex - current.StartIndex + 1))); } }