public IEnumerable <SpannedToken> GetTokens(IEnumerable <string> segments, int offset)
{
var lexer = new Parser.G4.SdmapLexer(new UnbufferedCharStream(new TextSegmentsCharStream(segments)));
while (true)
{
IToken current = null;
try
{
current = lexer.NextToken();
}
catch (InvalidOperationException e)
when(e.HResult == -2146233079) // stack empty
{
break;
}
if (current.Type == Parser.G4.SdmapLexer.Eof)
{
break;
}
if (current != null)
{
yield return(new SpannedToken(
current.Type,
new Span(current.StartIndex + offset, current.StopIndex - current.StartIndex + 1)));
}
}
}
public IEnumerable <SpannedToken> Run(IEnumerable <string> segments, int offset)
{
var lexer = new Parser.G4.SdmapLexer(new UnbufferedCharStream(new TextSegmentsCharStream(segments)));
while (true)
{
IToken current = lexer.NextToken();
if (current.Type == Parser.G4.SdmapLexer.Eof)
{
break;
}
yield return(new SpannedToken(
current.Type,
new Span(current.StartIndex + offset, current.StopIndex - current.StartIndex + 1)));
}
}