public override IEnumerable <Token> Tokenize(TextReader textReader)
{
var text = new LinedInputText(textReader);
while (text.ReadLine())
{
while (!text.EndOfLine)
{
var current = GetToken(text);
if (current == null)
{
yield return(CreateUnknownToken(text));
yield break;
}
text.Advance(current.Lexeme.Length);
if (SkippableTokenKinds.Contains(current.Kind))
{
continue;
}
yield return(current);
}
}
}
public override IEnumerable<Token> Tokenize(string input)
{
var text = new InputText(input);
while (!text.EndOfInput)
{
var current = GetToken(text);
if (current == null)
{
yield return CreateUnknownToken(text);
yield break;
}
text.Advance(current.Lexeme.Length);
if (SkippableTokenKinds.Contains(current.Kind))
continue;
yield return current;
}
}
protected LexerBase(SpecialTokenConfig specialTokens, IReadOnlyList <MatchableTokenKindSpec> tokenKindSpecs)
{
ArgumentCheck.NotNullOrEmptyOrWithNulls(tokenKindSpecs, nameof(tokenKindSpecs));
var tokenKinds = new List <MatchableTokenKind>(tokenKindSpecs.Count);
foreach (var tokenKindSpec in tokenKindSpecs)
{
var tokenKind = tokenKindSpec.TokenKind;
tokenKinds.Add(tokenKind);
if (tokenKindSpec.Skipped)
{
SkippableTokenKinds.Add(tokenKind);
}
}
TokenKinds = tokenKinds;
_unknown = specialTokens?.Unknown ?? TokenKind.Unknown;
}
protected virtual bool ShouldSkipToken(Token token, TTokenizationContext context) => SkippableTokenKinds.Contains(token.Kind);
