public IEnumerable <Token> Tokenize(Plaintext input)
{
List <Match> matches = input.FirstMatches(this.Patterns).ToList();
Location location = input.Beginning;
while (matches.Any() && location is InnerLocation inner)
{
Token output = TokenAt(location, input, matches);
if (!(output is Ignored))
{
yield return(output);
}
matches = this.Advance(matches, output.LocationAfter).ToList();
location = output.LocationAfter;
}
if (location is InnerLocation remaining)
{
yield return(new InvalidInput(remaining, EndOfText.Value, input.Substring(remaining)));
}
yield return(new EndOfInput(location));
}
