protected override IEnumerable <Result <DMToken> > Tokenize(TextSpan span)
{
var next = SkipWhiteSpace(span);
var lastFail = next.Location;
while (next.HasValue)
{
if (SkipWhiteSpace(ref next, DMToken.LeadWhitespace, out var whitespace))
{
yield return(whitespace);
}
if (!next.HasValue)
{
yield break;
}
if (next.Location.Length >= 2 && next.Location.First(2).ToString() == "//")
{
while (next.HasValue && next.Value != '\n')
{
next = next.Remainder.ConsumeChar();
}
}
else if (next.Location.Length >= 4 && next.Location.First(2).ToString() == "/*")
{
while (next.Location.First(2).ToString() != "*/")
{
next = next.Remainder.ConsumeChar();
}
next = next.Remainder.ConsumeChar();
next = next.Remainder.ConsumeChar();
}
else if (Numerics.Decimal(next.Location).HasValue)
{
var integer = Numerics.Decimal(next.Location);
next = integer.Remainder.ConsumeChar();
yield return(Result.Value(DMToken.NumericLiteral, integer.Location, integer.Remainder));
}
else if (TryKeyword(ref next, out var keyword))
{
yield return(keyword);
}