public Token ConsumeToken(StringWalker w)
{
CodePos pos = w.Position;
string sign = ConsumeSign();
string wholePart = ConsumeWholePart();
string decimalPart = ConsumeDecimalPart();
string content = $"{sign}{wholePart}{decimalPart}";
return(new Token(pos, TokenType.Number, content));
string ConsumeSign() => w.Peek() == '-'
? w.Consume(1)
: "";
string ConsumeWholePart() => w.ConsumeWhile(char.IsDigit);
string ConsumeDecimalPart()
{
if (w.IsEmpty() || w.Peek() != '.')
{
return("");
}
string decimalPoint = w.Consume(1);
string digits = w.ConsumeWhile(char.IsDigit);
// If there is nothing after the decimal point, that's an error
if (digits.Length == 0)
{
throw new CompileErrorException(w.Position, "There must be digits after the decimal point");
}
// There can only be one decimal point in the number. If there's
// another, that's an error.
if (!w.IsEmpty() && w.Peek() == '.')
{
throw new CompileErrorException(w.Position, "A number may only have one decimal point");
}
return($".{digits}");
}
}
public IEnumerable <Token> ToTokens(IEnumerable <char> src)
{
var walker = new StringWalker(src);
var rules = new ILexerRule[]
{
new NumberRule(),
new KeywordRule(_keywords),
new WordRule(),
new StringRule(),
new SingleCharSymbolRule()
};
while (!walker.IsEmpty())
{
char c = walker.Peek();
// Skip all whitespace
if (char.IsWhiteSpace(c))
{
walker.Consume(1);
continue;
}
// Get the standard rules out of the way first.
bool handledByStandardRule = false;
foreach (var rule in rules)
{
if (rule.IsStartOfToken(walker))
{
handledByStandardRule = true;
yield return(rule.ConsumeToken(walker));
break;
}
}
if (handledByStandardRule)
{
continue;
}
// TODO: multi-character symbol tokens
// TODO: Special cases go here
// We didn't find any rule that matches what we're seeing,
// so throw an error.
CodePos pos = walker.Position;
throw new CompileErrorException(pos, $"Unexpected character '{c}'");
}
}
