public Token ConsumeToken(StringWalker w)
{
CodePos pos = w.Position;
string content = w.ConsumeWhile(char.IsLetterOrDigit);
return(new Token(pos, TokenType.Keyword, content));
}
public Token ConsumeToken(StringWalker w)
{
CodePos startPos = w.Position;
// Skip the opening quote, after asserting that it is indeed
// a quote.
char openingQuote = w.Peek();
if (openingQuote != '"')
{
throw new CompileErrorException(w.Position, "StringRule.ConsumeToken() called, but it didn't start on a quote.");
}
w.Consume(1);
var strContent = new StringBuilder();
while (w.Peek() != '"')
{
char c = w.Peek();
// If it's a backslash, it must be an escape sequence.
if (c == '\\')
{
// Skip the backslash, then use the next char
// to determine which character this escape sequence
// represents
w.Consume(1);
char codeChar = w.Consume(1)[0];
char resultChar = codeChar switch
{
'\\' => '\\',
'"' => '"',
'n' => '\n',
'r' => '\r',
_ => throw new CompileErrorException(w.Position, $"Invalid escape sequence \\{codeChar}")
};
strContent.Append(resultChar);
continue;
}
strContent.Append(w.Consume(1));
}
// Skip the ending quote
w.Consume(1);
return(new Token(
startPos,
TokenType.String,
strContent.ToString()
));
}
}
public IEnumerable <Token> ToTokens(IEnumerable <char> src)
{
var walker = new StringWalker(src);
var rules = new ILexerRule[]
{
new NumberRule(),
new KeywordRule(_keywords),
new WordRule(),
new StringRule(),
new SingleCharSymbolRule()
};
while (!walker.IsEmpty())
{
char c = walker.Peek();
// Skip all whitespace
if (char.IsWhiteSpace(c))
{
walker.Consume(1);
continue;
}
// Get the standard rules out of the way first.
bool handledByStandardRule = false;
foreach (var rule in rules)
{
if (rule.IsStartOfToken(walker))
{
handledByStandardRule = true;
yield return(rule.ConsumeToken(walker));
break;
}
}
if (handledByStandardRule)
{
continue;
}
// TODO: multi-character symbol tokens
// TODO: Special cases go here
// We didn't find any rule that matches what we're seeing,
// so throw an error.
CodePos pos = walker.Position;
throw new CompileErrorException(pos, $"Unexpected character '{c}'");
}
}
public Token ConsumeToken(StringWalker w)
{
CodePos pos = w.Position;
string sign = ConsumeSign();
string wholePart = ConsumeWholePart();
string decimalPart = ConsumeDecimalPart();
string content = $"{sign}{wholePart}{decimalPart}";
return(new Token(pos, TokenType.Number, content));
string ConsumeSign() => w.Peek() == '-'
? w.Consume(1)
: "";
string ConsumeWholePart() => w.ConsumeWhile(char.IsDigit);
string ConsumeDecimalPart()
{
if (w.IsEmpty() || w.Peek() != '.')
{
return("");
}
string decimalPoint = w.Consume(1);
string digits = w.ConsumeWhile(char.IsDigit);
// If there is nothing after the decimal point, that's an error
if (digits.Length == 0)
{
throw new CompileErrorException(w.Position, "There must be digits after the decimal point");
}
// There can only be one decimal point in the number. If there's
// another, that's an error.
if (!w.IsEmpty() && w.Peek() == '.')
{
throw new CompileErrorException(w.Position, "A number may only have one decimal point");
}
return($".{digits}");
}
}
public CompileErrorException(CodePos pos, string message)
: base($"Line {pos.LineNumber}, char {pos.CharNumber}: {message}")
{
Position = pos;
}
public Token(CodePos position, TokenType type, string content)
{
Position = position;
Type = type;
Content = content;
}