public Token ConsumeToken(StringWalker w)
{
CodePos pos = w.Position;
string content = w.ConsumeWhile(char.IsLetterOrDigit);
return(new Token(pos, TokenType.Keyword, content));
}
public Token ConsumeToken(StringWalker w)
{
return(new Token(
w.Position,
TokenType.Symbol,
w.Consume(1)
));
}
public bool IsStartOfToken(StringWalker w)
{
if (!char.IsLetter(w.Peek()))
{
return(false);
}
string word = w.PeekWhile(char.IsLetterOrDigit);
return(_keywords.Contains(word));
}
public Token ConsumeToken(StringWalker w)
{
CodePos startPos = w.Position;
// Skip the opening quote, after asserting that it is indeed
// a quote.
char openingQuote = w.Peek();
if (openingQuote != '"')
{
throw new CompileErrorException(w.Position, "StringRule.ConsumeToken() called, but it didn't start on a quote.");
}
w.Consume(1);
var strContent = new StringBuilder();
while (w.Peek() != '"')
{
char c = w.Peek();
// If it's a backslash, it must be an escape sequence.
if (c == '\\')
{
// Skip the backslash, then use the next char
// to determine which character this escape sequence
// represents
w.Consume(1);
char codeChar = w.Consume(1)[0];
char resultChar = codeChar switch
{
'\\' => '\\',
'"' => '"',
'n' => '\n',
'r' => '\r',
_ => throw new CompileErrorException(w.Position, $"Invalid escape sequence \\{codeChar}")
};
strContent.Append(resultChar);
continue;
}
strContent.Append(w.Consume(1));
}
// Skip the ending quote
w.Consume(1);
return(new Token(
startPos,
TokenType.String,
strContent.ToString()
));
}
}
public IEnumerable <Token> ToTokens(IEnumerable <char> src)
{
var walker = new StringWalker(src);
var rules = new ILexerRule[]
{
new NumberRule(),
new KeywordRule(_keywords),
new WordRule(),
new StringRule(),
new SingleCharSymbolRule()
};
while (!walker.IsEmpty())
{
char c = walker.Peek();
// Skip all whitespace
if (char.IsWhiteSpace(c))
{
walker.Consume(1);
continue;
}
// Get the standard rules out of the way first.
bool handledByStandardRule = false;
foreach (var rule in rules)
{
if (rule.IsStartOfToken(walker))
{
handledByStandardRule = true;
yield return(rule.ConsumeToken(walker));
break;
}
}
if (handledByStandardRule)
{
continue;
}
// TODO: multi-character symbol tokens
// TODO: Special cases go here
// We didn't find any rule that matches what we're seeing,
// so throw an error.
CodePos pos = walker.Position;
throw new CompileErrorException(pos, $"Unexpected character '{c}'");
}
}
public bool IsStartOfToken(StringWalker walker)
{
char c = walker.Peek();
if (char.IsDigit(c))
{
return(true);
}
if (c != '-')
{
return(false);
}
// It's a dash. That means this COULD be the start of a negative number.
// Or...it might just be a standalone minus sign.
// We need to look at the surrounding characters to disambiguate.
// If the next char is not a digit, then it can't possibly be a negative
// number.
string group = walker.Peek(2);
if (group.Length < 2)
{
return(false);
}
if (!char.IsDigit(group[1]))
{
return(false);
}
// OK, it's a dash followed by a digit.
// But that's still not enough information!
// The string "32-31" should be interpreted as subtracting 32 and 31, while
// the string "32 -31" should be interpreted as 32 and -31.
// To tell the difference, we need to look back 1 character and see if it
// was whitespace.
char prevChar = walker.PeekBehind();
if (char.IsWhiteSpace(prevChar))
{
return(true);
}
return(false);
}
public Token ConsumeToken(StringWalker w)
{
CodePos pos = w.Position;
string sign = ConsumeSign();
string wholePart = ConsumeWholePart();
string decimalPart = ConsumeDecimalPart();
string content = $"{sign}{wholePart}{decimalPart}";
return(new Token(pos, TokenType.Number, content));
string ConsumeSign() => w.Peek() == '-'
? w.Consume(1)
: "";
string ConsumeWholePart() => w.ConsumeWhile(char.IsDigit);
string ConsumeDecimalPart()
{
if (w.IsEmpty() || w.Peek() != '.')
{
return("");
}
string decimalPoint = w.Consume(1);
string digits = w.ConsumeWhile(char.IsDigit);
// If there is nothing after the decimal point, that's an error
if (digits.Length == 0)
{
throw new CompileErrorException(w.Position, "There must be digits after the decimal point");
}
// There can only be one decimal point in the number. If there's
// another, that's an error.
if (!w.IsEmpty() && w.Peek() == '.')
{
throw new CompileErrorException(w.Position, "A number may only have one decimal point");
}
return($".{digits}");
}
}
public bool IsStartOfToken(StringWalker w)
{
return(_allowedSymbols.Contains(w.Peek()));
}
public bool IsStartOfToken(StringWalker w) => w.Peek() == '"';
public bool IsStartOfToken(StringWalker w)
{
return(char.IsLetter(w.Peek()));
}
