public Token ConsumeToken(StringWalker w) { CodePos pos = w.Position; string content = w.ConsumeWhile(char.IsLetterOrDigit); return(new Token(pos, TokenType.Keyword, content)); }
public Token ConsumeToken(StringWalker w) { return(new Token( w.Position, TokenType.Symbol, w.Consume(1) )); }
public bool IsStartOfToken(StringWalker w) { if (!char.IsLetter(w.Peek())) { return(false); } string word = w.PeekWhile(char.IsLetterOrDigit); return(_keywords.Contains(word)); }
public Token ConsumeToken(StringWalker w) { CodePos startPos = w.Position; // Skip the opening quote, after asserting that it is indeed // a quote. char openingQuote = w.Peek(); if (openingQuote != '"') { throw new CompileErrorException(w.Position, "StringRule.ConsumeToken() called, but it didn't start on a quote."); } w.Consume(1); var strContent = new StringBuilder(); while (w.Peek() != '"') { char c = w.Peek(); // If it's a backslash, it must be an escape sequence. if (c == '\\') { // Skip the backslash, then use the next char // to determine which character this escape sequence // represents w.Consume(1); char codeChar = w.Consume(1)[0]; char resultChar = codeChar switch { '\\' => '\\', '"' => '"', 'n' => '\n', 'r' => '\r', _ => throw new CompileErrorException(w.Position, $"Invalid escape sequence \\{codeChar}") }; strContent.Append(resultChar); continue; } strContent.Append(w.Consume(1)); } // Skip the ending quote w.Consume(1); return(new Token( startPos, TokenType.String, strContent.ToString() )); } }
public IEnumerable <Token> ToTokens(IEnumerable <char> src) { var walker = new StringWalker(src); var rules = new ILexerRule[] { new NumberRule(), new KeywordRule(_keywords), new WordRule(), new StringRule(), new SingleCharSymbolRule() }; while (!walker.IsEmpty()) { char c = walker.Peek(); // Skip all whitespace if (char.IsWhiteSpace(c)) { walker.Consume(1); continue; } // Get the standard rules out of the way first. bool handledByStandardRule = false; foreach (var rule in rules) { if (rule.IsStartOfToken(walker)) { handledByStandardRule = true; yield return(rule.ConsumeToken(walker)); break; } } if (handledByStandardRule) { continue; } // TODO: multi-character symbol tokens // TODO: Special cases go here // We didn't find any rule that matches what we're seeing, // so throw an error. CodePos pos = walker.Position; throw new CompileErrorException(pos, $"Unexpected character '{c}'"); } }
public bool IsStartOfToken(StringWalker walker) { char c = walker.Peek(); if (char.IsDigit(c)) { return(true); } if (c != '-') { return(false); } // It's a dash. That means this COULD be the start of a negative number. // Or...it might just be a standalone minus sign. // We need to look at the surrounding characters to disambiguate. // If the next char is not a digit, then it can't possibly be a negative // number. string group = walker.Peek(2); if (group.Length < 2) { return(false); } if (!char.IsDigit(group[1])) { return(false); } // OK, it's a dash followed by a digit. // But that's still not enough information! // The string "32-31" should be interpreted as subtracting 32 and 31, while // the string "32 -31" should be interpreted as 32 and -31. // To tell the difference, we need to look back 1 character and see if it // was whitespace. char prevChar = walker.PeekBehind(); if (char.IsWhiteSpace(prevChar)) { return(true); } return(false); }
public Token ConsumeToken(StringWalker w) { CodePos pos = w.Position; string sign = ConsumeSign(); string wholePart = ConsumeWholePart(); string decimalPart = ConsumeDecimalPart(); string content = $"{sign}{wholePart}{decimalPart}"; return(new Token(pos, TokenType.Number, content)); string ConsumeSign() => w.Peek() == '-' ? w.Consume(1) : ""; string ConsumeWholePart() => w.ConsumeWhile(char.IsDigit); string ConsumeDecimalPart() { if (w.IsEmpty() || w.Peek() != '.') { return(""); } string decimalPoint = w.Consume(1); string digits = w.ConsumeWhile(char.IsDigit); // If there is nothing after the decimal point, that's an error if (digits.Length == 0) { throw new CompileErrorException(w.Position, "There must be digits after the decimal point"); } // There can only be one decimal point in the number. If there's // another, that's an error. if (!w.IsEmpty() && w.Peek() == '.') { throw new CompileErrorException(w.Position, "A number may only have one decimal point"); } return($".{digits}"); } }
public bool IsStartOfToken(StringWalker w) { return(_allowedSymbols.Contains(w.Peek())); }
public bool IsStartOfToken(StringWalker w) => w.Peek() == '"';
public bool IsStartOfToken(StringWalker w) { return(char.IsLetter(w.Peek())); }