private static Token ParseStringLiteral(CharacterReader reader) { reader.ReadCharacter(); var position = reader.CurrentPosition; var buffer = new StringBuilder(); var literalBuffer = new StringBuilder(); literalBuffer.Append('"'); while (true) { var character = reader.PeekCharacter().Value; if (Char.IsLetterOrDigit(character) || character == '\\') { literalBuffer.Append(character); if (Char.IsLetterOrDigit(character)) { buffer.Append(reader.ReadCharacter()); } else if (character == '\\') { reader.ReadCharacter(); character = reader.ReadCharacter().Value; buffer.Append(character); literalBuffer.Append(character); } } else { return(new Token(TokenType.StringLiteral, buffer.ToString(), literalBuffer.ToString(), position)); } } }
private static Token ParseKeywordOrIdentifier(CharacterReader reader) { var buffer = new StringBuilder(); TokenPosition position = null; while (true) { var character = reader.PeekCharacter().Value; if (IsIdentifierCharacter(character)) { buffer.Append(character); reader.ReadCharacter(); if (position == null) { position = reader.CurrentPosition; } } else { var value = buffer.ToString().ToLower(); if (value == "maak" || value == "naam") { return(new Token(TokenType.Make, value, position)); } if (value == "lokaal") { return(new Token(TokenType.Local, value, position)); } if (value == "welwaar") { return(new Token(TokenType.True, value, position)); } if (value == "nietwaar") { return(new Token(TokenType.False, value, position)); } if (value == "leer") { return(new Token(TokenType.Learn, value, position)); } if (value == "eind") { return(new Token(TokenType.End, value, position)); } if (value == "uitvoer") { return(new Token(TokenType.Return, value, position)); } if (value == "stop") { return(new Token(TokenType.Stop, value, position)); } return(new Token(TokenType.Identifier, value, position)); } } }
public static List <Token> Tokenize(string input) { var tokens = new List <Token>(); var reader = new CharacterReader(input + "\n"); var buffer = new StringBuilder(); while (!reader.AtEnd) { char character = reader.PeekCharacter().Value; if (character == '~') { reader.ReadCharacter(); } else if (character == '+') { tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.Plus)); } else if (character == '-') { tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.Minus)); } else if (character == '*') { tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.Multiply)); } else if (character == '/') { tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.Divide)); } else if (character == '=') { tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.Equals)); } else if (character == '<') { reader.ReadCharacter(); var position = reader.CurrentPosition; var nextCharacter = reader.PeekCharacter().Value; if (nextCharacter == '=') { reader.ReadCharacter(); tokens.Add(new Token(TokenType.SmallerOrEqualThan, "<=", position)); } else if (nextCharacter == '>') { reader.ReadCharacter(); tokens.Add(new Token(TokenType.NotEqual, "<>", position)); } else { tokens.Add(new Token(TokenType.SmallerThan, "<", position)); } } else if (character == '>') { reader.ReadCharacter(); var position = reader.CurrentPosition; var nextCharacter = reader.PeekCharacter().Value; if (nextCharacter == '=') { reader.ReadCharacter(); tokens.Add(new Token(TokenType.GreaterOrEqualThan, ">=", position)); } else { tokens.Add(new Token(TokenType.GreaterThan, ">", position)); } } else if (character == ';') { while (reader.PeekCharacter() != '\n') { reader.ReadCharacter(); } } else if (character == '(') { tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.ParenthesisLeft)); } else if (character == ')') { tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.ParenthesisRight)); } else if (character == '[') { tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.BracketLeft)); } else if (character == ']') { tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.BracketRight)); } else if (character == '"') { tokens.Add(ParseStringLiteral(reader)); } else if (character == ':') { tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.Colon)); } else if (character == ',') { tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.Comma)); } else if (Char.IsLetter(character)) { tokens.Add(ParseKeywordOrIdentifier(reader)); } else if (Char.IsNumber(character)) { tokens.Add(ParseNumber(reader)); } else if (Char.IsWhiteSpace(character) || character == '\n' || character == '\r') { // Skip reader.ReadCharacter(); } else { throw new Exception($"Unexpected character: {character}"); } } tokens.Add(new Token(TokenType.ProgramEnd, "", reader.CurrentPosition)); return(tokens); }