private static Token ParseStringLiteral(CharacterReader reader)
{
reader.ReadCharacter();
var position = reader.CurrentPosition;
var buffer = new StringBuilder();
var literalBuffer = new StringBuilder();
literalBuffer.Append('"');
while (true)
{
var character = reader.PeekCharacter().Value;
if (Char.IsLetterOrDigit(character) || character == '\\')
{
literalBuffer.Append(character);
if (Char.IsLetterOrDigit(character))
{
buffer.Append(reader.ReadCharacter());
}
else if (character == '\\')
{
reader.ReadCharacter();
character = reader.ReadCharacter().Value;
buffer.Append(character);
literalBuffer.Append(character);
}
}
else
{
return(new Token(TokenType.StringLiteral, buffer.ToString(), literalBuffer.ToString(), position));
}
}
}
private static Token ParseKeywordOrIdentifier(CharacterReader reader)
{
var buffer = new StringBuilder();
TokenPosition position = null;
while (true)
{
var character = reader.PeekCharacter().Value;
if (IsIdentifierCharacter(character))
{
buffer.Append(character);
reader.ReadCharacter();
if (position == null)
{
position = reader.CurrentPosition;
}
}
else
{
var value = buffer.ToString().ToLower();
if (value == "maak" || value == "naam")
{
return(new Token(TokenType.Make, value, position));
}
if (value == "lokaal")
{
return(new Token(TokenType.Local, value, position));
}
if (value == "welwaar")
{
return(new Token(TokenType.True, value, position));
}
if (value == "nietwaar")
{
return(new Token(TokenType.False, value, position));
}
if (value == "leer")
{
return(new Token(TokenType.Learn, value, position));
}
if (value == "eind")
{
return(new Token(TokenType.End, value, position));
}
if (value == "uitvoer")
{
return(new Token(TokenType.Return, value, position));
}
if (value == "stop")
{
return(new Token(TokenType.Stop, value, position));
}
return(new Token(TokenType.Identifier, value, position));
}
}
}
public static List <Token> Tokenize(string input)
{
var tokens = new List <Token>();
var reader = new CharacterReader(input + "\n");
var buffer = new StringBuilder();
while (!reader.AtEnd)
{
char character = reader.PeekCharacter().Value;
if (character == '~')
{
reader.ReadCharacter();
}
else if (character == '+')
{
tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.Plus));
}
else if (character == '-')
{
tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.Minus));
}
else if (character == '*')
{
tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.Multiply));
}
else if (character == '/')
{
tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.Divide));
}
else if (character == '=')
{
tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.Equals));
}
else if (character == '<')
{
reader.ReadCharacter();
var position = reader.CurrentPosition;
var nextCharacter = reader.PeekCharacter().Value;
if (nextCharacter == '=')
{
reader.ReadCharacter();
tokens.Add(new Token(TokenType.SmallerOrEqualThan, "<=", position));
}
else if (nextCharacter == '>')
{
reader.ReadCharacter();
tokens.Add(new Token(TokenType.NotEqual, "<>", position));
}
else
{
tokens.Add(new Token(TokenType.SmallerThan, "<", position));
}
}
else if (character == '>')
{
reader.ReadCharacter();
var position = reader.CurrentPosition;
var nextCharacter = reader.PeekCharacter().Value;
if (nextCharacter == '=')
{
reader.ReadCharacter();
tokens.Add(new Token(TokenType.GreaterOrEqualThan, ">=", position));
}
else
{
tokens.Add(new Token(TokenType.GreaterThan, ">", position));
}
}
else if (character == ';')
{
while (reader.PeekCharacter() != '\n')
{
reader.ReadCharacter();
}
}
else if (character == '(')
{
tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.ParenthesisLeft));
}
else if (character == ')')
{
tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.ParenthesisRight));
}
else if (character == '[')
{
tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.BracketLeft));
}
else if (character == ']')
{
tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.BracketRight));
}
else if (character == '"')
{
tokens.Add(ParseStringLiteral(reader));
}
else if (character == ':')
{
tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.Colon));
}
else if (character == ',')
{
tokens.Add(ReadCharacterAndCreateToken(reader, TokenType.Comma));
}
else if (Char.IsLetter(character))
{
tokens.Add(ParseKeywordOrIdentifier(reader));
}
else if (Char.IsNumber(character))
{
tokens.Add(ParseNumber(reader));
}
else if (Char.IsWhiteSpace(character) || character == '\n' || character == '\r')
{
// Skip
reader.ReadCharacter();
}
else
{
throw new Exception($"Unexpected character: {character}");
}
}
tokens.Add(new Token(TokenType.ProgramEnd, "", reader.CurrentPosition));
return(tokens);
}
