private Token GetToken()
{
Token token = new Token();
while (token.Type == TokenType.None)
{
switch (_reader.Current)
{
case ' ':
while (_reader.Current == ' ' && !_reader.IsEOF)
{
_reader.ConsumeChar();
}
continue;
case '\t':
break;
case '#': // comments
if (_reader.Next == '[') // block comment
{
while (_reader.Current != ']' || _reader.Next != '#' && !_reader.IsEOF)
{
_reader.ConsumeChar();
}
_reader.ConsumeChar(); // consume last #
}
else
{
_reader.ConsumeLine();
}
break;
case '\r':
break;
case '\n':
if (_reader.IsEOF)
{
return(GetToken(TokenType.EOF));
}
break;
case '$':
switch (_reader.Next)
{
case '\'':
//return new Token(TokenType.String, _line, _col, text: GetInterpolatedString('\''));
GetInterpolatedString('\'');
return(_interpolatedTokens.Dequeue());
case '"':
GetInterpolatedString('"');
return(_interpolatedTokens.Dequeue());
default:
throw new LexerException($"Unexpected token {_reader.Current} ({(int) _reader.Current})");
}
case '\'':
return(new Token(TokenType.String, _reader.Line, _reader.Col, text: _commonLexer.GetString('\'')));
case '"':
return(new Token(TokenType.String, _reader.Line, _reader.Col, text: _commonLexer.GetString('"')));
case TokenValues.Colon:
if (_reader.Next == TokenValues.Colon)
{
token = GetToken(TokenType.DoubleColon);
_reader.ConsumeChar();
}
else
{
token = GetToken(TokenType.Colon);
}
break;
case TokenValues.QuestionMark:
token = GetToken(TokenType.QuestionMark);
break;
case TokenValues.At:
token = GetToken(TokenType.At);
break;
case TokenValues.Tilde:
token = GetToken(TokenType.Tilde);
break;
case TokenValues.Semicolon:
token = GetToken(TokenType.Semicolon);
break;
case TokenValues.LeftParenthesis:
token = GetToken(TokenType.LeftParenthesis);
break;
case TokenValues.RightParenthesis:
token = GetToken(TokenType.RightParenthesis);
break;
case TokenValues.LeftBracket:
token = GetToken(TokenType.LeftBracket);
break;
case TokenValues.RightBracket:
token = GetToken(TokenType.RightBracket);
break;
case TokenValues.Not:
if (_reader.Next == '=')
{
token = GetToken(TokenType.NotEquals);
_reader.ConsumeChar();
}
else
{
token = GetToken(TokenType.Not);
}
break;
case TokenValues.EqualsAssign:
if (_reader.Next == '=')
{
token = GetToken(TokenType.EqualsEquals);
_reader.ConsumeChar();
}
else if (_reader.Next == '>')
{
token = GetToken(TokenType.LambdaArrow);
_reader.ConsumeChar();
}
else
{
token = GetToken(TokenType.Equals);
}
break;
case TokenValues.LeftBrace:
token = GetToken(TokenType.LeftBrace);
break;
case TokenValues.RightBrace:
token = GetToken(TokenType.RightBrace);
break;
case TokenValues.Comma:
token = GetToken(TokenType.Comma);
break;
case TokenValues.Plus:
if (_reader.Next == '=')
{
token = GetToken(TokenType.PlusEquals);
_reader.ConsumeChar();
}
else if (_reader.Next == TokenValues.Plus)
{
token = GetToken(TokenType.PlusPlus);
_reader.ConsumeChar();
}
else
{
token = GetToken(TokenType.Plus);
}
break;
case TokenValues.Minus:
if (_reader.Next == '=')
{
token = GetToken(TokenType.MinusEquals);
_reader.ConsumeChar();
}
else if (_reader.Next == TokenValues.GreaterThan)
{
token = GetToken(TokenType.Arrow);
_reader.ConsumeChar();
}
else if (_reader.Next == TokenValues.Minus)
{
token = GetToken(TokenType.MinusMinus);
_reader.ConsumeChar();
}
else
{
token = GetToken(TokenType.Minus);
}
break;
case TokenValues.Slash:
if (_reader.Next == '=')
{
token = GetToken(TokenType.DivideEquals);
_reader.ConsumeChar();
}
else
{
token = GetToken(TokenType.Slash);
}
break;
case TokenValues.Dot:
if (_reader.Next == TokenValues.Dot)
{
token = GetToken(TokenType.DoubleDot);
_reader.ConsumeChar();
}
else
{
token = GetToken(TokenType.Dot);
}
break;
case TokenValues.Asterisk:
if (_reader.Next == '=')
{
token = GetToken(TokenType.TimesEquals);
_reader.ConsumeChar();
}
else
{
token = GetToken(TokenType.Asterisk);
}
break;
case TokenValues.Percent:
if (_reader.Next == '=')
{
token = GetToken(TokenType.ModuloEquals);
_reader.ConsumeChar();
}
else
{
token = GetToken(TokenType.Percent);
}
break;
case '&':
if (_reader.Next == '&')
{
token = GetToken(TokenType.And);
_reader.ConsumeChar();
}
break;
case '|':
if (_reader.Next == '|')
{
token = GetToken(TokenType.Or);
_reader.ConsumeChar();
}
else
{
token = GetToken(TokenType.Pipe);
}
break;
case TokenValues.LessThan:
if (_reader.Next == '=')
{
token = GetToken(TokenType.LessThanEquals);
_reader.ConsumeChar();
}
else
{
token = GetToken(TokenType.LessThan);
}
break;
case TokenValues.GreaterThan:
if (_reader.Next == '=')
{
token = GetToken(TokenType.GreaterThanEquals);
_reader.ConsumeChar();
}
else
{
token = GetToken(TokenType.GreaterThan);
}
break;
default:
if (IsIdentifierStart(_reader.Current))
{
var identifier = _commonLexer.GetIdentifier();
if (Keywords.TryGetValue(identifier, out var kw))
{
return(new Token(kw, _reader.Line, _reader.Col));
}
else
{
return(new Token(TokenType.Identifier, _reader.Line, _reader.Col, text: identifier));
}
}
else if (IsNumberStart(_reader.Current))
{
return(GetNumberToken());
// parse number
}
throw new LexerException($"Unexpected token {_reader.Current} ({(int) _reader.Current})");
}
_reader.ConsumeChar();
}
return(token);
}
private Token NextToken()
{
var token = new Token();
while (token.Type == TokenType.None)
{
switch (_reader.Current)
{
case ' ':
while (_reader.Current == ' ' && !_reader.IsEOF)
{
_reader.ConsumeChar();
}
continue;
case '\t':
break;
case '#': // comments
if (_reader.Next == '+')
{
while (_reader.Current != '-' || _reader.Next != '#' && !_reader.IsEOF)
{
_reader.ConsumeChar();
}
_reader.ConsumeChar(); // consume last #
}
else
{
_reader.ConsumeLine();
}
break;
case '\r':
break;
case '\n':
if (_reader.IsEOF)
{
return(GetToken(TokenType.EOF));
}
break;
case ';':
token = GetToken(TokenType.Semicolon);
break;
case '\'':
return(new Token(TokenType.String, _reader.Line, _reader.Col, text: _commonLexer.GetString('\'')));
case '"':
return(new Token(TokenType.String, _reader.Line, _reader.Col, text: _commonLexer.GetString('"')));
default:
if (IsIdentifierStart(_reader.Current))
{
var identifier = _commonLexer.GetIdentifier();
if (identifier == ImportKeyword)
{
return(new Token(TokenType.Import, _reader.Line, _reader.Col));
}
return(new Token(TokenType.Identifier, _reader.Line, _reader.Col, text: identifier));
}
_reader.ConsumeChar();
return(new Token());
}
_reader.ConsumeChar();
}
return(token);
}
