public SyntaxToken Lex()
{
_start = _position;
_kind = SyntaxKind.UnknownToken;
_value = null;
var text = _text;
switch (Current)
{
case '\0':
_kind = SyntaxKind.EndOfFileToken;
break;
case ';':
_position++;
_kind = SyntaxKind.SemicolonToken;
break;
case ',':
_position++;
_kind = SyntaxKind.CommaToken;
break;
case '<':
_position++;
if (Current == '=')
{
_position++;
_kind = SyntaxKind.LessThanEqualsToken;
}
else
{
_kind = SyntaxKind.LessThanToken;
}
break;
case '>':
_position++;
if (Current == '=')
{
_position++;
_kind = SyntaxKind.GreaterThanEqualsToken;
}
else
{
_kind = SyntaxKind.GreaterThanToken;
}
break;
case '=':
_position++;
if (Current == '=')
{
_position++;
_kind = SyntaxKind.EqualsEqualsToken;
}
else
{
_kind = SyntaxKind.EqualsToken;
}
break;
case '!':
_position++;
if (Current == '=')
{
_position++;
_kind = SyntaxKind.BangEqualsToken;
}
else
{
_kind = SyntaxKind.BangToken;
}
break;
case '|':
_position++;
if (Current == '|')
{
_position++;
_kind = SyntaxKind.PipePipeToken;
}
else
{
_kind = SyntaxKind.PipeToken;
}
break;
case '&':
_position++;
if (Current == '&')
{
_position++;
_kind = SyntaxKind.AmpersandAmpersandToken;
}
else
{
_kind = SyntaxKind.AmpersandToken;
}
break;
case '+':
_position++;
if (Current == '+')
{
_position++;
_kind = SyntaxKind.PlusPlusToken;
}
else
{
_kind = SyntaxKind.PlusToken;
}
break;
case '-':
_position++;
if (Current == '-')
{
_position++;
_kind = SyntaxKind.MinusMinusToken;
}
else
{
_kind = SyntaxKind.MinusToken;
}
break;
case '*':
_position++;
_kind = SyntaxKind.StarToken;
break;
case '/':
_position++;
_kind = SyntaxKind.SlashToken;
break;
case ':':
_position++;
_kind = SyntaxKind.ColonToken;
break;
case '?':
_position++;
_kind = SyntaxKind.QuestionMarkToken;
break;
case '(':
_position++;
_kind = SyntaxKind.OpenParenthesisToken;
break;
case ')':
_position++;
_kind = SyntaxKind.CloseParenthesisToken;
break;
case '{':
_position++;
_kind = SyntaxKind.OpenBraceToken;
break;
case '}':
_position++;
_kind = SyntaxKind.CloseBraceToken;
break;
case '%':
_position++;
_kind = SyntaxKind.PercentToken;
break;
case '~':
_position++;
_kind = SyntaxKind.TildeToken;
break;
case '^':
_position++;
_kind = SyntaxKind.HatToken;
break;
default:
var currentChar = Current;
if (Char.IsWhiteSpace(currentChar))
{
ReadWhiteSpace();
}
else if (Char.IsLetter(currentChar))
{
ReadKeywordOrIdentifier();
}
else if (Char.IsDigit(currentChar))
{
ReadNumber();
}
else if (currentChar == '"')
{
ReadString();
}
else
{
_diagnostics.ReportBadCharacter(_start, currentChar);
_position++;
_kind = SyntaxKind.UnknownToken;
}
break;
}
var tokenText = SyntaxFacts.GetText(_kind);
if (tokenText == null)
{
tokenText = text.ToString(_start, _position - _start);
}
return(new SyntaxToken(_kind, _start, tokenText, _value));
}
public SyntaxToken Lex()
{
_start = _position;
_kind = SyntaxKind.BadToken;
_value = null;
switch (Current)
{
case '\0':
_kind = SyntaxKind.EOFToken;
break;
case '+':
_kind = SyntaxKind.PlusToken;
_position += 1;
break;
case '-':
_kind = SyntaxKind.MinusToken;
_position += 1;
break;
case '*':
_kind = SyntaxKind.StarToken;
_position += 1;
break;
case '/':
_kind = SyntaxKind.SlashToken;
_position += 1;
break;
case ',':
_kind = SyntaxKind.CommaToken;
_position += 1;
break;
case '(':
_kind = SyntaxKind.OpenParenthesisToken;
_position += 1;
break;
case ')':
_kind = SyntaxKind.CloseParenthesisToken;
_position += 1;
break;
case '{':
_kind = SyntaxKind.OpenBraceToken;
_position += 1;
break;
case '}':
_kind = SyntaxKind.CloseBraceToken;
_position += 1;
break;
case '~':
_kind = SyntaxKind.TildeToken;
_position += 1;
break;
case '^':
_kind = SyntaxKind.HatToken;
_position += 1;
break;
case ':':
_kind = SyntaxKind.ColonToken;
_position += 1;
break;
case '&':
{
if (Lookahead == '&')
{
_position += 2;
_kind = SyntaxKind.AmpersandAmpersandToken;
}
else
{
_position += 1;
_kind = SyntaxKind.AmpersandToken;
}
break;
}
case '|':
{
if (Lookahead == '|')
{
_position += 2;
_kind = SyntaxKind.PipePipeToken;
}
else
{
_position += 1;
_kind = SyntaxKind.PipeToken;
}
break;
}
case '=':
{
if (Lookahead == '=')
{
_position += 2;
_kind = SyntaxKind.EqualsEqualsToken;
}
else
{
_position += 1;
_kind = SyntaxKind.EqualsToken;
}
}
break;
case '!':
{
if (Lookahead == '=')
{
_position += 2;
_kind = SyntaxKind.BangEqualsToken;
}
else
{
_position += 1;
_kind = SyntaxKind.BangToken;
}
}
break;
case '<':
{
if (Lookahead == '=')
{
_position += 2;
_kind = SyntaxKind.LessEqualsToken;
}
else
{
_position += 1;
_kind = SyntaxKind.LessToken;
}
}
break;
case '>':
{
if (Lookahead == '=')
{
_position += 2;
_kind = SyntaxKind.GreaterEqualsToken;
}
else
{
_position += 1;
_kind = SyntaxKind.GreaterToken;
}
}
break;
case '"':
ReadString();
break;
default:
if (char.IsDigit(this.Current))
{
ReadNumberToken();
}
else if (char.IsWhiteSpace(Current))
{
ReadWhiteSpace();
}
else if (char.IsLetter(Current))
{
ReadIdentifierOrKeyword();
}
else
{
_diagnostics.ReportBadCharacter(_position, Current);
_position += 1;
_kind = SyntaxKind.BadToken;
}
break;
}
var length = _position - _start;
var text = SyntaxFacts.GetText(_kind);
if (text == null)
{
text = _text.ToString(_start, length);
}
return(new SyntaxToken(_kind, _start, text, _value));
}
public SyntaxToken Lex()
{
// numbers
// + - * / ()
//whitespace
if (_position >= _text.Length)
{
return(new SyntaxToken(SyntaxKind.EndOfFileToken, _position, "\0", null));
}
var start = _position;
//keep reading numbers, create word that represents the number
if (char.IsDigit(Current))
{
while (char.IsDigit(Current))
{
Next();
}
var length = _position - start;
var text = _text.Substring(start, length);
if (!int.TryParse(text, out var value))
{
_diagnostics.ReportInvalidNumber(new TextSpan(start, length), _text, typeof(int));
}
return(new SyntaxToken(SyntaxKind.NumberToken, start, text, value));
}
if (char.IsWhiteSpace(Current))
{
while (char.IsWhiteSpace(Current))
{
Next();
}
var length = _position - start;
var text = _text.Substring(start, length);
return(new SyntaxToken(SyntaxKind.WhitespaceToken, start, text, null));
}
if (char.IsLetter(Current))
{
while (char.IsLetter(Current))
{
Next();
}
var length = _position - start;
var text = _text.Substring(start, length);
var kind = SyntaxFacts.GetKeyWordKind(text);
return(new SyntaxToken(kind, start, text, null));
}
switch (Current)
{
case '+':
return(new SyntaxToken(SyntaxKind.PlusToken, _position++, "+", null));
case '-':
return(new SyntaxToken(SyntaxKind.MinusToken, _position++, "-", null));
case '*':
return(new SyntaxToken(SyntaxKind.StarToken, _position++, "*", null));
case '/':
return(new SyntaxToken(SyntaxKind.SlashToken, _position++, "/", null));
case '(':
return(new SyntaxToken(SyntaxKind.OpenParenthesisToken, _position++, "(", null));
case ')':
return(new SyntaxToken(SyntaxKind.CloseParenthesisToken, _position++, ")", null));
case '&':
if (Lookahead == '&')
{
_position += 2;
return(new SyntaxToken(SyntaxKind.AmpersandAmpersandToken, start, "&&", null));
}
break;
case '|':
if (Lookahead == '|')
{
_position += 2;
return(new SyntaxToken(SyntaxKind.PipePipeToken, start, "||", null));
}
break;
case '=':
if (Lookahead == '=')
{
_position += 2;
return(new SyntaxToken(SyntaxKind.EqualsEqualsToken, start, "==", null));
}
else
{
_position += 1;
return(new SyntaxToken(SyntaxKind.EqualsToken, start, "=", null));
}
break;
case '!':
if (Lookahead == '=')
{
_position += 2;
return(new SyntaxToken(SyntaxKind.BangEqualsToken, start, "!=", null));
}
else
{
_position += 1;
return(new SyntaxToken(SyntaxKind.BangToken, start, "!", null));
}
}
_diagnostics.ReportBadCharacter(_position, Current);
return(new SyntaxToken(SyntaxKind.BadToken, _position++, _text.Substring(_position - 1, 1), null));
}
//Break input into corresponding lexemes
public SyntaxToken Lex()
{
//End of File
if (_position >= _text.Length)
{
return(new SyntaxToken(TokenType.EndOfFile, _position, "\0", null));
}
var start = _position;
//Integer
if (char.IsDigit(Current))
{
while (char.IsDigit(Current))
{
Next();
}
var length = _position - start;
var text = _text.Substring(start, length);
if (!int.TryParse(text, out var value))
{
_diagnostics.ReportInvalidNumber(new TextSpan(start, length), _text, typeof(int));
}
return(new SyntaxToken(TokenType.Number, start, text, value));
}
//Whitespace
if (char.IsWhiteSpace(Current))
{
while (char.IsWhiteSpace(Current))
{
Next();
}
var length = _position - start;
var text = _text.Substring(start, length);
return(new SyntaxToken(TokenType.Whitespace, start, text, null));
}
//True-False and catch-all identifiers
if (char.IsLetter(Current))
{
while (char.IsLetter(Current))
{
Next();
}
var length = _position - start;
var text = _text.Substring(start, length);
var kind = ParserRules.GetKeywordKind(text);
return(new SyntaxToken(kind, start, text, null));
}
//TODO refactor this for readability & scalability
//Operators - Arithmetic & Binary
switch (Current)
{
//Arithmetic Operators
case '+':
return(new SyntaxToken(TokenType.Plus, _position++, "+", null));
case '-':
return(new SyntaxToken(TokenType.Minus, _position++, "-", null));
case '*':
return(new SyntaxToken(TokenType.Star, _position++, "*", null));
case '/':
return(new SyntaxToken(TokenType.Slash, _position++, "/", null));
case '\\':
return(new SyntaxToken(TokenType.ReverseSlash, _position++, "/", null));
case '%':
return(new SyntaxToken(TokenType.Modulo, _position++, "%", null));
case '^':
return(new SyntaxToken(TokenType.BitwiseXor, _position++, "^", null));
case '#':
return(new SyntaxToken(TokenType.Power, _position++, "^", null));
//Boolean Operators
case '!':
{
switch (Ahead)
{
case '=':
_position += 2;
return(new SyntaxToken(TokenType.Negation, start, "!=", null));
default:
return(new SyntaxToken(TokenType.Negation, start, "!", null));
}
}
case '&':
{
switch (Ahead)
{
case '&':
_position += 2;
return(new SyntaxToken(TokenType.LogicalAnd, start, "&&", null));
default:
return(new SyntaxToken(TokenType.BitwiseAnd, _position++, "&", null));
}
}
case '|':
{
switch (Ahead)
{
case '|':
_position += 2;
return(new SyntaxToken(TokenType.LogicalOr, start, "||", null));
default:
return(new SyntaxToken(TokenType.BitwiseOr, _position++, "&", null));
}
}
case '=':
{
switch (Ahead)
{
//We haven't implemented the assignment operator
case '=':
_position += 2;
return(new SyntaxToken(TokenType.Equality, start, "==", null));
default:
return(new SyntaxToken(TokenType.Assign, _position++, "=", null));
}
}
case '>':
{
switch (Ahead)
{
case '=':
_position += 2;
return(new SyntaxToken(TokenType.GreaterThanEquals, start, ">=", null));
case '>':
_position += 2;
return(new SyntaxToken(TokenType.RightShift, start, ">>", null));
default:
return(new SyntaxToken(TokenType.GreaterThan, _position++, ">", null));
}
}
case '<':
{
switch (Ahead)
{
case '=':
_position += 2;
return(new SyntaxToken(TokenType.LessThanEquals, start, "<=", null));
case '<':
_position += 2;
return(new SyntaxToken(TokenType.LeftShift, start, "<<", null));
default:
return(new SyntaxToken(TokenType.LessThan, _position++, "<", null));
}
}
case '~':
return(new SyntaxToken(TokenType.BitwiseNegation, _position++, "~", null));
case '(':
return(new SyntaxToken(TokenType.OpenParenthesis, _position++, "(", null));
case ')':
return(new SyntaxToken(TokenType.CloseParenthesis, _position++, ")", null));
}
_diagnostics.ReportBadCharacter(_position, Current);
return(new SyntaxToken(TokenType.Bad, _position++, _text.Substring(_position - 1, 1), null));
}
public SyntaxToken Lex()
{
_start = _position;
_kind = SyntaxKind.BadToken;
_value = null;
switch (Current)
{
case '\0':
_kind = SyntaxKind.EndOfFileToken;
break;
case '+':
_kind = SyntaxKind.PlusToken;
_position++;
break;
case '-':
_kind = SyntaxKind.MinusToken;
_position++;
break;
case '*':
_kind = SyntaxKind.StarToken;
_position++;
break;
case '/':
_kind = SyntaxKind.SlashToken;
_position++;
break;
case '(':
_kind = SyntaxKind.OpenParenthesisToken;
_position++;
break;
case ')':
_kind = SyntaxKind.CloseParenthesisToken;
_position++;
break;
case '{':
_kind = SyntaxKind.OpenBraceToken;
_position++;
break;
case '}':
_kind = SyntaxKind.CloseBraceToken;
_position++;
break;
case '~':
_kind = SyntaxKind.TildeToken;
_position++;
break;
case '^':
_kind = SyntaxKind.HatToken;
_position++;
break;
case '&':
_position++;
if (Current != '&')
{
_kind = SyntaxKind.AmpersandToken;
}
else
{
_kind = SyntaxKind.AmpersandAmpersandToken;
_position++;
}
break;
case '|':
_position++;
if (Current != '|')
{
_kind = SyntaxKind.PipeToken;
}
else
{
_kind = SyntaxKind.PipePipeToken;
_position++;
}
break;
case '=':
_position++;
if (Current != '=')
{
_kind = SyntaxKind.EqualsToken;
}
else
{
_kind = SyntaxKind.EqualsEqualsToken;
_position++;
}
break;
case '!':
_position++;
if (Current != '=')
{
_kind = SyntaxKind.BangToken;
}
else
{
_kind = SyntaxKind.BangEqualsToken;
_position++;
}
break;
case '<':
_position++;
if (Current != '=')
{
_kind = SyntaxKind.LessToken;
}
else
{
_kind = SyntaxKind.LessOrEqualsToken;
_position++;
}
break;
case '>':
_position++;
if (Current != '=')
{
_kind = SyntaxKind.GreaterToken;
}
else
{
_kind = SyntaxKind.GreaterOrEqualsToken;
_position++;
}
break;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
ReadNumber();
break;
case ' ':
case '\t':
case '\n':
case '\r':
ReadWhiteSpace();
break;
default:
if (char.IsLetter(Current))
{
ReadIdentifierOrKeyword();
}
else if (char.IsWhiteSpace(Current))
{
ReadWhiteSpace();
}
else
{
_diagnostics.ReportBadCharacter(_position, Current);
_position++;
}
break;
}
var length = _position - _start;
var text = SyntaxFacts.GetText(_kind);
if (text == null)
{
text = _text.ToString(_start, length);
}
return(new SyntaxToken(_kind, _start, text, _value));
}
public SyntaxToken Lex()
{
_start = _position;
_kind = SyntaxKind.BadToken;
_value = null;
// <operator> + - / ( )
switch (Current)
{
case '+':
{
_kind = SyntaxKind.PlusToken;
_position++;
break;
}
case '-':
{
_kind = SyntaxKind.MinusToken;
_position++;
break;
}
case '*':
{
_kind = SyntaxKind.StarToken;
_position++;
break;
}
case '/':
{
_kind = SyntaxKind.ForwardSlashToken;
_position++;
break;
}
case '(':
{
_kind = SyntaxKind.OpenParenToken;
_position++;
break;
}
case ')':
{
_kind = SyntaxKind.CloseParenToken;
_position++;
break;
}
case '{':
{
_kind = SyntaxKind.OpenBraceToken;
_position++;
break;
}
case '}':
{
_kind = SyntaxKind.CloseBraceToken;
_position++;
break;
}
case '&':
{
if (Lookahead == '&')
{
_kind = SyntaxKind.AmpersandAmpersandToken;
_position += 2;
}
break;
}
case '|':
{
if (Lookahead == '|')
{
_kind = SyntaxKind.PipePipeToken;
_position += 2;
}
break;
}
case '=':
{
_position++;
if (Current != '=')
{
_kind = SyntaxKind.EqualToken;
}
else
{
_kind = SyntaxKind.EqualEqualToken;
_position++;
}
break;
}
case '!':
{
_position++;
if (Current != '=')
{
_kind = SyntaxKind.BangToken;
}
else
{
_kind = SyntaxKind.BangEqualToken;
_position++;
}
break;
}
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
{
ReadNumberToken();
break;
}
case ' ':
case '\t':
case '\n':
case '\r':
{
ReadWhiteSpaceToken();
break;
}
case '\0':
{
_kind = SyntaxKind.EOFToken;
break;
}
default:
{
if (char.IsLetter(Current))
{
ReadIdentifierOrKeyword();
}
// <whitespace>
else if (char.IsWhiteSpace(Current))
{
ReadWhiteSpaceToken();
}
else
{
_diagnostics.ReportBadCharacter(_position, Current);
_position++;
}
break;
}
}
var length = _position - _start;
var text = SyntaxRules.GetText(_kind);
if (text == null)
{
text = _text.ToString(_start, length);
}
return(new SyntaxToken(_kind, _start, text, _value));
}
public SyntaxToken Lex()
{
if (_position >= _text.Length)
{
return(new SyntaxToken(SyntaxKind.EndOfFileToken, _position, "\0", null));
}
if (char.IsDigit(Current))
{
var start = _position;
var isHex = false;
while (char.IsDigit(Current))
{
Next();
}
if (char.IsUpper(Current))
{
isHex = true;
while (char.IsUpper(Current))
{
Next();
}
}
var length = _position - start;
var text = _text.Substring(start, length);
if (isHex == false)
{
if (!int.TryParse(text, out var value))
{
_diagnostics.ReportInvalidNumber(text, typeof(int));
}
}
return(isHex ||
text.Contains("8") ||
text.Contains("9")?
new SyntaxToken(SyntaxKind.HexToken, start, text, text)
:
new SyntaxToken(SyntaxKind.OctToken, start, text, text));
}
if (char.IsLetter(Current))
{
var start = _position;
if (char.IsUpper(Current))
{
while (char.IsUpper(Current))
{
Next();
}
if (char.IsDigit(Current))
{
while (char.IsDigit(Current))
{
Next();
}
}
var length = _position - start;
var text = _text.Substring(start, length);
// var kind = SyntaxFacts.GetKeywordKind(text);
return(new SyntaxToken(SyntaxKind.HexToken, start, text, text));
}
}
if (char.IsWhiteSpace(Current))
{
var start = _position;
while (char.IsWhiteSpace(Current))
{
Next();
}
var length = _position - start;
var text = _text.Substring(start, length);
return(new SyntaxToken(SyntaxKind.WhiteSpaceToken, start, text, null));
}
switch (Current)
{
case '+':
return(new SyntaxToken(SyntaxKind.PlusToken, _position++, "+", null));
case '-':
return(new SyntaxToken(SyntaxKind.MinusToken, _position++, "-", null));
case '*':
return(new SyntaxToken(SyntaxKind.StarToken, _position++, "+", null));
// case '/':
// return new SyntaxToken(SyntaxKind.SlashToken, _position++, "/", null);
// case '(':
// return new SyntaxToken(SyntaxKind.OpenParenthesisToken, _position++, "(", null);
// case ')':
// return new SyntaxToken(SyntaxKind.CloseParenthesisToken, _position++, ")", null);
// case '!':
// if(LookAhead == '=') return new SyntaxToken(SyntaxKind.BangEqualsToken, _position +=2, "!=", null);
// else return new SyntaxToken(SyntaxKind.BangToken, _position++, "!", null);
// case '&':
// if (LookAhead == '&') return new SyntaxToken(SyntaxKind.AmpersandAmpersandToken, _position +=2, "&&", null);
// break;
// case '|':
// if (LookAhead == '|') return new SyntaxToken(SyntaxKind.PipePipeToken, _position +=2, "||", null);
// break;
// case '=':
// if (LookAhead == '=') return new SyntaxToken(SyntaxKind.EqualsEqualsToken, _position +=2, "==", null);
// break;
}
_diagnostics.ReportBadCharacter(_position, Current);
return(new SyntaxToken(SyntaxKind.BadToken, _position++, _text.Substring(_position - 1, 1), null));
}
public Token NextToken()
{
start = position;
kind = NodeKind.BadToken;
value = null;
switch (Current)
{
case '\0':
kind = NodeKind.EndOfFileToken;
break;
case '+':
kind = NodeKind.PlusToken;
position++;
break;
case '-':
kind = NodeKind.MinusToken;
position++;
break;
case '*':
kind = NodeKind.StarToken;
position++;
break;
case '/':
kind = NodeKind.SlashToken;
position++;
break;
case '(':
kind = NodeKind.OpenParenthesisToken;
position++;
break;
case ')':
kind = NodeKind.CloseParenthesisToken;
position++;
break;
case '&':
if (Lookahead == '&')
{
kind = NodeKind.AmpersandAmpersandToken;
position += 2;
}
break;
case '|':
if (Lookahead == '|')
{
kind = NodeKind.PipePipeToken;
position += 2;
}
break;
case '=':
position++;
if (Current != '=')
{
kind = NodeKind.EqualsToken;
}
else
{
kind = NodeKind.EqualsEqualsToken;
position++;
}
break;
case '!':
position++;
if (Current != '=')
{
kind = NodeKind.BangToken;
}
else
{
kind = NodeKind.BangEqualsToken;
position++;
}
break;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
ReadNumber();
break;
case ' ':
case '\t':
case '\n':
case '\r':
ReadWhiteSpace();
break;
default:
if (char.IsLetter(Current))
{
ReadIdentifierOrKeyword();
}
else if (char.IsWhiteSpace(Current))
{
ReadWhiteSpace();
}
else
{
diagnostics.ReportBadCharacter(position, Current);
position++;
}
break;
}
int length = position - start;
string txt = Facts.GetText(kind);
if (txt == null)
{
txt = text.ToString(start, length);
}
return(new Token(kind, start, txt, value));
}
public SyntaxToken Lex()
{
// numbers
// + - / * ( )
//<whitespaces>
_start = _position;
_kind = SyntaxKind.BadToken;
_value = null;
switch (Current)
{
case '\0':
_kind = SyntaxKind.EndOfFileToken;
break;
case '+':
_position++;
_kind = SyntaxKind.PlusToken;
break;
case '-':
_position++;
_kind = SyntaxKind.MinusToken;
break;
case '*':
_position++;
_kind = SyntaxKind.StarToken;
break;
case '/':
_position++;
_kind = SyntaxKind.SlashToken;
break;
case '(':
_position++;
_kind = SyntaxKind.OpenParenthesisToken;
break;
case ')':
_position++;
_kind = SyntaxKind.CloseParanthesisToken;
break;
case '&':
if (LookAhead == '&')
{
_position += 2;
_kind = SyntaxKind.AmpersandAmpersandToken;
break;
}
break;
case '|':
if (LookAhead == '|')
{
_position += 2;
_kind = SyntaxKind.PipePipeToken;
break;
}
break;
case '=':
if (LookAhead == '=')
{
_position += 2;
_kind = SyntaxKind.EqualsEqualsToken;
}
else
{
_position++;
_kind = SyntaxKind.EqualsToken;
}
break;
case '!':
if (LookAhead == '=')
{
_position += 2;
_kind = SyntaxKind.BangEqualsToken;
}
else
{
_position++;
_kind = SyntaxKind.BangToken;
}
break;
default:
if (char.IsDigit(Current))
{
ReadNumberToken();
}
else if (char.IsWhiteSpace(Current))
{
ReadWhiteSpace();
}
else if (char.IsLetter(Current))
{
ReadIdentifierOrKeyword();
}
else
{
_diagnostics.ReportBadCharacter(_position, Current);
_position++;
}
break;
}
var length = _position - _start;
var text = SyntaxFacts.GetText(_kind);
if (text == null)
{
text = _text.ToString(_start, length);
}
return(new SyntaxToken(_kind, _start, text, _value));
}
private void ReadToken()
{
_start = _position;
_kind = SyntaxKind.BadToken;
_value = null;
switch (Current)
{
case '\0':
_kind = SyntaxKind.EndOfFileToken;
break;
case '+':
_position++;
if (Current != '=')
{
_kind = SyntaxKind.PlusToken;
}
else
{
_kind = SyntaxKind.PlusEqualsToken;
_position++;
}
break;
case '-':
_position++;
if (Current != '=')
{
_kind = SyntaxKind.MinusToken;
}
else
{
_kind = SyntaxKind.MinusEqualsToken;
_position++;
}
break;
case '*':
_position++;
if (Current != '=')
{
_kind = SyntaxKind.StarToken;
}
else
{
_kind = SyntaxKind.StarEqualsToken;
_position++;
}
break;
case '/':
_position++;
if (Current != '=')
{
_kind = SyntaxKind.SlashToken;
}
else
{
_kind = SyntaxKind.SlashEqualsToken;
_position++;
}
break;
case '(':
_kind = SyntaxKind.OpenParenthesisToken;
_position++;
break;
case ')':
_kind = SyntaxKind.CloseParenthesisToken;
_position++;
break;
case '{':
_kind = SyntaxKind.OpenBraceToken;
_position++;
break;
case '}':
_kind = SyntaxKind.CloseBraceToken;
_position++;
break;
case ':':
_kind = SyntaxKind.ColonToken;
_position++;
break;
case ',':
_kind = SyntaxKind.CommaToken;
_position++;
break;
case '~':
_kind = SyntaxKind.TildeToken;
_position++;
break;
case '^':
_position++;
if (Current != '=')
{
_kind = SyntaxKind.HatToken;
}
else
{
_kind = SyntaxKind.HatEqualsToken;
_position++;
}
break;
case '&':
_position++;
if (Current == '&')
{
_kind = SyntaxKind.AmpersandAmpersandToken;
_position++;
}
else if (Current == '=')
{
_kind = SyntaxKind.AmpersandEqualsToken;
_position++;
}
else
{
_kind = SyntaxKind.AmpersandToken;
}
break;
case '|':
_position++;
if (Current == '|')
{
_kind = SyntaxKind.PipePipeToken;
_position++;
}
else if (Current == '=')
{
_kind = SyntaxKind.PipeEqualsToken;
_position++;
}
else
{
_kind = SyntaxKind.PipeToken;
}
break;
case '=':
_position++;
if (Current != '=')
{
_kind = SyntaxKind.EqualsToken;
}
else
{
_kind = SyntaxKind.EqualsEqualsToken;
_position++;
}
break;
case '!':
_position++;
if (Current != '=')
{
_kind = SyntaxKind.BangToken;
}
else
{
_kind = SyntaxKind.BangEqualsToken;
_position++;
}
break;
case '<':
_position++;
if (Current != '=')
{
_kind = SyntaxKind.LessToken;
}
else
{
_kind = SyntaxKind.LessOrEqualsToken;
_position++;
}
break;
case '>':
_position++;
if (Current != '=')
{
_kind = SyntaxKind.GreaterToken;
}
else
{
_kind = SyntaxKind.GreaterOrEqualsToken;
_position++;
}
break;
case '"':
ReadString();
break;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
ReadNumber();
break;
case '_':
ReadIdentifierOrKeyword();
break;
default:
if (char.IsLetter(Current))
{
ReadIdentifierOrKeyword();
}
else
{
var span = new TextSpan(_position, 1);
var location = new TextLocation(_text, span);
_diagnostics.ReportBadCharacter(location, Current);
_position++;
}
break;
}
}
public SyntaxToken Lex()
{
_start = _position;
_kind = SyntaxKind.BadToken;
_value = null;
switch (Current)
{
case '\0':
_kind = SyntaxKind.EndOfFileToken;
break;
case ':' when Lookahead != ':':
_kind = SyntaxKind.ColonToken;
_position++;
break;
case ':' when Lookahead == ':':
_kind = SyntaxKind.NamespaceToken;
_value = null;
_position += 2;
break;
case '<' when Lookahead == '-' && Peek(2) == '+':
_kind = SyntaxKind.EditVariableToken;
_value = SyntaxKind.PlusToken;
_position += 3;
break;
case '<' when Lookahead == '-' && Peek(2) == '-':
_kind = SyntaxKind.EditVariableToken;
_value = SyntaxKind.MinusToken;
_position += 3;
break;
case '<' when Lookahead == '-' && Peek(2) == '*':
_kind = SyntaxKind.EditVariableToken;
_value = SyntaxKind.StarToken;
_position += 3;
break;
case '<' when Lookahead == '-' && Peek(2) == '/':
_kind = SyntaxKind.EditVariableToken;
_value = SyntaxKind.SlashToken;
_position += 3;
break;
case '+' when Lookahead == '+':
_kind = SyntaxKind.SingleEditVariableToken;
_value = SyntaxKind.PlusToken;
_position += 2;
break;
case '-' when Lookahead == '-':
_kind = SyntaxKind.SingleEditVariableToken;
_value = SyntaxKind.MinusToken;
_position += 2;
break;
case '+':
_kind = SyntaxKind.PlusToken;
_position++;
break;
case '-' when Lookahead != '>':
_kind = SyntaxKind.MinusToken;
_position++;
break;
case '*':
_kind = SyntaxKind.StarToken;
_position++;
break;
case '/' when Lookahead != '/' && Lookahead != '*':
_kind = SyntaxKind.SlashToken;
_position++;
break;
case '%':
_kind = SyntaxKind.PercentToken;
_position++;
break;
case '(':
_kind = SyntaxKind.OpenParenthesisToken;
_position++;
break;
case ')':
_kind = SyntaxKind.CloseParenthesisToken;
_position++;
break;
case '{':
_kind = SyntaxKind.OpenBraceToken;
_position++;
break;
case '}':
_kind = SyntaxKind.CloseBraceToken;
_position++;
break;
case '[':
_kind = SyntaxKind.OpenBracketToken;
_position++;
break;
case ']':
_kind = SyntaxKind.CloseBracketToken;
_position++;
break;
case ',':
_kind = SyntaxKind.CommaToken;
_position++;
break;
case '~':
_kind = SyntaxKind.AccessKeyword;
_position++;
break;
case '^':
_kind = SyntaxKind.HatToken;
_position++;
break;
case '&' when Lookahead != '&':
_kind = SyntaxKind.AmpersandToken;
_position++;
break;
case '&' when Lookahead == '&':
_kind = SyntaxKind.AmpersandAmpersandToken;
_position += 2;
break;
case '|' when Lookahead != '|':
_kind = SyntaxKind.PipeToken;
_position++;
break;
case '|' when Lookahead == '|':
_kind = SyntaxKind.PipePipeToken;
_position += 2;
break;
case '=':
_kind = SyntaxKind.EqualsToken;
_position++;
break;
case '!' when Lookahead != '=':
_kind = SyntaxKind.NotToken;
_position++;
break;
case '!' when Lookahead == '=':
_kind = SyntaxKind.NotEqualsToken;
_position += 2;
break;
case '<' when Lookahead == '-':
_kind = SyntaxKind.AssignToken;
_position += 2;
break;
case '<' when Lookahead == '=':
_kind = SyntaxKind.LessOrEqualsToken;
_position += 2;
break;
case '<' when Lookahead != '=' && Lookahead != '-' && Lookahead != '<':
_kind = SyntaxKind.LessToken;
_position++;
break;
case '>' when Lookahead != '=' && Lookahead != '>':
_position++;
_kind = SyntaxKind.GreaterToken;
break;
case '>' when Lookahead == '=':
_kind = SyntaxKind.GreaterOrEqualsToken;
_position += 2;
break;
case '-' when Lookahead == '>':
_kind = SyntaxKind.AccessToken;
_position += 2;
break;
case '>' when Lookahead == '>':
_kind = SyntaxKind.ShiftRight;
_position += 2;
break;
case '<' when Lookahead == '<':
_kind = SyntaxKind.ShiftLeft;
_position += 2;
break;
case '?':
_kind = SyntaxKind.QuestionMarkToken;
_position++;
break;
case '"':
ReadString();
break;
case '#':
case '/' when Lookahead == '/':
ReadComment();
break;
case '/' when Lookahead == '*':
ReadMLComment();
break;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
ReadNumber();
break;
case ' ':
case '\t':
case '\n':
case '\r':
case ';':
ReadWhiteSpace();
break;
default:
if (char.IsLetter(Current) || Current == '$')
{
ReadIdentifierOrKeyword();
}
else if (char.IsWhiteSpace(Current))
{
ReadWhiteSpace();
}
else
{
var span = new TextSpan(_position, 1);
var location = new TextLocation(_text, span);
_diagnostics.ReportBadCharacter(location, Current);
_position++;
}
break;
}
var length = _position - _start;
var text = SyntaxFacts.GetText(_kind);
if (text == null)
{
text = _text.ToString(_start, length);
}
return(new SyntaxToken(_syntaxTree, _kind, _start, text, _value));
}
public SyntaxToken Lex()
{
// <numbers>
// + - * / ()
// <whitespaces>
if (_position >= _text.Length)
{
return(new SyntaxToken(SyntaxKind.EndOfFileToken, _position, "\0", null));
}
var start = _position;
if (char.IsDigit(CurrentChar))
{
//var start = _position;
while (char.IsDigit(CurrentChar))
{
NextChar();
}
var length = _position - start;
var text = _text.Substring(start, length);
if (!int.TryParse(text, out var value))
{
_diagnostics.ReportInvalidNumber(new TextSpan(start, length), _text, typeof(int));
//_diagnostics.Add($"The number {_text} is NOT a valid Int32.");
//var textHint = "[Hint]: (Int32 is an immutable value type that represents signed 32-bit integers with values that range from negative 2,147,483,648 through positive 2,147,483,647.)";
//_diagnostics.Add(textHint);
Console.WriteLine();
Console.WriteLine("[Hint]: (Int32 is an immutable value type that represents signed 32-bit integers with values that range from negative 2,147,483,648 through positive 2,147,483,647.)");
}
return(new SyntaxToken(SyntaxKind.NumberToken, start, text, value));
}
if (char.IsWhiteSpace(CurrentChar))
{
while (char.IsWhiteSpace(CurrentChar))
{
NextChar();
}
var length = _position - start;
var text = _text.Substring(start, length);
//int.TryParse(text, out var value);
return(new SyntaxToken(SyntaxKind.WhitespaceToken, start, text, null));
}
// True
// False
if (char.IsLetter(CurrentChar))
{
while (char.IsLetter(CurrentChar))
{
NextChar();
}
var length = _position - start;
var text = _text.Substring(start, length);
var kind = SyntaxFacts.GetKeywordKind(text);
//int.TryParse(text, out var value);
return(new SyntaxToken(kind, start, text, null));
}
switch (CurrentChar)
{
case '+':
{
return(new SyntaxToken(SyntaxKind.PlusToken, _position++, "+", null));
}
case '-':
{
return(new SyntaxToken(SyntaxKind.MinusToken, _position++, "-", null));
}
case '*':
{
return(new SyntaxToken(SyntaxKind.MultiplyToken, _position++, "*", null));
}
case '/':
{
return(new SyntaxToken(SyntaxKind.DivideToken, _position++, "/", null));
}
case '(':
{
return(new SyntaxToken(SyntaxKind.OpenParenthesisToken, _position++, "(", null));
}
case ')':
{
return(new SyntaxToken(SyntaxKind.CloseParenthesisToken, _position++, ")", null));
}
case '&':
{
if (LookAhead == '&')
{
_position += 2;
return(new SyntaxToken(SyntaxKind.AmpersandAmpersandToken, start, "&&", null));
}
break;
}
case '|':
{
if (LookAhead == '|')
{
_position += 2;
return(new SyntaxToken(SyntaxKind.PipePipeToken, start, "||", null));
}
break;
}
case '=':
{
if (LookAhead == '=')
{
_position += 2;
return(new SyntaxToken(SyntaxKind.EqualsEqualsToken, start, "==", null));
}
else
{
_position += 1;
return(new SyntaxToken(SyntaxKind.EqualsToken, start, "=", null));
}
}
case '!':
{
if (LookAhead == '=')
{
_position += 2;
return(new SyntaxToken(SyntaxKind.ExclamationEqualsToken, start, "!=", null));
}
else
{
_position += 1;
return(new SyntaxToken(SyntaxKind.ExclamationToken, start, "!", null));
}
}
}
_diagnostics.ReportBadCharacter(_position, CurrentChar);
return(new SyntaxToken(SyntaxKind.BadToken, _position++, _text.Substring(_position - 1, 1), null));
}
internal SyntaxToken Lex()
{
_start = _position;
_kind = SyntaxKind.BadToken;
_value = null;
switch (Current)
{
case '\0':
{
_kind = SyntaxKind.EndOfFileToken;
break;
}
case '+':
{
MoveNext();
_kind = SyntaxKind.PlusToken;
break;
}
case '-':
{
MoveNext();
_kind = SyntaxKind.MinusToken;
break;
}
case '*':
{
MoveNext();
_kind = SyntaxKind.StarToken;
break;
}
case '/':
{
MoveNext();
_kind = SyntaxKind.ForwardSlashToken;
break;
}
case '(':
{
MoveNext();
_kind = SyntaxKind.OpenParenthesisToken;
break;
}
case ')':
{
MoveNext();
_kind = SyntaxKind.CloseParenthesisToken;
break;
}
case '{':
{
MoveNext();
_kind = SyntaxKind.OpenBraceToken;
break;
}
case '}':
{
MoveNext();
_kind = SyntaxKind.CloseBraceToken;
break;
}
case '~':
{
MoveNext();
_kind = SyntaxKind.TildeToken;
break;
}
case '^':
{
MoveNext();
_kind = SyntaxKind.HatToken;
break;
}
case '!':
{
if (LookAhead == '=')
{
MoveNext(2);
_kind = SyntaxKind.BangEqualsToken;
}
else
{
MoveNext();
_kind = SyntaxKind.BangToken;
}
break;
}
case '&':
{
if (LookAhead == '&')
{
MoveNext(2);
_kind = SyntaxKind.AndAndToken;
}
else
{
MoveNext();
_kind = SyntaxKind.AndToken;
}
break;
}
case '|':
{
if (LookAhead == '|')
{
MoveNext(2);
_kind = SyntaxKind.PipePipeToken;
}
else
{
MoveNext();
_kind = SyntaxKind.PipeToken;
}
break;
}
case '=':
{
if (LookAhead == '=')
{
MoveNext(2);
_kind = SyntaxKind.EqualsEqualsToken;
}
else
{
MoveNext();
_kind = SyntaxKind.EqualsToken;
}
break;
}
case '<':
{
if (LookAhead == '=')
{
MoveNext(2);
_kind = SyntaxKind.LessOrEqualsToken;
}
else
{
MoveNext();
_kind = SyntaxKind.LessToken;
}
break;
}
case '>':
{
if (LookAhead == '=')
{
MoveNext(2);
_kind = SyntaxKind.GreaterOrEqualsToken;
}
else
{
MoveNext();
_kind = SyntaxKind.GreaterToken;
}
break;
}
case '"':
{
ReadString();
break;
}
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
{
ReadNumbersToken();
break;
}
case ' ':
case '\t':
{
ReadWhitespaceTokens();
break;
}
default:
{
if (char.IsLetter(Current))
{
ReadLettersToken();
}
else if (char.IsWhiteSpace(Current))
{
ReadWhitespaceTokens();
}
else
{
_diagnostics.ReportBadCharacter(_position, Current);
MoveNext();
}
break;
}
}
{
var text = SyntaxFacts.GetText(_kind);
var length = _position - _start;
if (text is null)
{
text = _text.ToString(_start, length);
}
return(new SyntaxToken(_kind, _start, text, _value));
}
void ReadNumbersToken()
{
do
{
MoveNext();
}while (char.IsDigit(Current));
var length = _position - _start;
var text = _text.ToString(_start, length);
if (!int.TryParse(text, out var value))
{
_diagnostics.ReportInvalidNumber(new TextSpan(_start, length), text, TypeSymbol.Int);
}
_kind = SyntaxKind.NumberToken;
_value = value;
}
void ReadLettersToken()
{
do
{
MoveNext();
}while (char.IsLetter(Current));
var text = _text.ToString(_start, _position - _start);
_kind = SyntaxFacts.GetKeywordKind(text);
}
void ReadWhitespaceTokens()
{
do
{
MoveNext();
}while (char.IsWhiteSpace(Current));
_kind = SyntaxKind.WhiteSpaceToken;
}
void ReadString()
{
MoveNext();
var done = false;
var builder = new StringBuilder();
while (!done)
{
switch (Current)
{
case '\0':
case '\r':
case '\n':
var span = new TextSpan(_start, 1);
Diagnostics.ReportUnterminatedString(span);
done = true;
break;
case '\\':
{
if (LookAhead == '"')
{
MoveNext();
}
builder.Append(Current);
MoveNext();
break;
}
case '"':
{
done = true;
MoveNext();
break;
}
default:
{
builder.Append(Current);
MoveNext();
break;
}
}
}
_kind = SyntaxKind.StringToken;
_value = builder.ToString();
}
}
