private bool ScanToken(ref TokenInfo info)
{
info.Text = string.Empty;
info.TokenType = TokenKind.None;
var currentChar = _scanner.Advance();
ConsumeWhiteSpace(ref currentChar);
switch (currentChar)
{
case '+':
case '-':
case '%':
case '*':
case '/':
case '\\':
case '^':
info.TokenType = TokenKind.Operator;
break;
case '&':
if (_scanner.Peek() == '&')
{
_scanner.Advance();
}
info.TokenType = TokenKind.Operator;
break;
case '|':
if (_scanner.Peek() == '|')
{
_scanner.Advance();
}
info.TokenType = TokenKind.Operator;
break;
case '!':
if (_scanner.Peek() == '=')
{
_scanner.Advance();
}
info.TokenType = TokenKind.Operator;
break;
case '<':
if (_scanner.Peek() == '=' ||
_scanner.Peek() == '<')
{
_scanner.Advance();
}
info.TokenType = TokenKind.Operator;
break;
case '>':
if (_scanner.Peek() == '=' ||
_scanner.Peek() == '>')
{
_scanner.Advance();
}
info.TokenType = TokenKind.Operator;
break;
case '=':
{
if (_scanner.Peek() == '>')
{
_scanner.Advance();
info.TokenType = TokenKind.Define;
break;
}
if (_scanner.Peek() == '=')
{
_scanner.Advance();
info.TokenType = TokenKind.Operator;
break;
}
info.TokenType = TokenKind.Assign;
break;
}
case ':':
info.TokenType = TokenKind.Colon;
break;
case ';':
info.TokenType = TokenKind.Semicolon;
break;
case ',':
info.TokenType = TokenKind.Comma;
break;
case '.':
{
var peekChar = _scanner.Peek();
if (peekChar == '.')
{
_scanner.Advance();
info.TokenType = TokenKind.DotDot;
break;
}
// Real literals don't have to start with a digit (e.g. .123)
if (char.IsDigit(peekChar))
{
ScanNumeric(ref info, ref currentChar);
break;
}
info.TokenType = TokenKind.Dot;
break;
}
case '?':
info.TokenType = TokenKind.QuestionMark;
break;
case '\'':
ScanCharLiteral(ref info, ref currentChar);
break;
case '\"':
ScanStringLiteral(ref info, ref currentChar);
break;
case '(':
{
if (_scanner.Peek() == '*')
{
currentChar = _scanner.Advance();
ConsumeComment(ref currentChar);
return(ScanToken(ref info));
}
info.TokenType = TokenKind.LeftParen;
break;
}
case ')':
// ScanComment will handle comment endings
info.TokenType = TokenKind.RightParen;
break;
case '{':
info.TokenType = TokenKind.LeftBrack;
break;
case '}':
info.TokenType = TokenKind.RightBrack;
break;
case '[':
info.TokenType = TokenKind.LeftSquareBrack;
break;
case ']':
info.TokenType = TokenKind.RightSquareBrack;
break;
case '_':
case char c when char.IsLetter(c):
ScanIdentifierOrKeyword(ref info, ref currentChar);
break;
case char c when char.IsDigit(c):
ScanNumeric(ref info, ref currentChar);
break;
case char c when c == _scanner.InvalidItem:
// True end of file, exit completely
if (_scanner.IsAtEnd())
{
return(false);
}
// Check if is just a bad char in source.
// If not at end and is a bad char don't stop the scanning
// process - exit method normally. Parsing will catch the bad token
if (_scanner.Position <= _scanner.SourceLength &&
_scanner.Base < _scanner.SourceLength)
{
break;
}
// Should be end of file, otherwise it is in a bad state, exit
return(false);
default:
if (!_flags.AllowUnicode())
{
throw new IllegalCharacterException("Character not allowed", _scanner.CurrentSourceLine)
{
Character = currentChar
}
}
;
throw new FeatureNotSupportedException("Unicode identifiers currently not supported",
_scanner.CurrentSourceLine)
{
FeatureName = "Unicode Identifiers"
};
}
info.Text = _scanner.CurrentWindow;
info.SourceLineNumber = _scanner.CurrentSourceLine;
_scanner.Base = _scanner.Position;
return(true);
}
private static IToken CreateValueToken(ref TokenInfo info, ref LexerFlag flags)
{
switch (info.TokenType)
{
case TokenKind.SignedInt:
{
if (!long.TryParse(info.Text, out var longValue))
{
throw new InvalidTokenException("Unable to parse 64bit signed integer value", info.SourceLineNumber)
{
InvalidText = info.Text
}
}
;
return(new SignedIntToken
{
Value = longValue,
LineNumber = info.SourceLineNumber
});
}
case TokenKind.UnsignedInt:
{
if (!ulong.TryParse(info.Text.TrimEnd('u').TrimEnd('U'), out var longValue))
{
throw new InvalidTokenException("Unable to parse 64bit unsigned integer value", info.SourceLineNumber)
{
InvalidText = info.Text
}
}
;
return(new UnsignedIntToken
{
Value = longValue,
LineNumber = info.SourceLineNumber
});
}
case TokenKind.Real:
{
if (!double.TryParse(info.Text, out var longValue))
{
throw new InvalidTokenException("Unable to parse double precision floating point value", info.SourceLineNumber)
{
InvalidText = info.Text
}
}
;
return(new RealToken
{
Value = longValue,
LineNumber = info.SourceLineNumber
});
}
case TokenKind.Keyword:
case TokenKind.Identifier:
{
var text = info.Text;
var keyword = System.Enum.GetNames(typeof(Keyword)).FirstOrDefault(kw => kw.ToLower() == text);
if (keyword != null)
{
return new KeywordToken
{
Value = System.Enum.Parse <Keyword>(keyword),
LineNumber = info.SourceLineNumber
}
}
;
return(new IdToken
{
Value = info.Text,
LineNumber = info.SourceLineNumber
});
}
case TokenKind.BoolLiteral:
{
switch (info.Text)
{
case "true":
case "false":
if (!bool.TryParse(info.Text, out var boolValue))
{
throw new InvalidTokenException("Unable to parse bool literal", info.SourceLineNumber)
{
InvalidText = info.Text
}
}
;
return(new BoolLiteralToken
{
Value = boolValue,
LineNumber = info.SourceLineNumber
});
case string s when s.ToLower() == "true" || s.ToLower() == "false":
throw new InvalidTokenException("Unable to parse bool literal", info.SourceLineNumber)
{
InvalidText = info.Text
};
default:
// Lexer only does a basic check for literal, there is an edge case where an identifer
// can sneek past that check
return(new IdToken
{
Value = info.Text,
LineNumber = info.SourceLineNumber
});
}
}
case TokenKind.StringLiteral:
{
var stringValue = info.Text.TrimStart('\"').TrimEnd('\"');
return(new StringLiteralToken
{
Value = stringValue,
LineNumber = info.SourceLineNumber
});
}
case TokenKind.CharLiteral:
{
var rawChar = info.Text.TrimStart('\'').TrimEnd('\'');
if (rawChar.Length > 1 && (rawChar[0] != '\\' || rawChar.Length > 2))
{
throw new InvalidTokenException("Char literal too long", info.SourceLineNumber)
{
InvalidText = rawChar
}
}
;
if (!char.TryParse(rawChar, out var charValue))
{
throw new InvalidTokenException("Char value invalid", info.SourceLineNumber)
{
InvalidText = rawChar
}
}
;
if (charValue > 127 && !flags.AllowUnicode())
{
throw new IllegalCharacterException("Only ASCII characters are currently supported",
info.SourceLineNumber)
{
Character = charValue
}
}
;
return(new CharLiteralToken
{
Value = charValue,
LineNumber = info.SourceLineNumber
});
}
case TokenKind.Operator:
{
var text = info.Text;
var operatorValue = TokenHelper.OperatorValueMap
.FirstOrDefault(kv => kv.Value == text.Trim()).Key;
if (operatorValue == default(Operator))
{
throw new InvalidTokenException("Unknown operator found", info.SourceLineNumber)
{
InvalidText = info.Text
}
}
;
return(new OperatorToken
{
Value = operatorValue,
LineNumber = info.SourceLineNumber
});
}
default:
throw new NotSupportedException($"TokenType {info.TokenType} currently not supported");
}
}