public TokenType GetTokenType(Token previous, string token)
{
if (string.IsNullOrEmpty(token)) return TokenType.Null;
if (char.IsNumber(token[0]) && token.All(c => char.IsDigit(c)))
{
return TokenType.Number;
}
if ((previous.Type == TokenType.Word || previous.Type == TokenType.Number) && token == ".")
{
return TokenType.SentenceEnd;
}
if (token.All(c => char.IsWhiteSpace(c))) return TokenType.Space;
if (token.Any(c => !char.IsLetterOrDigit(c))) return TokenType.Symbol;
return TokenType.Word;
}
public Token(Token previous, string value, int index)
{
Text = value;
Index = index;
Type = GetTokenType(previous, value);
}
