Exemple #1
0
        public TokenType GetTokenType(Token previous, string token)
        {
            if (string.IsNullOrEmpty(token)) return TokenType.Null;

            if (char.IsNumber(token[0]) && token.All(c => char.IsDigit(c)))
            {
                return TokenType.Number;
            }
            if ((previous.Type == TokenType.Word || previous.Type == TokenType.Number) && token == ".")
            {
                return TokenType.SentenceEnd;
            }

            if (token.All(c => char.IsWhiteSpace(c))) return TokenType.Space;

            if (token.Any(c => !char.IsLetterOrDigit(c))) return TokenType.Symbol;

            return TokenType.Word;
        }
Exemple #2
0
 public Token(Token previous, string value, int index)
 {
     Text = value;
     Index = index;
     Type = GetTokenType(previous, value);
 }