예제 #1
0
        public void regular_string_tokenized(string theString, int count)
        {
            var res = new TokenizerTestHelper(theString);

            res.Tokens.Count.Should().Be(count);
            if (count == 1)
            {
                res.Tokens.Single().TokenType.Should().Be(TokenType.StringDeclaration);
            }
        }
예제 #2
0
        public void regular_words_tokenized(string theString, int count, int[] wordPositions)
        {
            var res = new TokenizerTestHelper(theString);

            res.Tokens.Count.Should().Be(count);
            for (int i = 0; i < res.Tokens.Count; i++)
            {
                bool isWord       = res.Tokens[i].TokenType == TokenType.Word;
                bool shouldBeWord = wordPositions.Contains(i);
                isWord.Should().Be(shouldBeWord);
            }
        }
예제 #3
0
        public void tokens_can_be_separated_by_something_else_than_whitespace(string input, int tokenCount)
        {
            var res = new TokenizerTestHelper(input);

            res.Tokens.Count.Should().Be(tokenCount);
        }