Exemple #1
0
        public void Test()
        {
            List <WordToken> tokens = TokenBuilderTestUtils
                                      .Tokenise <WordTokenBuilder, WordToken>("ab1 .z\nab");

            Assert.Equal(new [] { "ab1", "z", "ab" }, tokens.Select(t => t.Value));
            Assert.Equal(new long[] { 0, 5, 7 }, tokens.Select(t => t.Position));
        }
Exemple #2
0
        public void TestAccept()
        {
            List <NumberToken> tokens = TokenBuilderTestUtils
                                        .Tokenise <NumberTokenBuilder, NumberToken>("ab1 a12 123a \n777 3 avx 999");

            Assert.Equal(new [] { 123, 777, 3, 999 }, tokens.Select(t => t.Value));
            Assert.Equal(new long[] { 8, 14, 18, 24 }, tokens.Select(t => t.Position));
        }
Exemple #3
0
        public void Test()
        {
            string text = "ab1 .z\na";
            List <CharacterToken> tokens = TokenBuilderTestUtils
                                           .Tokenise <CharacterTokenBuilder, CharacterToken>(text);

            Assert.Equal(new [] { 'a', 'b', '1', ' ', '.', 'z', '\n', 'a' }, tokens.Select(t => t.Value));
            Assert.Equal(new long[] { 0, 1, 2, 3, 4, 5, 6, 7 }, tokens.Select(t => t.Position));
        }
Exemple #4
0
        public void Test()
        {
            string text = "  Hello world!!! \nHello world.\n   Hello worl";
            List <SentenceToken> tokens = TokenBuilderTestUtils
                                          .Tokenise <SentenceTokenBuilder, SentenceToken>(text);

            Assert.Equal(new [] { "Hello world!!!", "Hello world.", "Hello worl" },
                         tokens.Select(t => t.Value));
            Assert.Equal(new long[] { 2, 18, 34 }, tokens.Select(t => t.Position));
        }