public void Test()
{
List <WordToken> tokens = TokenBuilderTestUtils
.Tokenise <WordTokenBuilder, WordToken>("ab1 .z\nab");
Assert.Equal(new [] { "ab1", "z", "ab" }, tokens.Select(t => t.Value));
Assert.Equal(new long[] { 0, 5, 7 }, tokens.Select(t => t.Position));
}
public void TestAccept()
{
List <NumberToken> tokens = TokenBuilderTestUtils
.Tokenise <NumberTokenBuilder, NumberToken>("ab1 a12 123a \n777 3 avx 999");
Assert.Equal(new [] { 123, 777, 3, 999 }, tokens.Select(t => t.Value));
Assert.Equal(new long[] { 8, 14, 18, 24 }, tokens.Select(t => t.Position));
}
public void Test()
{
string text = "ab1 .z\na";
List <CharacterToken> tokens = TokenBuilderTestUtils
.Tokenise <CharacterTokenBuilder, CharacterToken>(text);
Assert.Equal(new [] { 'a', 'b', '1', ' ', '.', 'z', '\n', 'a' }, tokens.Select(t => t.Value));
Assert.Equal(new long[] { 0, 1, 2, 3, 4, 5, 6, 7 }, tokens.Select(t => t.Position));
}
public void Test()
{
string text = " Hello world!!! \nHello world.\n Hello worl";
List <SentenceToken> tokens = TokenBuilderTestUtils
.Tokenise <SentenceTokenBuilder, SentenceToken>(text);
Assert.Equal(new [] { "Hello world!!!", "Hello world.", "Hello worl" },
tokens.Select(t => t.Value));
Assert.Equal(new long[] { 2, 18, 34 }, tokens.Select(t => t.Position));
}