public virtual void TestOffsets()
{
string input = "abc- def geh 1234- 5678-";
TokenStream ts = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
ts = new HyphenatedWordsFilter(ts);
AssertTokenStreamContents(ts, new string[] { "abcdef", "geh", "12345678-" }, new int[] { 0, 9, 13 }, new int[] { 8, 12, 24 });
}
public virtual void TestOffsets()
{
string input = "abc- def geh 1234- 5678-";
TokenStream ts = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
ts = new HyphenatedWordsFilter(ts);
AssertTokenStreamContents(ts, new string[] { "abcdef", "geh", "12345678-" }, new int[] { 0, 9, 13 }, new int[] { 8, 12, 24 });
}
public virtual void TestHyphenAtEnd()
{
string input = "ecologi-\r\ncal devel-\r\n\r\nop compre-\u0009hensive-hands-on and ecology-";
// first test
TokenStream ts = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
ts = new HyphenatedWordsFilter(ts);
AssertTokenStreamContents(ts, new string[] { "ecological", "develop", "comprehensive-hands-on", "and", "ecology-" });
}
public virtual void TestHyphenAtEnd()
{
string input = "ecologi-\r\ncal devel-\r\n\r\nop compre-\u0009hensive-hands-on and ecology-";
// first test
TokenStream ts = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
ts = new HyphenatedWordsFilter(ts);
AssertTokenStreamContents(ts, new string[] { "ecological", "develop", "comprehensive-hands-on", "and", "ecology-" });
}