public virtual void TestOffsets() { string input = "abc- def geh 1234- 5678-"; TokenStream ts = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false); ts = new HyphenatedWordsFilter(ts); AssertTokenStreamContents(ts, new string[] { "abcdef", "geh", "12345678-" }, new int[] { 0, 9, 13 }, new int[] { 8, 12, 24 }); }
public virtual void TestOffsets() { string input = "abc- def geh 1234- 5678-"; TokenStream ts = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false); ts = new HyphenatedWordsFilter(ts); AssertTokenStreamContents(ts, new string[] { "abcdef", "geh", "12345678-" }, new int[] { 0, 9, 13 }, new int[] { 8, 12, 24 }); }
public virtual void TestHyphenAtEnd() { string input = "ecologi-\r\ncal devel-\r\n\r\nop compre-\u0009hensive-hands-on and ecology-"; // first test TokenStream ts = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false); ts = new HyphenatedWordsFilter(ts); AssertTokenStreamContents(ts, new string[] { "ecological", "develop", "comprehensive-hands-on", "and", "ecology-" }); }
public virtual void TestHyphenAtEnd() { string input = "ecologi-\r\ncal devel-\r\n\r\nop compre-\u0009hensive-hands-on and ecology-"; // first test TokenStream ts = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false); ts = new HyphenatedWordsFilter(ts); AssertTokenStreamContents(ts, new string[] { "ecological", "develop", "comprehensive-hands-on", "and", "ecology-" }); }