public void IncrementsOffsetCorrectly() { string input = string.Empty; for (int repeat = 0; repeat < 4000; repeat++) { input += "test test test test "; } StreamLemmatizer sl = new StreamLemmatizer(new StringReader(input)); string token = string.Empty; List<Token> results = new List<Token>(); int previousOffest = -5; while (sl.LemmatizeNextToken(out token, results) > 0) { Assert.AreEqual<int>(previousOffest, sl.StartOffset - 5); Assert.AreEqual<int>(4, sl.EndOffset - sl.StartOffset); previousOffest = sl.StartOffset; } }
public void IncrementsOffsetCorrectly() { string input = string.Empty; for (int repeat = 0; repeat < 4000; repeat++) { input += "test test test test "; } StreamLemmatizer sl = new StreamLemmatizer(new StringReader(input)); string token = string.Empty; List <Token> results = new List <Token>(); int previousOffest = -5; while (sl.LemmatizeNextToken(out token, results) > 0) { Assert.Equal(previousOffest, sl.StartOffset - 5); Assert.Equal(4, sl.EndOffset - sl.StartOffset); previousOffest = sl.StartOffset; } }
public HtmlMorphAnalyzer(StreamLemmatizer hml) : base(hml) { }