public void IncrementsOffsetCorrectly()
{
string input = string.Empty;
for (int repeat = 0; repeat < 4000; repeat++)
{
input += "test test test test ";
}
StreamLemmatizer sl = new StreamLemmatizer(new StringReader(input));
string token = string.Empty;
List<Token> results = new List<Token>();
int previousOffest = -5;
while (sl.LemmatizeNextToken(out token, results) > 0)
{
Assert.AreEqual<int>(previousOffest, sl.StartOffset - 5);
Assert.AreEqual<int>(4, sl.EndOffset - sl.StartOffset);
previousOffest = sl.StartOffset;
}
}
public void IncrementsOffsetCorrectly()
{
string input = string.Empty;
for (int repeat = 0; repeat < 4000; repeat++)
{
input += "test test test test ";
}
StreamLemmatizer sl = new StreamLemmatizer(new StringReader(input));
string token = string.Empty;
List <Token> results = new List <Token>();
int previousOffest = -5;
while (sl.LemmatizeNextToken(out token, results) > 0)
{
Assert.Equal(previousOffest, sl.StartOffset - 5);
Assert.Equal(4, sl.EndOffset - sl.StartOffset);
previousOffest = sl.StartOffset;
}
}
public HtmlMorphAnalyzer(StreamLemmatizer hml)
: base(hml)
{
}
public HtmlMorphAnalyzer(StreamLemmatizer hml) : base(hml)
{
}