public void IncrementsOffsetCorrectlyAlsoWhenBuffered()
{
string token = string.Empty;
string input = string.Empty;
for (int repeat = 0; repeat < 4000; repeat++)
{
input += "test test test test ";
}
Tokenizer t = GetTokenizer(input);
int previousOffest = -5;
while (true)
{
HebMorph.Tokenizer.TokenType token_type = t.NextToken(out token);
if (token_type == 0)
{
break;
}
Assert.Equal <int>(previousOffest, t.Offset - 5);
Assert.Equal <int>(4, t.LengthInSource);
previousOffest = t.Offset;
}
}
public void IncrementsOffsetCorrectly()
{
int[] expectedOffsets = { 0, 5, 10, 15 };
int curPos = 0;
string token = string.Empty;
Tokenizer t = GetTokenizer("test test test test");
while (true)
{
HebMorph.Tokenizer.TokenType token_type = t.NextToken(out token);
if (token_type == 0)
{
break;
}
Assert.Equal(expectedOffsets[curPos++], t.Offset);
Assert.Equal(4, t.LengthInSource);
}
}
