public void IncrementsOffsetCorrectlyWithAnotherReader2()
{
const string input = @"test1 <a href=""foo"">testlink</a> test2 test3";
CharFilter filter = new HTMLStripCharFilter(CharReader.Get(new StringReader(input)));
Tokenizer t = new Tokenizer(filter);
string token = string.Empty;
List<Token> results = new List<Token>();
t.NextToken(out token);
Assert.Equal(0, filter.CorrectOffset(t.Offset));
Assert.Equal(5, t.LengthInSource);
t.NextToken(out token);
Assert.Equal(20, filter.CorrectOffset(t.Offset));
Assert.Equal(8, t.LengthInSource);
t.NextToken(out token);
Assert.Equal(33, filter.CorrectOffset(t.Offset));
Assert.Equal(5, t.LengthInSource);
t.NextToken(out token);
Assert.Equal(39, filter.CorrectOffset(t.Offset));
Assert.Equal(5, t.LengthInSource);
}
public void IncrementsOffsetCorrectlyWithAnotherReader()
{
int[] expectedOffsets = { 0, 5, 10, 15 };
int curPos = 0;
string token = string.Empty;
Tokenizer t =
new Tokenizer(
new HTMLStripCharFilter(CharReader.Get(new System.IO.StringReader(@"test<a href=""foo"">test</a>test test"))));
while (true)
{
Tokenizer.TokenType token_type = t.NextToken(out token);
if (token_type == 0)
break;
Assert.Equal(expectedOffsets[curPos++], t.Offset);
Assert.Equal(4, t.LengthInSource);
}
}