public void TestNext()
{
var encoding = Encoding.UTF8;
String test = "The quick|JJ red|JJ fox|NN jumped|VB over the lazy|JJ brown|JJ dogs|NN";
DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new WhitespaceTokenizer(new StringReader(test)));
AssertTermEquals("The", filter, null);
AssertTermEquals("quick", filter, encoding.GetBytes("JJ"));
AssertTermEquals("red", filter, encoding.GetBytes("JJ"));
AssertTermEquals("fox", filter, encoding.GetBytes("NN"));
AssertTermEquals("jumped", filter, encoding.GetBytes("VB"));
AssertTermEquals("over", filter, null);
AssertTermEquals("the", filter, null);
AssertTermEquals("lazy", filter, encoding.GetBytes("JJ"));
AssertTermEquals("brown", filter, encoding.GetBytes("JJ"));
AssertTermEquals("dogs", filter, encoding.GetBytes("NN"));
Assert.False(filter.IncrementToken());
}
public void TestFloatEncoding()
{
String test = "The quick|1.0 red|2.0 fox|3.5 jumped|0.5 over the lazy|5 brown|99.3 dogs|83.7";
DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new WhitespaceTokenizer(new StringReader(test)), '|', new FloatEncoder());
ITermAttribute termAtt = filter.GetAttribute <ITermAttribute>();
IPayloadAttribute payAtt = filter.GetAttribute <IPayloadAttribute>();
AssertTermEquals("The", filter, termAtt, payAtt, null);
AssertTermEquals("quick", filter, termAtt, payAtt, PayloadHelper.EncodeFloat(1.0f));
AssertTermEquals("red", filter, termAtt, payAtt, PayloadHelper.EncodeFloat(2.0f));
AssertTermEquals("fox", filter, termAtt, payAtt, PayloadHelper.EncodeFloat(3.5f));
AssertTermEquals("jumped", filter, termAtt, payAtt, PayloadHelper.EncodeFloat(0.5f));
AssertTermEquals("over", filter, termAtt, payAtt, null);
AssertTermEquals("the", filter, termAtt, payAtt, null);
AssertTermEquals("lazy", filter, termAtt, payAtt, PayloadHelper.EncodeFloat(5.0f));
AssertTermEquals("brown", filter, termAtt, payAtt, PayloadHelper.EncodeFloat(99.3f));
AssertTermEquals("dogs", filter, termAtt, payAtt, PayloadHelper.EncodeFloat(83.7f));
Assert.False(filter.IncrementToken());
}
public void TestIntEncoding()
{
String test = "The quick|1 red|2 fox|3 jumped over the lazy|5 brown|99 dogs|83";
DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new WhitespaceTokenizer(new StringReader(test)), '|', new IntegerEncoder());
ITermAttribute termAtt = filter.GetAttribute <ITermAttribute>();
IPayloadAttribute payAtt = filter.GetAttribute <IPayloadAttribute>();
AssertTermEquals("The", filter, termAtt, payAtt, null);
AssertTermEquals("quick", filter, termAtt, payAtt, PayloadHelper.EncodeInt(1));
AssertTermEquals("red", filter, termAtt, payAtt, PayloadHelper.EncodeInt(2));
AssertTermEquals("fox", filter, termAtt, payAtt, PayloadHelper.EncodeInt(3));
AssertTermEquals("jumped", filter, termAtt, payAtt, null);
AssertTermEquals("over", filter, termAtt, payAtt, null);
AssertTermEquals("the", filter, termAtt, payAtt, null);
AssertTermEquals("lazy", filter, termAtt, payAtt, PayloadHelper.EncodeInt(5));
AssertTermEquals("brown", filter, termAtt, payAtt, PayloadHelper.EncodeInt(99));
AssertTermEquals("dogs", filter, termAtt, payAtt, PayloadHelper.EncodeInt(83));
Assert.False(filter.IncrementToken());
}