public virtual void TestNext() { string test = "The quick|JJ red|JJ fox|NN jumped|VB over the lazy|JJ brown|JJ dogs|NN"; DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), DelimitedPayloadTokenFilter.DEFAULT_DELIMITER, new IdentityEncoder()); filter.Reset(); AssertTermEquals("The", filter, null); AssertTermEquals("quick", filter, "JJ".getBytes(Encoding.UTF8)); AssertTermEquals("red", filter, "JJ".getBytes(Encoding.UTF8)); AssertTermEquals("fox", filter, "NN".getBytes(Encoding.UTF8)); AssertTermEquals("jumped", filter, "VB".getBytes(Encoding.UTF8)); AssertTermEquals("over", filter, null); AssertTermEquals("the", filter, null); AssertTermEquals("lazy", filter, "JJ".getBytes(Encoding.UTF8)); AssertTermEquals("brown", filter, "JJ".getBytes(Encoding.UTF8)); AssertTermEquals("dogs", filter, "NN".getBytes(Encoding.UTF8)); assertFalse(filter.IncrementToken()); filter.End(); filter.Dispose(); }
public virtual void TestIntEncoding() { string test = "The quick|1 red|2 fox|3 jumped over the lazy|5 brown|99 dogs|83"; DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), '|', new IntegerEncoder()); ICharTermAttribute termAtt = filter.GetAttribute <ICharTermAttribute>(); IPayloadAttribute payAtt = filter.GetAttribute <IPayloadAttribute>(); filter.Reset(); AssertTermEquals("The", filter, termAtt, payAtt, null); AssertTermEquals("quick", filter, termAtt, payAtt, PayloadHelper.EncodeInt32(1)); AssertTermEquals("red", filter, termAtt, payAtt, PayloadHelper.EncodeInt32(2)); AssertTermEquals("fox", filter, termAtt, payAtt, PayloadHelper.EncodeInt32(3)); AssertTermEquals("jumped", filter, termAtt, payAtt, null); AssertTermEquals("over", filter, termAtt, payAtt, null); AssertTermEquals("the", filter, termAtt, payAtt, null); AssertTermEquals("lazy", filter, termAtt, payAtt, PayloadHelper.EncodeInt32(5)); AssertTermEquals("brown", filter, termAtt, payAtt, PayloadHelper.EncodeInt32(99)); AssertTermEquals("dogs", filter, termAtt, payAtt, PayloadHelper.EncodeInt32(83)); assertFalse(filter.IncrementToken()); filter.End(); filter.Dispose(); }
public virtual void TestFloatEncoding() { string test = "The quick|1.0 red|2.0 fox|3.5 jumped|0.5 over the lazy|5 brown|99.3 dogs|83.7"; DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), '|', new SingleEncoder()); ICharTermAttribute termAtt = filter.GetAttribute <ICharTermAttribute>(); IPayloadAttribute payAtt = filter.GetAttribute <IPayloadAttribute>(); filter.Reset(); AssertTermEquals("The", filter, termAtt, payAtt, null); AssertTermEquals("quick", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(1.0f)); AssertTermEquals("red", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(2.0f)); AssertTermEquals("fox", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(3.5f)); AssertTermEquals("jumped", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(0.5f)); AssertTermEquals("over", filter, termAtt, payAtt, null); AssertTermEquals("the", filter, termAtt, payAtt, null); AssertTermEquals("lazy", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(5.0f)); AssertTermEquals("brown", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(99.3f)); AssertTermEquals("dogs", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(83.7f)); assertFalse(filter.IncrementToken()); filter.End(); filter.Dispose(); }
public virtual void TestPayloads() { string test = "The quick|JJ red|JJ fox|NN jumped|VB over the lazy|JJ brown|JJ dogs|NN"; DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), DelimitedPayloadTokenFilter.DEFAULT_DELIMITER, new IdentityEncoder()); ICharTermAttribute termAtt = filter.GetAttribute <ICharTermAttribute>(); IPayloadAttribute payAtt = filter.GetAttribute <IPayloadAttribute>(); filter.Reset(); AssertTermEquals("The", filter, termAtt, payAtt, null); AssertTermEquals("quick", filter, termAtt, payAtt, "JJ".getBytes(Encoding.UTF8)); AssertTermEquals("red", filter, termAtt, payAtt, "JJ".getBytes(Encoding.UTF8)); AssertTermEquals("fox", filter, termAtt, payAtt, "NN".getBytes(Encoding.UTF8)); AssertTermEquals("jumped", filter, termAtt, payAtt, "VB".getBytes(Encoding.UTF8)); AssertTermEquals("over", filter, termAtt, payAtt, null); AssertTermEquals("the", filter, termAtt, payAtt, null); AssertTermEquals("lazy", filter, termAtt, payAtt, "JJ".getBytes(Encoding.UTF8)); AssertTermEquals("brown", filter, termAtt, payAtt, "JJ".getBytes(Encoding.UTF8)); AssertTermEquals("dogs", filter, termAtt, payAtt, "NN".getBytes(Encoding.UTF8)); assertFalse(filter.IncrementToken()); filter.End(); filter.Dispose(); }
public virtual void TestIntEncoding() { string test = "The quick|1 red|2 fox|3 jumped over the lazy|5 brown|99 dogs|83"; DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), '|', new IntegerEncoder()); ICharTermAttribute termAtt = filter.GetAttribute<ICharTermAttribute>(); IPayloadAttribute payAtt = filter.GetAttribute<IPayloadAttribute>(); filter.Reset(); AssertTermEquals("The", filter, termAtt, payAtt, null); AssertTermEquals("quick", filter, termAtt, payAtt, PayloadHelper.EncodeInt(1)); AssertTermEquals("red", filter, termAtt, payAtt, PayloadHelper.EncodeInt(2)); AssertTermEquals("fox", filter, termAtt, payAtt, PayloadHelper.EncodeInt(3)); AssertTermEquals("jumped", filter, termAtt, payAtt, null); AssertTermEquals("over", filter, termAtt, payAtt, null); AssertTermEquals("the", filter, termAtt, payAtt, null); AssertTermEquals("lazy", filter, termAtt, payAtt, PayloadHelper.EncodeInt(5)); AssertTermEquals("brown", filter, termAtt, payAtt, PayloadHelper.EncodeInt(99)); AssertTermEquals("dogs", filter, termAtt, payAtt, PayloadHelper.EncodeInt(83)); assertFalse(filter.IncrementToken()); filter.End(); filter.Dispose(); }
public virtual void TestFloatEncoding() { string test = "The quick|1.0 red|2.0 fox|3.5 jumped|0.5 over the lazy|5 brown|99.3 dogs|83.7"; DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), '|', new FloatEncoder()); ICharTermAttribute termAtt = filter.GetAttribute<ICharTermAttribute>(); IPayloadAttribute payAtt = filter.GetAttribute<IPayloadAttribute>(); filter.Reset(); AssertTermEquals("The", filter, termAtt, payAtt, null); AssertTermEquals("quick", filter, termAtt, payAtt, PayloadHelper.EncodeFloat(1.0f)); AssertTermEquals("red", filter, termAtt, payAtt, PayloadHelper.EncodeFloat(2.0f)); AssertTermEquals("fox", filter, termAtt, payAtt, PayloadHelper.EncodeFloat(3.5f)); AssertTermEquals("jumped", filter, termAtt, payAtt, PayloadHelper.EncodeFloat(0.5f)); AssertTermEquals("over", filter, termAtt, payAtt, null); AssertTermEquals("the", filter, termAtt, payAtt, null); AssertTermEquals("lazy", filter, termAtt, payAtt, PayloadHelper.EncodeFloat(5.0f)); AssertTermEquals("brown", filter, termAtt, payAtt, PayloadHelper.EncodeFloat(99.3f)); AssertTermEquals("dogs", filter, termAtt, payAtt, PayloadHelper.EncodeFloat(83.7f)); assertFalse(filter.IncrementToken()); filter.End(); filter.Dispose(); }