public virtual void Test()
{
string test = "The quick red fox jumped over the lazy brown dogs";
NumericPayloadTokenFilter nptf = new NumericPayloadTokenFilter(new WordTokenFilter(this, new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false)), 3, "D");
bool seenDogs = false;
ICharTermAttribute termAtt = nptf.GetAttribute <ICharTermAttribute>();
ITypeAttribute typeAtt = nptf.GetAttribute <ITypeAttribute>();
IPayloadAttribute payloadAtt = nptf.GetAttribute <IPayloadAttribute>();
nptf.Reset();
while (nptf.IncrementToken())
{
if (termAtt.ToString().Equals("dogs", StringComparison.Ordinal))
{
seenDogs = true;
assertTrue(typeAtt.Type + " is not equal to " + "D", typeAtt.Type.Equals("D", StringComparison.Ordinal) == true);
assertTrue("payloadAtt.getPayload() is null and it shouldn't be", payloadAtt.Payload != null);
byte[] bytes = payloadAtt.Payload.Bytes; //safe here to just use the bytes, otherwise we should use offset, length
assertTrue(bytes.Length + " does not equal: " + payloadAtt.Payload.Length, bytes.Length == payloadAtt.Payload.Length);
assertTrue(payloadAtt.Payload.Offset + " does not equal: " + 0, payloadAtt.Payload.Offset == 0);
float pay = PayloadHelper.DecodeSingle(bytes);
assertTrue(pay + " does not equal: " + 3, pay == 3);
}
else
{
assertTrue(typeAtt.Type + " is not null and it should be", typeAtt.Type.Equals("word", StringComparison.Ordinal));
}
}
assertTrue(seenDogs + " does not equal: " + true, seenDogs == true);
}
public override BytesRef Encode(char[] buffer, int offset, int length)
{
float payload = float.Parse(new string(buffer, offset, length), CultureInfo.InvariantCulture); //TODO: improve this so that we don't have to new Strings
byte[] bytes = PayloadHelper.EncodeSingle(payload);
BytesRef result = new BytesRef(bytes);
return(result);
}
public override BytesRef Encode(char[] buffer, int offset, int length)
{
int payload = ArrayUtil.ParseInt32(buffer, offset, length); //TODO: improve this so that we don't have to new Strings
byte[] bytes = PayloadHelper.EncodeInt32(payload);
BytesRef result = new BytesRef(bytes);
return(result);
}
public override Payload Encode(char[] buffer, int offset, int length)
{
Payload result = new Payload();
float payload = Single.Parse(new string(buffer, offset, length)); // TODO: improve this so that we don't have to new Strings
byte[] bytes = PayloadHelper.EncodeFloat(payload);
result.SetData(bytes);
return(result);
}
public override Payload Encode(char[] buffer, int offset, int length)
{
Payload result = new Payload();
int payload = ArrayUtil.ParseInt(buffer, offset, length);//TODO: improve this so that we don't have to new Strings
byte[] bytes = PayloadHelper.EncodeInt(payload);
result.SetData(bytes);
return(result);
}
public NumericPayloadTokenFilter(TokenStream input, float payload, String typeMatch)
: base(input)
{
//Need to encode the payload
thePayload = new Payload(PayloadHelper.EncodeFloat(payload));
this.typeMatch = typeMatch;
payloadAtt = AddAttribute <IPayloadAttribute>();
typeAtt = AddAttribute <ITypeAttribute>();
}
public NumericPayloadTokenFilter(TokenStream input, float payload, string typeMatch) : base(input)
{
if (typeMatch == null)
{
throw new System.ArgumentException("typeMatch cannot be null");
}
//Need to encode the payload
thePayload = new BytesRef(PayloadHelper.EncodeFloat(payload));
this.typeMatch = typeMatch;
this.payloadAtt = AddAttribute <IPayloadAttribute>();
this.typeAtt = AddAttribute <ITypeAttribute>();
}
public NumericPayloadTokenFilter(TokenStream input, float payload, string typeMatch)
: base(input)
{
if (typeMatch is null)
{
throw new ArgumentNullException(nameof(typeMatch), "typeMatch cannot be null"); // LUCENENET specific - changed from IllegalArgumentException to ArgumentNullException (.NET convention)
}
//Need to encode the payload
thePayload = new BytesRef(PayloadHelper.EncodeSingle(payload));
this.typeMatch = typeMatch;
this.payloadAtt = AddAttribute <IPayloadAttribute>();
this.typeAtt = AddAttribute <ITypeAttribute>();
}
public override sealed bool IncrementToken()
{
if (m_input.IncrementToken())
{
byte[] data = new byte[8];
PayloadHelper.EncodeInt32(offsetAtt.StartOffset, data, 0);
PayloadHelper.EncodeInt32(offsetAtt.EndOffset, data, 4);
BytesRef payload = new BytesRef(data);
payAtt.Payload = payload;
return(true);
}
else
{
return(false);
}
}
public sealed override bool IncrementToken()
{
if (input.IncrementToken())
{
byte[] data = new byte[8];
PayloadHelper.EncodeInt(offsetAtt.StartOffset, data, 0);
PayloadHelper.EncodeInt(offsetAtt.EndOffset, data, 4);
Payload payload = new Payload(data);
payAtt.Payload = payload;
return(true);
}
else
{
return(false);
}
}
public virtual void TestDelim()
{
TextReader reader = new StringReader("the*0.1 quick*0.1 red*0.1");
TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
stream = TokenFilterFactory("DelimitedPayload", "encoder", "float", "delimiter", "*").Create(stream);
stream.Reset();
while (stream.IncrementToken())
{
IPayloadAttribute payAttr = stream.GetAttribute <IPayloadAttribute>();
assertNotNull(payAttr);
byte[] payData = payAttr.Payload.Bytes;
assertNotNull(payData);
float payFloat = PayloadHelper.DecodeSingle(payData);
assertEquals(0.1f, payFloat, 0.0f);
}
stream.End();
stream.Dispose();
}
public virtual void TestIntEncoding()
{
string test = "The quick|1 red|2 fox|3 jumped over the lazy|5 brown|99 dogs|83";
DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), '|', new IntegerEncoder());
ICharTermAttribute termAtt = filter.GetAttribute <ICharTermAttribute>();
IPayloadAttribute payAtt = filter.GetAttribute <IPayloadAttribute>();
filter.Reset();
AssertTermEquals("The", filter, termAtt, payAtt, null);
AssertTermEquals("quick", filter, termAtt, payAtt, PayloadHelper.EncodeInt32(1));
AssertTermEquals("red", filter, termAtt, payAtt, PayloadHelper.EncodeInt32(2));
AssertTermEquals("fox", filter, termAtt, payAtt, PayloadHelper.EncodeInt32(3));
AssertTermEquals("jumped", filter, termAtt, payAtt, null);
AssertTermEquals("over", filter, termAtt, payAtt, null);
AssertTermEquals("the", filter, termAtt, payAtt, null);
AssertTermEquals("lazy", filter, termAtt, payAtt, PayloadHelper.EncodeInt32(5));
AssertTermEquals("brown", filter, termAtt, payAtt, PayloadHelper.EncodeInt32(99));
AssertTermEquals("dogs", filter, termAtt, payAtt, PayloadHelper.EncodeInt32(83));
assertFalse(filter.IncrementToken());
filter.End();
filter.Dispose();
}
public virtual void TestFloatEncoding()
{
string test = "The quick|1.0 red|2.0 fox|3.5 jumped|0.5 over the lazy|5 brown|99.3 dogs|83.7";
DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), '|', new SingleEncoder());
ICharTermAttribute termAtt = filter.GetAttribute <ICharTermAttribute>();
IPayloadAttribute payAtt = filter.GetAttribute <IPayloadAttribute>();
filter.Reset();
AssertTermEquals("The", filter, termAtt, payAtt, null);
AssertTermEquals("quick", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(1.0f));
AssertTermEquals("red", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(2.0f));
AssertTermEquals("fox", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(3.5f));
AssertTermEquals("jumped", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(0.5f));
AssertTermEquals("over", filter, termAtt, payAtt, null);
AssertTermEquals("the", filter, termAtt, payAtt, null);
AssertTermEquals("lazy", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(5.0f));
AssertTermEquals("brown", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(99.3f));
AssertTermEquals("dogs", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(83.7f));
assertFalse(filter.IncrementToken());
filter.End();
filter.Dispose();
}
public virtual void Test()
{
string test = "The quick red fox jumped over the lazy brown dogs";
TokenOffsetPayloadTokenFilter nptf = new TokenOffsetPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
int count = 0;
IPayloadAttribute payloadAtt = nptf.GetAttribute <IPayloadAttribute>();
IOffsetAttribute offsetAtt = nptf.GetAttribute <IOffsetAttribute>();
nptf.Reset();
while (nptf.IncrementToken())
{
BytesRef pay = payloadAtt.Payload;
assertTrue("pay is null and it shouldn't be", pay != null);
byte[] data = pay.Bytes;
int start = PayloadHelper.DecodeInt32(data, 0);
assertTrue(start + " does not equal: " + offsetAtt.StartOffset, start == offsetAtt.StartOffset);
int end = PayloadHelper.DecodeInt32(data, 4);
assertTrue(end + " does not equal: " + offsetAtt.EndOffset, end == offsetAtt.EndOffset);
count++;
}
assertTrue(count + " does not equal: " + 10, count == 10);
}
