//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void test() throws java.io.IOException
public virtual void test()
{
string test = "The quick red fox jumped over the lazy brown dogs";
NumericPayloadTokenFilter nptf = new NumericPayloadTokenFilter(new WordTokenFilter(this, new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false)), 3, "D");
bool seenDogs = false;
CharTermAttribute termAtt = nptf.getAttribute(typeof(CharTermAttribute));
TypeAttribute typeAtt = nptf.getAttribute(typeof(TypeAttribute));
PayloadAttribute payloadAtt = nptf.getAttribute(typeof(PayloadAttribute));
nptf.reset();
while (nptf.incrementToken())
{
if (termAtt.ToString().Equals("dogs"))
{
seenDogs = true;
assertTrue(typeAtt.type() + " is not equal to " + "D", typeAtt.type().Equals("D") == true);
assertTrue("payloadAtt.getPayload() is null and it shouldn't be", payloadAtt.Payload != null);
sbyte[] bytes = payloadAtt.Payload.bytes; //safe here to just use the bytes, otherwise we should use offset, length
assertTrue(bytes.Length + " does not equal: " + payloadAtt.Payload.length, bytes.Length == payloadAtt.Payload.length);
assertTrue(payloadAtt.Payload.offset + " does not equal: " + 0, payloadAtt.Payload.offset == 0);
float pay = PayloadHelper.decodeFloat(bytes);
assertTrue(pay + " does not equal: " + 3, pay == 3);
}
else
{
assertTrue(typeAtt.type() + " is not null and it should be", typeAtt.type().Equals("word"));
}
}
assertTrue(seenDogs + " does not equal: " + true, seenDogs == true);
}
/**
* Converts the original query string to a collection of Lucene Tokens.
* @param original the original query string
* @return a Collection of Lucene Tokens
*/
public override Collection /*<Token>*/ convert(string original)
{
if (original == null) // this can happen with q.alt = and no query
{
return(Collections.emptyList());
}
Collection /*<Token>*/ result = new ArrayList/*<Token>*/ ();
//TODO: Extract the words using a simple regex, but not query stuff, and then analyze them to produce the token stream
Matcher matcher = QUERY_REGEX.matcher(original);
TokenStream stream;
while (matcher.find())
{
string word = matcher.group(0);
if (word.Equals("AND") == false && word.Equals("OR") == false)
{
try {
stream = analyzer.reusableTokenStream("", new StringReader(word));
// TODO: support custom attributes
TermAttribute termAtt = (TermAttribute)stream.addAttribute(typeof(TermAttribute));
FlagsAttribute flagsAtt = (FlagsAttribute)stream.addAttribute(typeof(FlagsAttribute));
TypeAttribute typeAtt = (TypeAttribute)stream.addAttribute(typeof(TypeAttribute));
PayloadAttribute payloadAtt = (PayloadAttribute)stream.addAttribute(typeof(PayloadAttribute));
PositionIncrementAttribute posIncAtt = (PositionIncrementAttribute)stream.addAttribute(typeof(PositionIncrementAttribute));
stream.reset();
while (stream.incrementToken())
{
Token token = new Token();
token.setTermBuffer(termAtt.termBuffer(), 0, termAtt.termLength());
token.setStartOffset(matcher.start());
token.setEndOffset(matcher.end());
token.setFlags(flagsAtt.getFlags());
token.setType(typeAtt.type());
token.setPayload(payloadAtt.getPayload());
token.setPositionIncrement(posIncAtt.getPositionIncrement());
result.add(token);
}
}
#pragma warning disable 168
catch (IOException e)
{
}
#pragma warning restore 168
}
}
return(result);
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testDelim() throws Exception
public virtual void testDelim()
{
Reader reader = new StringReader("the*0.1 quick*0.1 red*0.1");
TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
stream = tokenFilterFactory("DelimitedPayload", "encoder", "float", "delimiter", "*").create(stream);
stream.reset();
while (stream.incrementToken())
{
PayloadAttribute payAttr = stream.getAttribute(typeof(PayloadAttribute));
assertNotNull(payAttr);
sbyte[] payData = payAttr.Payload.bytes;
assertNotNull(payData);
float payFloat = PayloadHelper.decodeFloat(payData);
assertEquals(0.1f, payFloat, 0.0f);
}
stream.end();
stream.close();
}
public PrefixAwareTokenFilter(TokenStream prefix, TokenStream suffix) : base(suffix)
{
this.suffix = suffix;
this.prefix = prefix;
prefixExhausted = false;
termAtt = addAttribute(typeof(CharTermAttribute));
posIncrAtt = addAttribute(typeof(PositionIncrementAttribute));
payloadAtt = addAttribute(typeof(PayloadAttribute));
offsetAtt = addAttribute(typeof(OffsetAttribute));
typeAtt = addAttribute(typeof(TypeAttribute));
flagsAtt = addAttribute(typeof(FlagsAttribute));
p_termAtt = prefix.addAttribute(typeof(CharTermAttribute));
p_posIncrAtt = prefix.addAttribute(typeof(PositionIncrementAttribute));
p_payloadAtt = prefix.addAttribute(typeof(PayloadAttribute));
p_offsetAtt = prefix.addAttribute(typeof(OffsetAttribute));
p_typeAtt = prefix.addAttribute(typeof(TypeAttribute));
p_flagsAtt = prefix.addAttribute(typeof(FlagsAttribute));
}
public PrefixAwareTokenFilter(TokenStream prefix, TokenStream suffix) : base(suffix)
{
this.suffix = suffix;
this.prefix = prefix;
prefixExhausted = false;
termAtt = addAttribute(typeof(CharTermAttribute));
posIncrAtt = addAttribute(typeof(PositionIncrementAttribute));
payloadAtt = addAttribute(typeof(PayloadAttribute));
offsetAtt = addAttribute(typeof(OffsetAttribute));
typeAtt = addAttribute(typeof(TypeAttribute));
flagsAtt = addAttribute(typeof(FlagsAttribute));
p_termAtt = prefix.addAttribute(typeof(CharTermAttribute));
p_posIncrAtt = prefix.addAttribute(typeof(PositionIncrementAttribute));
p_payloadAtt = prefix.addAttribute(typeof(PayloadAttribute));
p_offsetAtt = prefix.addAttribute(typeof(OffsetAttribute));
p_typeAtt = prefix.addAttribute(typeof(TypeAttribute));
p_flagsAtt = prefix.addAttribute(typeof(FlagsAttribute));
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testFloatEncoding() throws Exception
public virtual void testFloatEncoding()
{
string test = "The quick|1.0 red|2.0 fox|3.5 jumped|0.5 over the lazy|5 brown|99.3 dogs|83.7";
DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), '|', new FloatEncoder());
CharTermAttribute termAtt = filter.getAttribute(typeof(CharTermAttribute));
PayloadAttribute payAtt = filter.getAttribute(typeof(PayloadAttribute));
filter.reset();
assertTermEquals("The", filter, termAtt, payAtt, null);
assertTermEquals("quick", filter, termAtt, payAtt, PayloadHelper.encodeFloat(1.0f));
assertTermEquals("red", filter, termAtt, payAtt, PayloadHelper.encodeFloat(2.0f));
assertTermEquals("fox", filter, termAtt, payAtt, PayloadHelper.encodeFloat(3.5f));
assertTermEquals("jumped", filter, termAtt, payAtt, PayloadHelper.encodeFloat(0.5f));
assertTermEquals("over", filter, termAtt, payAtt, null);
assertTermEquals("the", filter, termAtt, payAtt, null);
assertTermEquals("lazy", filter, termAtt, payAtt, PayloadHelper.encodeFloat(5.0f));
assertTermEquals("brown", filter, termAtt, payAtt, PayloadHelper.encodeFloat(99.3f));
assertTermEquals("dogs", filter, termAtt, payAtt, PayloadHelper.encodeFloat(83.7f));
assertFalse(filter.incrementToken());
filter.end();
filter.close();
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testPayloads() throws Exception
public virtual void testPayloads()
{
string test = "The quick|JJ red|JJ fox|NN jumped|VB over the lazy|JJ brown|JJ dogs|NN";
DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), DelimitedPayloadTokenFilter.DEFAULT_DELIMITER, new IdentityEncoder());
CharTermAttribute termAtt = filter.getAttribute(typeof(CharTermAttribute));
PayloadAttribute payAtt = filter.getAttribute(typeof(PayloadAttribute));
filter.reset();
assertTermEquals("The", filter, termAtt, payAtt, null);
assertTermEquals("quick", filter, termAtt, payAtt, "JJ".GetBytes(StandardCharsets.UTF_8));
assertTermEquals("red", filter, termAtt, payAtt, "JJ".GetBytes(StandardCharsets.UTF_8));
assertTermEquals("fox", filter, termAtt, payAtt, "NN".GetBytes(StandardCharsets.UTF_8));
assertTermEquals("jumped", filter, termAtt, payAtt, "VB".GetBytes(StandardCharsets.UTF_8));
assertTermEquals("over", filter, termAtt, payAtt, null);
assertTermEquals("the", filter, termAtt, payAtt, null);
assertTermEquals("lazy", filter, termAtt, payAtt, "JJ".GetBytes(StandardCharsets.UTF_8));
assertTermEquals("brown", filter, termAtt, payAtt, "JJ".GetBytes(StandardCharsets.UTF_8));
assertTermEquals("dogs", filter, termAtt, payAtt, "NN".GetBytes(StandardCharsets.UTF_8));
assertFalse(filter.incrementToken());
filter.end();
filter.close();
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: void assertTermEquals(String expected, org.apache.lucene.analysis.TokenStream stream, byte[] expectPay) throws Exception
internal virtual void assertTermEquals(string expected, TokenStream stream, sbyte[] expectPay)
{
CharTermAttribute termAtt = stream.getAttribute(typeof(CharTermAttribute));
PayloadAttribute payloadAtt = stream.getAttribute(typeof(PayloadAttribute));
assertTrue(stream.incrementToken());
assertEquals(expected, termAtt.ToString());
BytesRef payload = payloadAtt.Payload;
if (payload != null)
{
assertTrue(payload.length + " does not equal: " + expectPay.Length, payload.length == expectPay.Length);
for (int i = 0; i < expectPay.Length; i++)
{
assertTrue(expectPay[i] + " does not equal: " + payload.bytes[i + payload.offset], expectPay[i] == payload.bytes[i + payload.offset]);
}
}
else
{
assertTrue("expectPay is not null and it should be", expectPay == null);
}
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testIntEncoding() throws Exception
public virtual void testIntEncoding()
{
string test = "The quick|1 red|2 fox|3 jumped over the lazy|5 brown|99 dogs|83";
DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), '|', new IntegerEncoder());
CharTermAttribute termAtt = filter.getAttribute(typeof(CharTermAttribute));
PayloadAttribute payAtt = filter.getAttribute(typeof(PayloadAttribute));
filter.reset();
assertTermEquals("The", filter, termAtt, payAtt, null);
assertTermEquals("quick", filter, termAtt, payAtt, PayloadHelper.encodeInt(1));
assertTermEquals("red", filter, termAtt, payAtt, PayloadHelper.encodeInt(2));
assertTermEquals("fox", filter, termAtt, payAtt, PayloadHelper.encodeInt(3));
assertTermEquals("jumped", filter, termAtt, payAtt, null);
assertTermEquals("over", filter, termAtt, payAtt, null);
assertTermEquals("the", filter, termAtt, payAtt, null);
assertTermEquals("lazy", filter, termAtt, payAtt, PayloadHelper.encodeInt(5));
assertTermEquals("brown", filter, termAtt, payAtt, PayloadHelper.encodeInt(99));
assertTermEquals("dogs", filter, termAtt, payAtt, PayloadHelper.encodeInt(83));
assertFalse(filter.incrementToken());
filter.end();
filter.close();
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void test() throws java.io.IOException
public virtual void test()
{
string test = "The quick red fox jumped over the lazy brown dogs";
TypeAsPayloadTokenFilter nptf = new TypeAsPayloadTokenFilter(new WordTokenFilter(this, new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false)));
int count = 0;
CharTermAttribute termAtt = nptf.getAttribute(typeof(CharTermAttribute));
TypeAttribute typeAtt = nptf.getAttribute(typeof(TypeAttribute));
PayloadAttribute payloadAtt = nptf.getAttribute(typeof(PayloadAttribute));
nptf.reset();
while (nptf.incrementToken())
{
assertTrue(typeAtt.type() + " is not null and it should be", typeAtt.type().Equals(char.ToUpper(termAtt.buffer()[0]).ToString()));
assertTrue("nextToken.getPayload() is null and it shouldn't be", payloadAtt.Payload != null);
string type = payloadAtt.Payload.utf8ToString();
assertTrue(type + " is not equal to " + typeAtt.type(), type.Equals(typeAtt.type()));
count++;
}
assertTrue(count + " does not equal: " + 10, count == 10);
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void test() throws java.io.IOException
public virtual void test()
{
string test = "The quick red fox jumped over the lazy brown dogs";
TokenOffsetPayloadTokenFilter nptf = new TokenOffsetPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
int count = 0;
PayloadAttribute payloadAtt = nptf.getAttribute(typeof(PayloadAttribute));
OffsetAttribute offsetAtt = nptf.getAttribute(typeof(OffsetAttribute));
nptf.reset();
while (nptf.incrementToken())
{
BytesRef pay = payloadAtt.Payload;
assertTrue("pay is null and it shouldn't be", pay != null);
sbyte[] data = pay.bytes;
int start = PayloadHelper.decodeInt(data, 0);
assertTrue(start + " does not equal: " + offsetAtt.startOffset(), start == offsetAtt.startOffset());
int end = PayloadHelper.decodeInt(data, 4);
assertTrue(end + " does not equal: " + offsetAtt.endOffset(), end == offsetAtt.endOffset());
count++;
}
assertTrue(count + " does not equal: " + 10, count == 10);
}
public PayloadSetter(TokenStream input)
: base(input)
{
if (!InstanceFieldsInitialized)
{
InitializeInstanceFields();
InstanceFieldsInitialized = true;
}
payloadAtt = addAttribute(typeof(PayloadAttribute));
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: void assertTermEquals(String expected, org.apache.lucene.analysis.TokenStream stream, org.apache.lucene.analysis.tokenattributes.CharTermAttribute termAtt, org.apache.lucene.analysis.tokenattributes.PayloadAttribute payAtt, byte[] expectPay) throws Exception
internal virtual void assertTermEquals(string expected, TokenStream stream, CharTermAttribute termAtt, PayloadAttribute payAtt, sbyte[] expectPay)
{
assertTrue(stream.incrementToken());
assertEquals(expected, termAtt.ToString());
BytesRef payload = payAtt.Payload;
if (payload != null)
{
assertTrue(payload.length + " does not equal: " + expectPay.Length, payload.length == expectPay.Length);
for (int i = 0; i < expectPay.Length; i++)
{
assertTrue(expectPay[i] + " does not equal: " + payload.bytes[i + payload.offset], expectPay[i] == payload.bytes[i + payload.offset]);
}
}
else
{
assertTrue("expectPay is not null and it should be", expectPay == null);
}
}
