//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: public void test() throws java.io.IOException public virtual void test() { string test = "The quick red fox jumped over the lazy brown dogs"; NumericPayloadTokenFilter nptf = new NumericPayloadTokenFilter(new WordTokenFilter(this, new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false)), 3, "D"); bool seenDogs = false; CharTermAttribute termAtt = nptf.getAttribute(typeof(CharTermAttribute)); TypeAttribute typeAtt = nptf.getAttribute(typeof(TypeAttribute)); PayloadAttribute payloadAtt = nptf.getAttribute(typeof(PayloadAttribute)); nptf.reset(); while (nptf.incrementToken()) { if (termAtt.ToString().Equals("dogs")) { seenDogs = true; assertTrue(typeAtt.type() + " is not equal to " + "D", typeAtt.type().Equals("D") == true); assertTrue("payloadAtt.getPayload() is null and it shouldn't be", payloadAtt.Payload != null); sbyte[] bytes = payloadAtt.Payload.bytes; //safe here to just use the bytes, otherwise we should use offset, length assertTrue(bytes.Length + " does not equal: " + payloadAtt.Payload.length, bytes.Length == payloadAtt.Payload.length); assertTrue(payloadAtt.Payload.offset + " does not equal: " + 0, payloadAtt.Payload.offset == 0); float pay = PayloadHelper.decodeFloat(bytes); assertTrue(pay + " does not equal: " + 3, pay == 3); } else { assertTrue(typeAtt.type() + " is not null and it should be", typeAtt.type().Equals("word")); } } assertTrue(seenDogs + " does not equal: " + true, seenDogs == true); }
/** * Converts the original query string to a collection of Lucene Tokens. * @param original the original query string * @return a Collection of Lucene Tokens */ public override Collection /*<Token>*/ convert(string original) { if (original == null) // this can happen with q.alt = and no query { return(Collections.emptyList()); } Collection /*<Token>*/ result = new ArrayList/*<Token>*/ (); //TODO: Extract the words using a simple regex, but not query stuff, and then analyze them to produce the token stream Matcher matcher = QUERY_REGEX.matcher(original); TokenStream stream; while (matcher.find()) { string word = matcher.group(0); if (word.Equals("AND") == false && word.Equals("OR") == false) { try { stream = analyzer.reusableTokenStream("", new StringReader(word)); // TODO: support custom attributes TermAttribute termAtt = (TermAttribute)stream.addAttribute(typeof(TermAttribute)); FlagsAttribute flagsAtt = (FlagsAttribute)stream.addAttribute(typeof(FlagsAttribute)); TypeAttribute typeAtt = (TypeAttribute)stream.addAttribute(typeof(TypeAttribute)); PayloadAttribute payloadAtt = (PayloadAttribute)stream.addAttribute(typeof(PayloadAttribute)); PositionIncrementAttribute posIncAtt = (PositionIncrementAttribute)stream.addAttribute(typeof(PositionIncrementAttribute)); stream.reset(); while (stream.incrementToken()) { Token token = new Token(); token.setTermBuffer(termAtt.termBuffer(), 0, termAtt.termLength()); token.setStartOffset(matcher.start()); token.setEndOffset(matcher.end()); token.setFlags(flagsAtt.getFlags()); token.setType(typeAtt.type()); token.setPayload(payloadAtt.getPayload()); token.setPositionIncrement(posIncAtt.getPositionIncrement()); result.add(token); } } #pragma warning disable 168 catch (IOException e) { } #pragma warning restore 168 } } return(result); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: public void testDelim() throws Exception public virtual void testDelim() { Reader reader = new StringReader("the*0.1 quick*0.1 red*0.1"); TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); stream = tokenFilterFactory("DelimitedPayload", "encoder", "float", "delimiter", "*").create(stream); stream.reset(); while (stream.incrementToken()) { PayloadAttribute payAttr = stream.getAttribute(typeof(PayloadAttribute)); assertNotNull(payAttr); sbyte[] payData = payAttr.Payload.bytes; assertNotNull(payData); float payFloat = PayloadHelper.decodeFloat(payData); assertEquals(0.1f, payFloat, 0.0f); } stream.end(); stream.close(); }
public PrefixAwareTokenFilter(TokenStream prefix, TokenStream suffix) : base(suffix) { this.suffix = suffix; this.prefix = prefix; prefixExhausted = false; termAtt = addAttribute(typeof(CharTermAttribute)); posIncrAtt = addAttribute(typeof(PositionIncrementAttribute)); payloadAtt = addAttribute(typeof(PayloadAttribute)); offsetAtt = addAttribute(typeof(OffsetAttribute)); typeAtt = addAttribute(typeof(TypeAttribute)); flagsAtt = addAttribute(typeof(FlagsAttribute)); p_termAtt = prefix.addAttribute(typeof(CharTermAttribute)); p_posIncrAtt = prefix.addAttribute(typeof(PositionIncrementAttribute)); p_payloadAtt = prefix.addAttribute(typeof(PayloadAttribute)); p_offsetAtt = prefix.addAttribute(typeof(OffsetAttribute)); p_typeAtt = prefix.addAttribute(typeof(TypeAttribute)); p_flagsAtt = prefix.addAttribute(typeof(FlagsAttribute)); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: public void testFloatEncoding() throws Exception public virtual void testFloatEncoding() { string test = "The quick|1.0 red|2.0 fox|3.5 jumped|0.5 over the lazy|5 brown|99.3 dogs|83.7"; DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), '|', new FloatEncoder()); CharTermAttribute termAtt = filter.getAttribute(typeof(CharTermAttribute)); PayloadAttribute payAtt = filter.getAttribute(typeof(PayloadAttribute)); filter.reset(); assertTermEquals("The", filter, termAtt, payAtt, null); assertTermEquals("quick", filter, termAtt, payAtt, PayloadHelper.encodeFloat(1.0f)); assertTermEquals("red", filter, termAtt, payAtt, PayloadHelper.encodeFloat(2.0f)); assertTermEquals("fox", filter, termAtt, payAtt, PayloadHelper.encodeFloat(3.5f)); assertTermEquals("jumped", filter, termAtt, payAtt, PayloadHelper.encodeFloat(0.5f)); assertTermEquals("over", filter, termAtt, payAtt, null); assertTermEquals("the", filter, termAtt, payAtt, null); assertTermEquals("lazy", filter, termAtt, payAtt, PayloadHelper.encodeFloat(5.0f)); assertTermEquals("brown", filter, termAtt, payAtt, PayloadHelper.encodeFloat(99.3f)); assertTermEquals("dogs", filter, termAtt, payAtt, PayloadHelper.encodeFloat(83.7f)); assertFalse(filter.incrementToken()); filter.end(); filter.close(); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: public void testPayloads() throws Exception public virtual void testPayloads() { string test = "The quick|JJ red|JJ fox|NN jumped|VB over the lazy|JJ brown|JJ dogs|NN"; DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), DelimitedPayloadTokenFilter.DEFAULT_DELIMITER, new IdentityEncoder()); CharTermAttribute termAtt = filter.getAttribute(typeof(CharTermAttribute)); PayloadAttribute payAtt = filter.getAttribute(typeof(PayloadAttribute)); filter.reset(); assertTermEquals("The", filter, termAtt, payAtt, null); assertTermEquals("quick", filter, termAtt, payAtt, "JJ".GetBytes(StandardCharsets.UTF_8)); assertTermEquals("red", filter, termAtt, payAtt, "JJ".GetBytes(StandardCharsets.UTF_8)); assertTermEquals("fox", filter, termAtt, payAtt, "NN".GetBytes(StandardCharsets.UTF_8)); assertTermEquals("jumped", filter, termAtt, payAtt, "VB".GetBytes(StandardCharsets.UTF_8)); assertTermEquals("over", filter, termAtt, payAtt, null); assertTermEquals("the", filter, termAtt, payAtt, null); assertTermEquals("lazy", filter, termAtt, payAtt, "JJ".GetBytes(StandardCharsets.UTF_8)); assertTermEquals("brown", filter, termAtt, payAtt, "JJ".GetBytes(StandardCharsets.UTF_8)); assertTermEquals("dogs", filter, termAtt, payAtt, "NN".GetBytes(StandardCharsets.UTF_8)); assertFalse(filter.incrementToken()); filter.end(); filter.close(); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: void assertTermEquals(String expected, org.apache.lucene.analysis.TokenStream stream, byte[] expectPay) throws Exception internal virtual void assertTermEquals(string expected, TokenStream stream, sbyte[] expectPay) { CharTermAttribute termAtt = stream.getAttribute(typeof(CharTermAttribute)); PayloadAttribute payloadAtt = stream.getAttribute(typeof(PayloadAttribute)); assertTrue(stream.incrementToken()); assertEquals(expected, termAtt.ToString()); BytesRef payload = payloadAtt.Payload; if (payload != null) { assertTrue(payload.length + " does not equal: " + expectPay.Length, payload.length == expectPay.Length); for (int i = 0; i < expectPay.Length; i++) { assertTrue(expectPay[i] + " does not equal: " + payload.bytes[i + payload.offset], expectPay[i] == payload.bytes[i + payload.offset]); } } else { assertTrue("expectPay is not null and it should be", expectPay == null); } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: public void testIntEncoding() throws Exception public virtual void testIntEncoding() { string test = "The quick|1 red|2 fox|3 jumped over the lazy|5 brown|99 dogs|83"; DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), '|', new IntegerEncoder()); CharTermAttribute termAtt = filter.getAttribute(typeof(CharTermAttribute)); PayloadAttribute payAtt = filter.getAttribute(typeof(PayloadAttribute)); filter.reset(); assertTermEquals("The", filter, termAtt, payAtt, null); assertTermEquals("quick", filter, termAtt, payAtt, PayloadHelper.encodeInt(1)); assertTermEquals("red", filter, termAtt, payAtt, PayloadHelper.encodeInt(2)); assertTermEquals("fox", filter, termAtt, payAtt, PayloadHelper.encodeInt(3)); assertTermEquals("jumped", filter, termAtt, payAtt, null); assertTermEquals("over", filter, termAtt, payAtt, null); assertTermEquals("the", filter, termAtt, payAtt, null); assertTermEquals("lazy", filter, termAtt, payAtt, PayloadHelper.encodeInt(5)); assertTermEquals("brown", filter, termAtt, payAtt, PayloadHelper.encodeInt(99)); assertTermEquals("dogs", filter, termAtt, payAtt, PayloadHelper.encodeInt(83)); assertFalse(filter.incrementToken()); filter.end(); filter.close(); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: public void test() throws java.io.IOException public virtual void test() { string test = "The quick red fox jumped over the lazy brown dogs"; TypeAsPayloadTokenFilter nptf = new TypeAsPayloadTokenFilter(new WordTokenFilter(this, new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false))); int count = 0; CharTermAttribute termAtt = nptf.getAttribute(typeof(CharTermAttribute)); TypeAttribute typeAtt = nptf.getAttribute(typeof(TypeAttribute)); PayloadAttribute payloadAtt = nptf.getAttribute(typeof(PayloadAttribute)); nptf.reset(); while (nptf.incrementToken()) { assertTrue(typeAtt.type() + " is not null and it should be", typeAtt.type().Equals(char.ToUpper(termAtt.buffer()[0]).ToString())); assertTrue("nextToken.getPayload() is null and it shouldn't be", payloadAtt.Payload != null); string type = payloadAtt.Payload.utf8ToString(); assertTrue(type + " is not equal to " + typeAtt.type(), type.Equals(typeAtt.type())); count++; } assertTrue(count + " does not equal: " + 10, count == 10); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: public void test() throws java.io.IOException public virtual void test() { string test = "The quick red fox jumped over the lazy brown dogs"; TokenOffsetPayloadTokenFilter nptf = new TokenOffsetPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false)); int count = 0; PayloadAttribute payloadAtt = nptf.getAttribute(typeof(PayloadAttribute)); OffsetAttribute offsetAtt = nptf.getAttribute(typeof(OffsetAttribute)); nptf.reset(); while (nptf.incrementToken()) { BytesRef pay = payloadAtt.Payload; assertTrue("pay is null and it shouldn't be", pay != null); sbyte[] data = pay.bytes; int start = PayloadHelper.decodeInt(data, 0); assertTrue(start + " does not equal: " + offsetAtt.startOffset(), start == offsetAtt.startOffset()); int end = PayloadHelper.decodeInt(data, 4); assertTrue(end + " does not equal: " + offsetAtt.endOffset(), end == offsetAtt.endOffset()); count++; } assertTrue(count + " does not equal: " + 10, count == 10); }
public PayloadSetter(TokenStream input) : base(input) { if (!InstanceFieldsInitialized) { InitializeInstanceFields(); InstanceFieldsInitialized = true; } payloadAtt = addAttribute(typeof(PayloadAttribute)); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: void assertTermEquals(String expected, org.apache.lucene.analysis.TokenStream stream, org.apache.lucene.analysis.tokenattributes.CharTermAttribute termAtt, org.apache.lucene.analysis.tokenattributes.PayloadAttribute payAtt, byte[] expectPay) throws Exception internal virtual void assertTermEquals(string expected, TokenStream stream, CharTermAttribute termAtt, PayloadAttribute payAtt, sbyte[] expectPay) { assertTrue(stream.incrementToken()); assertEquals(expected, termAtt.ToString()); BytesRef payload = payAtt.Payload; if (payload != null) { assertTrue(payload.length + " does not equal: " + expectPay.Length, payload.length == expectPay.Length); for (int i = 0; i < expectPay.Length; i++) { assertTrue(expectPay[i] + " does not equal: " + payload.bytes[i + payload.offset], expectPay[i] == payload.bytes[i + payload.offset]); } } else { assertTrue("expectPay is not null and it should be", expectPay == null); } }