Utility methods for encoding payloads.
        public virtual void Test()
        {
            string test = "The quick red fox jumped over the lazy brown dogs";

            NumericPayloadTokenFilter nptf = new NumericPayloadTokenFilter(new WordTokenFilter(this, new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false)), 3, "D");
            bool seenDogs = false;
            ICharTermAttribute termAtt    = nptf.GetAttribute <ICharTermAttribute>();
            ITypeAttribute     typeAtt    = nptf.GetAttribute <ITypeAttribute>();
            IPayloadAttribute  payloadAtt = nptf.GetAttribute <IPayloadAttribute>();

            nptf.Reset();
            while (nptf.IncrementToken())
            {
                if (termAtt.ToString().Equals("dogs", StringComparison.Ordinal))
                {
                    seenDogs = true;
                    assertTrue(typeAtt.Type + " is not equal to " + "D", typeAtt.Type.Equals("D", StringComparison.Ordinal) == true);
                    assertTrue("payloadAtt.getPayload() is null and it shouldn't be", payloadAtt.Payload != null);
                    byte[] bytes = payloadAtt.Payload.Bytes; //safe here to just use the bytes, otherwise we should use offset, length
                    assertTrue(bytes.Length + " does not equal: " + payloadAtt.Payload.Length, bytes.Length == payloadAtt.Payload.Length);
                    assertTrue(payloadAtt.Payload.Offset + " does not equal: " + 0, payloadAtt.Payload.Offset == 0);
                    float pay = PayloadHelper.DecodeSingle(bytes);
                    assertTrue(pay + " does not equal: " + 3, pay == 3);
                }
                else
                {
                    assertTrue(typeAtt.Type + " is not null and it should be", typeAtt.Type.Equals("word", StringComparison.Ordinal));
                }
            }
            assertTrue(seenDogs + " does not equal: " + true, seenDogs == true);
        }
Пример #2
0
        public override BytesRef Encode(char[] buffer, int offset, int length)
        {
            float payload = float.Parse(new string(buffer, offset, length), CultureInfo.InvariantCulture); //TODO: improve this so that we don't have to new Strings

            byte[]   bytes  = PayloadHelper.EncodeSingle(payload);
            BytesRef result = new BytesRef(bytes);

            return(result);
        }
Пример #3
0
        public override BytesRef Encode(char[] buffer, int offset, int length)
        {
            int payload = ArrayUtil.ParseInt32(buffer, offset, length); //TODO: improve this so that we don't have to new Strings

            byte[]   bytes  = PayloadHelper.EncodeInt32(payload);
            BytesRef result = new BytesRef(bytes);

            return(result);
        }
Пример #4
0
        public override Payload Encode(char[] buffer, int offset, int length)
        {
            Payload result  = new Payload();
            float   payload = Single.Parse(new string(buffer, offset, length)); // TODO: improve this so that we don't have to new Strings

            byte[] bytes = PayloadHelper.EncodeFloat(payload);
            result.SetData(bytes);
            return(result);
        }
Пример #5
0
        public override Payload Encode(char[] buffer, int offset, int length)
        {
            Payload result  = new Payload();
            int     payload = ArrayUtil.ParseInt(buffer, offset, length);//TODO: improve this so that we don't have to new Strings

            byte[] bytes = PayloadHelper.EncodeInt(payload);
            result.SetData(bytes);
            return(result);
        }
 public NumericPayloadTokenFilter(TokenStream input, float payload, String typeMatch)
     : base(input)
 {
     //Need to encode the payload
     thePayload     = new Payload(PayloadHelper.EncodeFloat(payload));
     this.typeMatch = typeMatch;
     payloadAtt     = AddAttribute <IPayloadAttribute>();
     typeAtt        = AddAttribute <ITypeAttribute>();
 }
Пример #7
0
 public NumericPayloadTokenFilter(TokenStream input, float payload, string typeMatch) : base(input)
 {
     if (typeMatch == null)
     {
         throw new System.ArgumentException("typeMatch cannot be null");
     }
     //Need to encode the payload
     thePayload      = new BytesRef(PayloadHelper.EncodeFloat(payload));
     this.typeMatch  = typeMatch;
     this.payloadAtt = AddAttribute <IPayloadAttribute>();
     this.typeAtt    = AddAttribute <ITypeAttribute>();
 }
Пример #8
0
 public NumericPayloadTokenFilter(TokenStream input, float payload, string typeMatch)
     : base(input)
 {
     if (typeMatch is null)
     {
         throw new ArgumentNullException(nameof(typeMatch), "typeMatch cannot be null"); // LUCENENET specific - changed from IllegalArgumentException to ArgumentNullException (.NET convention)
     }
     //Need to encode the payload
     thePayload      = new BytesRef(PayloadHelper.EncodeSingle(payload));
     this.typeMatch  = typeMatch;
     this.payloadAtt = AddAttribute <IPayloadAttribute>();
     this.typeAtt    = AddAttribute <ITypeAttribute>();
 }
 public override sealed bool IncrementToken()
 {
     if (m_input.IncrementToken())
     {
         byte[] data = new byte[8];
         PayloadHelper.EncodeInt32(offsetAtt.StartOffset, data, 0);
         PayloadHelper.EncodeInt32(offsetAtt.EndOffset, data, 4);
         BytesRef payload = new BytesRef(data);
         payAtt.Payload = payload;
         return(true);
     }
     else
     {
         return(false);
     }
 }
 public sealed override bool IncrementToken()
 {
     if (input.IncrementToken())
     {
         byte[] data = new byte[8];
         PayloadHelper.EncodeInt(offsetAtt.StartOffset, data, 0);
         PayloadHelper.EncodeInt(offsetAtt.EndOffset, data, 4);
         Payload payload = new Payload(data);
         payAtt.Payload = payload;
         return(true);
     }
     else
     {
         return(false);
     }
 }
Пример #11
0
        public virtual void TestDelim()
        {
            TextReader  reader = new StringReader("the*0.1 quick*0.1 red*0.1");
            TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);

            stream = TokenFilterFactory("DelimitedPayload", "encoder", "float", "delimiter", "*").Create(stream);
            stream.Reset();
            while (stream.IncrementToken())
            {
                IPayloadAttribute payAttr = stream.GetAttribute <IPayloadAttribute>();
                assertNotNull(payAttr);
                byte[] payData = payAttr.Payload.Bytes;
                assertNotNull(payData);
                float payFloat = PayloadHelper.DecodeSingle(payData);
                assertEquals(0.1f, payFloat, 0.0f);
            }
            stream.End();
            stream.Dispose();
        }
Пример #12
0
        public virtual void TestIntEncoding()
        {
            string test = "The quick|1 red|2 fox|3 jumped over the lazy|5 brown|99 dogs|83";
            DelimitedPayloadTokenFilter filter  = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), '|', new IntegerEncoder());
            ICharTermAttribute          termAtt = filter.GetAttribute <ICharTermAttribute>();
            IPayloadAttribute           payAtt  = filter.GetAttribute <IPayloadAttribute>();

            filter.Reset();
            AssertTermEquals("The", filter, termAtt, payAtt, null);
            AssertTermEquals("quick", filter, termAtt, payAtt, PayloadHelper.EncodeInt32(1));
            AssertTermEquals("red", filter, termAtt, payAtt, PayloadHelper.EncodeInt32(2));
            AssertTermEquals("fox", filter, termAtt, payAtt, PayloadHelper.EncodeInt32(3));
            AssertTermEquals("jumped", filter, termAtt, payAtt, null);
            AssertTermEquals("over", filter, termAtt, payAtt, null);
            AssertTermEquals("the", filter, termAtt, payAtt, null);
            AssertTermEquals("lazy", filter, termAtt, payAtt, PayloadHelper.EncodeInt32(5));
            AssertTermEquals("brown", filter, termAtt, payAtt, PayloadHelper.EncodeInt32(99));
            AssertTermEquals("dogs", filter, termAtt, payAtt, PayloadHelper.EncodeInt32(83));
            assertFalse(filter.IncrementToken());
            filter.End();
            filter.Dispose();
        }
Пример #13
0
        public virtual void TestFloatEncoding()
        {
            string test = "The quick|1.0 red|2.0 fox|3.5 jumped|0.5 over the lazy|5 brown|99.3 dogs|83.7";
            DelimitedPayloadTokenFilter filter  = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), '|', new SingleEncoder());
            ICharTermAttribute          termAtt = filter.GetAttribute <ICharTermAttribute>();
            IPayloadAttribute           payAtt  = filter.GetAttribute <IPayloadAttribute>();

            filter.Reset();
            AssertTermEquals("The", filter, termAtt, payAtt, null);
            AssertTermEquals("quick", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(1.0f));
            AssertTermEquals("red", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(2.0f));
            AssertTermEquals("fox", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(3.5f));
            AssertTermEquals("jumped", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(0.5f));
            AssertTermEquals("over", filter, termAtt, payAtt, null);
            AssertTermEquals("the", filter, termAtt, payAtt, null);
            AssertTermEquals("lazy", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(5.0f));
            AssertTermEquals("brown", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(99.3f));
            AssertTermEquals("dogs", filter, termAtt, payAtt, PayloadHelper.EncodeSingle(83.7f));
            assertFalse(filter.IncrementToken());
            filter.End();
            filter.Dispose();
        }
Пример #14
0
        public virtual void Test()
        {
            string test = "The quick red fox jumped over the lazy brown dogs";

            TokenOffsetPayloadTokenFilter nptf = new TokenOffsetPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
            int count = 0;
            IPayloadAttribute payloadAtt = nptf.GetAttribute <IPayloadAttribute>();
            IOffsetAttribute  offsetAtt  = nptf.GetAttribute <IOffsetAttribute>();

            nptf.Reset();
            while (nptf.IncrementToken())
            {
                BytesRef pay = payloadAtt.Payload;
                assertTrue("pay is null and it shouldn't be", pay != null);
                byte[] data  = pay.Bytes;
                int    start = PayloadHelper.DecodeInt32(data, 0);
                assertTrue(start + " does not equal: " + offsetAtt.StartOffset, start == offsetAtt.StartOffset);
                int end = PayloadHelper.DecodeInt32(data, 4);
                assertTrue(end + " does not equal: " + offsetAtt.EndOffset, end == offsetAtt.EndOffset);
                count++;
            }
            assertTrue(count + " does not equal: " + 10, count == 10);
        }