/// <summary>Returns the next token in the stream, or null at EOS. /// The returned Token is a "full private copy" (not /// re-used across calls to next()) but will be slower /// than calling {@link #Next(Token)} instead.. /// </summary> public virtual Token Next() { Token result = Next(new Token()); if (result != null) { Payload p = result.GetPayload(); if (p != null) { result.SetPayload((Payload)p.Clone()); } } return(result); }
/// <summary>Returns the next token in the stream, or null at EOS. /// @deprecated The returned Token is a "full private copy" (not /// re-used across calls to next()) but will be slower /// than calling {@link #Next(Token)} instead.. /// </summary> public virtual Token Next() { Token reusableToken = new Token(); Token nextToken = Next(reusableToken); if (nextToken != null) { Payload p = nextToken.GetPayload(); if (p != null) { nextToken.SetPayload((Payload)p.Clone()); } } return(nextToken); }
public virtual void TestClone() { Token t = new Token(0, 5); char[] content = "hello".ToCharArray(); t.SetTermBuffer(content, 0, 5); char[] buf = t.TermBuffer(); Token copy = (Token)TestSimpleAttributeImpls.AssertCloneIsEqual(t); Assert.AreEqual(t.Term(), copy.Term()); Assert.AreNotSame(buf, copy.TermBuffer()); Payload pl = new Payload(new byte[] { 1, 2, 3, 4 }); t.SetPayload(pl); copy = (Token)TestSimpleAttributeImpls.AssertCloneIsEqual(t); Assert.AreEqual(pl, copy.GetPayload()); Assert.AreNotSame(pl, copy.GetPayload()); }
public virtual object Clone() { try { Token t = (Token)base.MemberwiseClone(); // Do a deep clone if (termBuffer != null) { t.termBuffer = (char[])termBuffer.Clone(); } if (payload != null) { t.SetPayload((Payload)payload.Clone()); } return(t); } catch (System.Exception e) { throw new System.SystemException("", e); // shouldn't happen } }
public virtual System.Object Clone() { try { Token t = (Token)base.MemberwiseClone(); if (termBuffer != null) { t.termBuffer = null; t.SetTermBuffer(termBuffer, 0, termLength); } if (payload != null) { t.SetPayload((Payload)payload.Clone()); } return(t); } catch (System.Exception e) { throw new System.SystemException("", e); // shouldn't happen } }
private Token GetNextInputToken(Token token) { if (!_input.IncrementToken()) return null; token.SetTermBuffer(_inTermAtt.TermBuffer(), 0, _inTermAtt.TermLength()); token.SetPositionIncrement(_inPosIncrAtt.GetPositionIncrement()); token.SetFlags(_inFlagsAtt.GetFlags()); token.SetOffset(_inOffsetAtt.StartOffset(), _inOffsetAtt.EndOffset()); token.SetType(_inTypeAtt.Type()); token.SetPayload(_inPayloadAtt.GetPayload()); return token; }
public void SetPayload(Payload payload) { delegate_Renamed.SetPayload(payload); }
/// <summary> /// Stores a 32 bit float in the payload, or set it to null if 1f; /// </summary> /// <param name="token"></param> /// <param name="weight"></param> public override void SetWeight(Token token, float weight) { token.SetPayload( weight == 1f ? null : new Payload(PayloadHelper.EncodeFloat(weight)) ); }
public virtual void TestCopyTo() { Token t = new Token(); Token copy = (Token) TestSimpleAttributeImpls.AssertCopyIsEqual(t); Assert.AreEqual("", t.Term()); Assert.AreEqual("", copy.Term()); t = new Token(0, 5); char[] content = "hello".ToCharArray(); t.SetTermBuffer(content, 0, 5); char[] buf = t.TermBuffer(); copy = (Token) TestSimpleAttributeImpls.AssertCopyIsEqual(t); Assert.AreEqual(t.Term(), copy.Term()); Assert.AreNotSame(buf, copy.TermBuffer()); Payload pl = new Payload(new byte[]{1, 2, 3, 4}); t.SetPayload(pl); copy = (Token) TestSimpleAttributeImpls.AssertCopyIsEqual(t); Assert.AreEqual(pl, copy.GetPayload()); Assert.AreNotSame(pl, copy.GetPayload()); }
private Token GetNextSuffixInputToken(Token token) { if (!Suffix.IncrementToken()) return null; token.SetTermBuffer(_termAtt.TermBuffer(), 0, _termAtt.TermLength()); token.SetPositionIncrement(_posIncrAtt.GetPositionIncrement()); token.SetFlags(_flagsAtt.GetFlags()); token.SetOffset(_offsetAtt.StartOffset(), _offsetAtt.EndOffset()); token.SetType(_typeAtt.Type()); token.SetPayload(_payloadAtt.GetPayload()); return token; }
public override Token Next(Token target) { target = input.Next(target); if (target == null) return null; target.SetPayload(p); // reuse the payload / byte[] data[0]++; return target; }
public override Token Next(Token result) { if (buffered != null) { Token t = buffered; buffered = null; return t; } Token t2 = input.Next(result); if (t2 == null) return null; if (System.Char.IsDigit(t2.TermBuffer()[0])) { t2.SetPositionIncrement(t2.TermBuffer()[0] - '0'); } if (first) { // set payload on first position only t2.SetPayload(new Payload(new byte[]{100})); first = false; } // index a "synonym" for every token buffered = (Token) t2.Clone(); buffered.SetPayload(null); buffered.SetPositionIncrement(0); buffered.SetTermBuffer(new char[]{'b'}, 0, 1); return t2; }