/// <summary>Returns the next token in the stream, or null at EOS.
/// The returned Token is a "full private copy" (not
/// re-used across calls to next()) but will be slower
/// than calling {@link #Next(Token)} instead..
/// </summary>
public virtual Token Next()
{
Token result = Next(new Token());
if (result != null)
{
Payload p = result.GetPayload();
if (p != null)
{
result.SetPayload((Payload)p.Clone());
}
}
return(result);
}
/// <summary>Returns the next token in the stream, or null at EOS.
/// @deprecated The returned Token is a "full private copy" (not
/// re-used across calls to next()) but will be slower
/// than calling {@link #Next(Token)} instead..
/// </summary>
public virtual Token Next()
{
Token reusableToken = new Token();
Token nextToken = Next(reusableToken);
if (nextToken != null)
{
Payload p = nextToken.GetPayload();
if (p != null)
{
nextToken.SetPayload((Payload)p.Clone());
}
}
return(nextToken);
}
public virtual void TestClone()
{
Token t = new Token(0, 5);
char[] content = "hello".ToCharArray();
t.SetTermBuffer(content, 0, 5);
char[] buf = t.TermBuffer();
Token copy = (Token)TestSimpleAttributeImpls.AssertCloneIsEqual(t);
Assert.AreEqual(t.Term(), copy.Term());
Assert.AreNotSame(buf, copy.TermBuffer());
Payload pl = new Payload(new byte[] { 1, 2, 3, 4 });
t.SetPayload(pl);
copy = (Token)TestSimpleAttributeImpls.AssertCloneIsEqual(t);
Assert.AreEqual(pl, copy.GetPayload());
Assert.AreNotSame(pl, copy.GetPayload());
}
public virtual object Clone()
{
try
{
Token t = (Token)base.MemberwiseClone();
// Do a deep clone
if (termBuffer != null)
{
t.termBuffer = (char[])termBuffer.Clone();
}
if (payload != null)
{
t.SetPayload((Payload)payload.Clone());
}
return(t);
}
catch (System.Exception e)
{
throw new System.SystemException("", e); // shouldn't happen
}
}
public virtual System.Object Clone()
{
try
{
Token t = (Token)base.MemberwiseClone();
if (termBuffer != null)
{
t.termBuffer = null;
t.SetTermBuffer(termBuffer, 0, termLength);
}
if (payload != null)
{
t.SetPayload((Payload)payload.Clone());
}
return(t);
}
catch (System.Exception e)
{
throw new System.SystemException("", e); // shouldn't happen
}
}
private Token GetNextInputToken(Token token)
{
if (!_input.IncrementToken()) return null;
token.SetTermBuffer(_inTermAtt.TermBuffer(), 0, _inTermAtt.TermLength());
token.SetPositionIncrement(_inPosIncrAtt.GetPositionIncrement());
token.SetFlags(_inFlagsAtt.GetFlags());
token.SetOffset(_inOffsetAtt.StartOffset(), _inOffsetAtt.EndOffset());
token.SetType(_inTypeAtt.Type());
token.SetPayload(_inPayloadAtt.GetPayload());
return token;
}
public void SetPayload(Payload payload)
{
delegate_Renamed.SetPayload(payload);
}
/// <summary>
/// Stores a 32 bit float in the payload, or set it to null if 1f;
/// </summary>
/// <param name="token"></param>
/// <param name="weight"></param>
public override void SetWeight(Token token, float weight)
{
token.SetPayload(
weight == 1f
? null
: new Payload(PayloadHelper.EncodeFloat(weight))
);
}
public virtual void TestCopyTo()
{
Token t = new Token();
Token copy = (Token) TestSimpleAttributeImpls.AssertCopyIsEqual(t);
Assert.AreEqual("", t.Term());
Assert.AreEqual("", copy.Term());
t = new Token(0, 5);
char[] content = "hello".ToCharArray();
t.SetTermBuffer(content, 0, 5);
char[] buf = t.TermBuffer();
copy = (Token) TestSimpleAttributeImpls.AssertCopyIsEqual(t);
Assert.AreEqual(t.Term(), copy.Term());
Assert.AreNotSame(buf, copy.TermBuffer());
Payload pl = new Payload(new byte[]{1, 2, 3, 4});
t.SetPayload(pl);
copy = (Token) TestSimpleAttributeImpls.AssertCopyIsEqual(t);
Assert.AreEqual(pl, copy.GetPayload());
Assert.AreNotSame(pl, copy.GetPayload());
}
private Token GetNextSuffixInputToken(Token token)
{
if (!Suffix.IncrementToken()) return null;
token.SetTermBuffer(_termAtt.TermBuffer(), 0, _termAtt.TermLength());
token.SetPositionIncrement(_posIncrAtt.GetPositionIncrement());
token.SetFlags(_flagsAtt.GetFlags());
token.SetOffset(_offsetAtt.StartOffset(), _offsetAtt.EndOffset());
token.SetType(_typeAtt.Type());
token.SetPayload(_payloadAtt.GetPayload());
return token;
}
public override Token Next(Token target)
{
target = input.Next(target);
if (target == null)
return null;
target.SetPayload(p); // reuse the payload / byte[]
data[0]++;
return target;
}
public override Token Next(Token result)
{
if (buffered != null)
{
Token t = buffered;
buffered = null;
return t;
}
Token t2 = input.Next(result);
if (t2 == null)
return null;
if (System.Char.IsDigit(t2.TermBuffer()[0]))
{
t2.SetPositionIncrement(t2.TermBuffer()[0] - '0');
}
if (first)
{
// set payload on first position only
t2.SetPayload(new Payload(new byte[]{100}));
first = false;
}
// index a "synonym" for every token
buffered = (Token) t2.Clone();
buffered.SetPayload(null);
buffered.SetPositionIncrement(0);
buffered.SetTermBuffer(new char[]{'b'}, 0, 1);
return t2;
}