/// <summary>
/// Returns a 32 bit float from the payload, or 1f it null.
/// </summary>
/// <param name="token"></param>
/// <returns></returns>
public override float GetWeight(Token token)
{
if (token.GetPayload() == null || token.GetPayload().GetData() == null)
return 1f;
return PayloadHelper.DecodeFloat(token.GetPayload().GetData());
}
public virtual void TestClone()
{
Token t = new Token(0, 5);
char[] content = "hello".ToCharArray();
t.SetTermBuffer(content, 0, 5);
char[] buf = t.TermBuffer();
Token copy = (Token)TestSimpleAttributeImpls.AssertCloneIsEqual(t);
Assert.AreEqual(t.Term(), copy.Term());
Assert.AreNotSame(buf, copy.TermBuffer());
Payload pl = new Payload(new byte[] { 1, 2, 3, 4 });
t.SetPayload(pl);
copy = (Token)TestSimpleAttributeImpls.AssertCloneIsEqual(t);
Assert.AreEqual(pl, copy.GetPayload());
Assert.AreNotSame(pl, copy.GetPayload());
}
/// <summary>Returns the next token in the stream, or null at EOS.
/// The returned Token is a "full private copy" (not
/// re-used across calls to next()) but will be slower
/// than calling {@link #Next(Token)} instead..
/// </summary>
public virtual Token Next()
{
Token result = Next(new Token());
if (result != null)
{
Payload p = result.GetPayload();
if (p != null)
{
result.SetPayload((Payload)p.Clone());
}
}
return(result);
}
/// <summary>Returns the next token in the stream, or null at EOS.
/// @deprecated The returned Token is a "full private copy" (not
/// re-used across calls to next()) but will be slower
/// than calling {@link #Next(Token)} instead..
/// </summary>
public virtual Token Next()
{
Token reusableToken = new Token();
Token nextToken = Next(reusableToken);
if (nextToken != null)
{
Payload p = nextToken.GetPayload();
if (p != null)
{
nextToken.SetPayload((Payload)p.Clone());
}
}
return(nextToken);
}
private static void ConsumeStreamOldAPI(TokenStream stream)
{
stream.Reset();
Token reusableToken = new Token();
int i = 0;
while ((reusableToken = stream.Next(reusableToken)) != null)
{
System.String term = reusableToken.Term();
Payload p = reusableToken.GetPayload();
if (p != null && p.GetData().Length == 1 && p.GetData()[0] == PartOfSpeechAnnotatingFilter.PROPER_NOUN_ANNOTATION)
{
Assert.IsTrue("tokenstream".Equals(term), "only TokenStream is a proper noun");
}
else
{
Assert.IsFalse("tokenstream".Equals(term), "all other tokens (if this test fails, the special POSToken subclass is not correctly passed through the chain)");
}
Assert.AreEqual(results[i], term);
i++;
}
}
private static void ConsumeStreamOldAPI(TokenStream stream)
{
stream.Reset();
Token reusableToken = new Token();
int i = 0;
while ((reusableToken = stream.Next(reusableToken)) != null)
{
System.String term = reusableToken.Term();
Payload p = reusableToken.GetPayload();
if (p != null && p.GetData().Length == 1 && p.GetData()[0] == PartOfSpeechAnnotatingFilter.PROPER_NOUN_ANNOTATION)
{
Assert.IsTrue("tokenstream".Equals(term), "only TokenStream is a proper noun");
}
else
{
Assert.IsFalse("tokenstream".Equals(term), "all other tokens (if this test fails, the special POSToken subclass is not correctly passed through the chain)");
}
Assert.AreEqual(results[i], term);
i++;
}
}
// PayloadAttribute
public Payload GetPayload()
{
return(delegate_Renamed.GetPayload());
}
private void SetCurrentToken(Token token)
{
if (token == null) return;
ClearAttributes();
_termAtt.SetTermBuffer(token.TermBuffer(), 0, token.TermLength());
_posIncrAtt.SetPositionIncrement(token.GetPositionIncrement());
_flagsAtt.SetFlags(token.GetFlags());
_offsetAtt.SetOffset(token.StartOffset(), token.EndOffset());
_typeAtt.SetType(token.Type());
_payloadAtt.SetPayload(token.GetPayload());
}
internal virtual void VerifyPayload(TokenStream ts)
{
Token t = new Token();
for (byte b = 1; ; b++)
{
t.Clear();
t = ts.Next(t);
if (t == null)
break;
// System.out.println("id="+System.identityHashCode(t) + " " + t);
// System.out.println("payload=" + (int)t.getPayload().toByteArray()[0]);
Assert.AreEqual(b, t.GetPayload().ToByteArray()[0]);
}
}