public virtual void TestNoPrefixIO()
{
IList <CoreLabel> testInput = LoadCoreLabelList(words, noprefix);
IOBUtils.EntitySubclassify(testInput, typeof(CoreAnnotations.AnswerAnnotation), "O", "io", true);
CheckAnswers(testInput, words, io);
}
public virtual void TestIOBESIOB1()
{
IList <CoreLabel> testInput = LoadCoreLabelList(words, iobes);
IOBUtils.EntitySubclassify(testInput, typeof(CoreAnnotations.AnswerAnnotation), "O", "iob1", true);
CheckAnswers(testInput, words, iob1);
}
public virtual void TestIOB2BILOU()
{
IList <CoreLabel> testInput = LoadCoreLabelList(words, iob2);
IOBUtils.EntitySubclassify(testInput, typeof(CoreAnnotations.AnswerAnnotation), "O", "BILOU", true);
CheckAnswers(testInput, words, bilou);
}
/// <summary>
/// Return the coding scheme to IOB1 coding, regardless of what was used
/// internally (unless retainEntitySubclassification is set).
/// </summary>
/// <remarks>
/// Return the coding scheme to IOB1 coding, regardless of what was used
/// internally (unless retainEntitySubclassification is set).
/// This is useful for scoring against CoNLL test output.
/// </remarks>
/// <param name="tokens">List of tokens in some NER encoding</param>
private void DeEndify(IList <CoreLabel> tokens)
{
if (flags.retainEntitySubclassification)
{
return;
}
IOBUtils.EntitySubclassify(tokens, typeof(CoreAnnotations.AnswerAnnotation), flags.backgroundSymbol, "iob1", flags.intern);
}
private IList <CoreLabel> ProcessDocument(string doc)
{
IList <CoreLabel> list = new List <CoreLabel>();
string[] lines = doc.Split("\n");
foreach (string line in lines)
{
if (!flags.deleteBlankLines || !white.Matcher(line).Matches())
{
list.Add(MakeCoreLabel(line));
}
}
IOBUtils.EntitySubclassify(list, typeof(CoreAnnotations.AnswerAnnotation), flags.backgroundSymbol, flags.entitySubclassification, flags.intern);
return(list);
}