public void TestCountTruePositives()
{
Assert.AreEqual(0, FMeasure <object> .CountTruePositives(new object[] {}, new object[] {}));
Assert.AreEqual(gold.Length, FMeasure <object> .CountTruePositives(gold, gold));
Assert.AreEqual(0, FMeasure <object> .CountTruePositives(gold, predictedCompletelyDistinct));
Assert.AreEqual(2, FMeasure <object> .CountTruePositives(gold, predicted));
}
public void TestMerge()
{
var fm = new FMeasure <object>();
fm.UpdateScores(gold, predicted);
fm.UpdateScores(goldToMerge, predictedToMerge);
var fmMerge = new FMeasure <object>();
fmMerge.UpdateScores(gold, predicted);
var toMerge = new FMeasure <object>();
toMerge.UpdateScores(goldToMerge, predictedToMerge);
fmMerge.MergeInto(toMerge);
double selected1 = predicted.Length;
double target1 = gold.Length;
double tp1 = FMeasure <object> .CountTruePositives(gold, predicted);
double selected2 = predictedToMerge.Length;
double target2 = goldToMerge.Length;
double tp2 = FMeasure <object> .CountTruePositives(goldToMerge, predictedToMerge);
Assert.AreEqual((tp1 + tp2) / (target1 + target2), fm.RecallScore, Delta);
Assert.AreEqual((tp1 + tp2) / (selected1 + selected2), fm.PrecisionScore, Delta);
Assert.AreEqual(fm.RecallScore, fmMerge.RecallScore, Delta);
Assert.AreEqual(fm.PrecisionScore, fmMerge.PrecisionScore, Delta);
}
/// <summary>
/// Evaluates the given reference sample object.
/// The implementation has to update the score after every invocation.
/// </summary>
/// <param name="reference">The reference sample.</param>
/// <returns>The predicted sample</returns>
protected override NameSample ProcessSample(NameSample reference)
{
if (reference.ClearAdaptiveData)
{
nameFinder.ClearAdaptiveData();
}
var predictedNames = nameFinder.Find(reference.Sentence);
var references = reference.Names;
// OPENNLP-396 When evaluating with a file in the old format
// the type of the span is null, but must be set to default to match
// the output of the name finder.
for (var i = 0; i < references.Length; i++)
{
if (references[i].Type == null)
{
references[i] = new Span(references[i].Start, references[i].End, "default");
}
}
FMeasure.UpdateScores(references, predictedNames);
return(new NameSample(reference.Sentence, predictedNames, reference.ClearAdaptiveData));
}
public static void TestEvaluator()
{
const string goldParseString =
"(TOP (S (NP (NNS Sales) (NNS executives)) (VP (VBD were) (VP (VBG examing) (NP (DT the) (NNS figures)) (PP (IN with) (NP (JJ great) (NN care))) )) (NP (NN yesterday)) (. .) ))";
const string testParseString =
"(TOP (S (NP (NNS Sales) (NNS executives)) (VP (VBD were) (VP (VBG examing) (NP (DT the) (NNS figures)) (PP (IN with) (NP (JJ great) (NN care) (NN yesterday))) )) (. .) ))";
var gold = ParserEvaluator.GetConstituencySpans(Parse.ParseParse(goldParseString));
var test = ParserEvaluator.GetConstituencySpans(Parse.ParseParse(testParseString));
var measure = new FMeasure <Span>();
measure.UpdateScores(gold, test);
// Java expected output:
// Precision: 0.42857142857142855
// Recall: 0.375
// F-Measure: 0.39999999999999997
// c# expected output - close enough :)
// Precision: 0,428571428571429
// Recall: 0,375
// F-Measure: 0,4
Assert.AreEqual(measure.RecallScore, 0.375d);
Assert.AreEqual(measure.Value, 0, 4d);
}
/// <summary>
/// Initializes a new instance of the <see cref="SentenceDetectorEvaluator"/> class.
/// </summary>
/// <param name="sentenceDetector">The sentence detector.</param>
/// <param name="listeners">The evaluation sample listeners.</param>
public SentenceDetectorEvaluator(
ISentenceDetector sentenceDetector,
params IEvaluationMonitor <SentenceSample>[] listeners) : base(listeners)
{
this.sentenceDetector = sentenceDetector;
FMeasure = new FMeasure <Span>();
}
public static void TestEvaluator() {
const string goldParseString =
"(TOP (S (NP (NNS Sales) (NNS executives)) (VP (VBD were) (VP (VBG examing) (NP (DT the) (NNS figures)) (PP (IN with) (NP (JJ great) (NN care))) )) (NP (NN yesterday)) (. .) ))";
const string testParseString =
"(TOP (S (NP (NNS Sales) (NNS executives)) (VP (VBD were) (VP (VBG examing) (NP (DT the) (NNS figures)) (PP (IN with) (NP (JJ great) (NN care) (NN yesterday))) )) (. .) ))";
var gold = ParserEvaluator.GetConstituencySpans(Parse.ParseParse(goldParseString));
var test = ParserEvaluator.GetConstituencySpans(Parse.ParseParse(testParseString));
var measure = new FMeasure<Span>();
measure.UpdateScores(gold, test);
// Java expected output:
// Precision: 0.42857142857142855
// Recall: 0.375
// F-Measure: 0.39999999999999997
// c# expected output - close enough :)
// Precision: 0,428571428571429
// Recall: 0,375
// F-Measure: 0,4
Assert.AreEqual(measure.RecallScore, 0.375d);
Assert.AreEqual(measure.Value, 0,4d);
}
public void TestPrecision()
{
Assert.AreEqual(1.0d, FMeasure <object> .Precision(gold, gold), Delta);
Assert.AreEqual(0, FMeasure <object> .Precision(gold, predictedCompletelyDistinct), Delta);
Assert.AreEqual(Double.NaN, FMeasure <object> .Precision(gold, new object[] { }), Delta);
Assert.AreEqual(0, FMeasure <object> .Precision(new object[] { }, gold), Delta);
Assert.AreEqual(2d / predicted.Length, FMeasure <object> .Precision(gold, predicted), Delta);
}
public void TestRecall()
{
Assert.AreEqual(1.0d, FMeasure <object> .Recall(gold, gold), DELTA);
Assert.AreEqual(0, FMeasure <object> .Recall(gold, predictedCompletelyDistinct), DELTA);
Assert.AreEqual(0, FMeasure <object> .Recall(gold, new object[] { }), DELTA);
Assert.AreEqual(Double.NaN, FMeasure <object> .Recall(new object[] { }, gold), DELTA);
Assert.AreEqual(2d / gold.Length, FMeasure <object> .Recall(gold, predicted), DELTA);
}
public void TestEmpty()
{
var fm = new FMeasure <object>();
Assert.AreEqual(-1, fm.Value, Delta);
Assert.AreEqual(0, fm.RecallScore, Delta);
Assert.AreEqual(0, fm.PrecisionScore, Delta);
}
public override void run(string format, string[] args)
{
base.run(format, args);
mlParams = CmdLineUtil.loadTrainingParameters(@params.Params, false);
if (mlParams == null)
{
mlParams = ModelUtil.createTrainingParameters(@params.Iterations.Value, @params.Cutoff.Value);
}
IList <EvaluationMonitor <ChunkSample> > listeners = new LinkedList <EvaluationMonitor <ChunkSample> >();
ChunkerDetailedFMeasureListener detailedFMeasureListener = null;
if (@params.Misclassified.Value)
{
listeners.Add(new ChunkEvaluationErrorListener());
}
if (@params.DetailedF.Value)
{
detailedFMeasureListener = new ChunkerDetailedFMeasureListener();
listeners.Add(detailedFMeasureListener);
}
ChunkerCrossValidator validator;
try
{
ChunkerFactory chunkerFactory = ChunkerFactory.create(@params.Factory);
validator = new ChunkerCrossValidator(@params.Lang, mlParams, chunkerFactory, listeners.ToArray());
validator.evaluate(sampleStream, @params.Folds.Value);
}
catch (IOException e)
{
throw new TerminateToolException(-1, "IO error while reading training data or indexing data: " + e.Message, e);
}
finally
{
try
{
sampleStream.close();
}
catch (IOException)
{
// sorry that this can fail
}
}
if (detailedFMeasureListener == null)
{
FMeasure result = validator.FMeasure;
Console.WriteLine(result.ToString());
}
else
{
Console.WriteLine(detailedFMeasureListener.ToString());
}
}
public void TestPerfect()
{
var fm = new FMeasure <object>();
fm.UpdateScores(gold, gold);
Assert.AreEqual(1, fm.Value, Delta);
Assert.AreEqual(1, fm.RecallScore, Delta);
Assert.AreEqual(1, fm.PrecisionScore, Delta);
}
/// <summary>
/// Evaluates the given reference <see cref="ChunkSample"/> object.
/// The implementation has to update the score after every invocation.
/// </summary>
/// <param name="reference">The reference sample.</param>
/// <returns>The predicted sample</returns>
protected override ChunkSample ProcessSample(ChunkSample reference)
{
var preds = chunker.Chunk(reference.Sentence.ToArray(), reference.Tags.ToArray());
var result = new ChunkSample(reference.Sentence.ToArray(), reference.Tags.ToArray(), preds);
FMeasure.UpdateScores(reference.GetPhrasesAsSpanList(), result.GetPhrasesAsSpanList());
return(result);
}
/// <summary>
/// Initializes a new instance of the <see cref="TokenNameFinderCrossValidator"/> class.
/// </summary>
/// <param name="languageCode">The language of the training data.</param>
/// <param name="type"><c>null</c> or an override type for all types in the training data.</param>
/// <param name="parameters">The machine learning train parameters.</param>
/// <param name="listeners">The listeners.</param>
public TokenNameFinderCrossValidator(string languageCode, string type, TrainingParameters parameters, params IEvaluationMonitor <NameSample>[] listeners)
{
this.languageCode = languageCode;
this.type = type;
this.parameters = parameters;
this.listeners = listeners;
factory = new TokenNameFinderFactory();
FMeasure = new FMeasure <Span>();
}
/// <summary>
/// Evaluates the given reference sample object.
/// The implementation has to update the score after every invocation.
/// </summary>
/// <param name="reference">The reference sample.</param>
/// <returns>The predicted sample</returns>
protected override SentenceSample ProcessSample(SentenceSample reference)
{
var doc = reference.Document;
var predictions = TrimSpans(doc, sentenceDetector.SentPosDetect(doc));
var references = TrimSpans(doc, reference.Sentences);
FMeasure.UpdateScores(references, predictions);
return(new SentenceSample(reference.Document, Array.ConvertAll(predictions, input => (Span)input)));
}
/// <summary>
/// Evaluates the given reference sample object.
/// The implementation has to update the score after every invocation.
/// </summary>
/// <param name="reference">The reference sample.</param>
/// <returns>The predicted sample</returns>
protected override DocumentSample ProcessSample(DocumentSample reference)
{
var document = reference.Text;
var probs = documentCategorizer.Categorize(document, reference.ExtraInformation);
var cat = documentCategorizer.GetBestCategory(probs);
FMeasure.UpdateScores(new[] { reference.Category }, new[] { cat });
return(new DocumentSample(cat, reference.Text));
}
/// <summary>
/// Initializes a new instance of the <see cref="TokenNameFinderCrossValidator"/> class.
/// </summary>
/// <param name="languageCode">The language of the training data.</param>
/// <param name="type"><c>null</c> or an override type for all types in the training data.</param>
/// <param name="parameters">The machine learning train parameters.</param>
/// <param name="factory">The name finder factory.</param>
/// <param name="listeners">The listeners.</param>
public TokenNameFinderCrossValidator(string languageCode, string type, TrainingParameters parameters, TokenNameFinderFactory factory, params IEvaluationMonitor<NameSample>[] listeners) {
if (factory == null)
throw new ArgumentNullException(nameof(factory));
this.languageCode = languageCode;
this.type = type;
this.parameters = parameters;
this.factory = factory;
this.listeners = listeners;
FMeasure = new FMeasure<Span>();
}
/// <summary>
/// Initializes a new instance of the <see cref="TokenNameFinderEvaluator"/> with the given <paramref name="nameFinder"/>.
/// </summary>
/// <param name="nameFinder">The <see cref="ITokenNameFinder"/> to evaluate.</param>
/// <param name="listeners">The evaluation sample listeners.</param>
/// <exception cref="System.ArgumentNullException">listeners</exception>
public TokenNameFinderEvaluator(ITokenNameFinder nameFinder, params IEvaluationMonitor <NameSample>[] listeners)
: base(listeners)
{
if (nameFinder == null)
{
throw new ArgumentNullException(nameof(listeners));
}
this.nameFinder = nameFinder;
FMeasure = new FMeasure <Span>();
}
public override void run(string format, string[] args)
{
base.run(format, args);
mlParams = CmdLineUtil.loadTrainingParameters(@params.Params, false);
if (mlParams == null)
{
mlParams = ModelUtil.createTrainingParameters(@params.Iterations.Value, @params.Cutoff.Value);
}
SDCrossValidator validator;
SentenceDetectorEvaluationMonitor errorListener = null;
if (@params.Misclassified.Value)
{
errorListener = new SentenceEvaluationErrorListener();
}
char[] eos = null;
if (@params.EosChars != null)
{
eos = @params.EosChars.ToCharArray();
}
try
{
Dictionary abbreviations = SentenceDetectorTrainerTool.loadDict(@params.AbbDict);
SentenceDetectorFactory sdFactory = SentenceDetectorFactory.create(@params.Factory, @params.Lang, true, abbreviations, eos);
validator = new SDCrossValidator(@params.Lang, mlParams, sdFactory, errorListener);
validator.evaluate(sampleStream, @params.Folds.Value);
}
catch (IOException e)
{
throw new TerminateToolException(-1, "IO error while reading training data or indexing data: " + e.Message, e);
}
finally
{
try
{
sampleStream.close();
}
catch (IOException)
{
// sorry that this can fail
}
}
FMeasure result = validator.FMeasure;
Console.WriteLine(result.ToString());
}
/// <summary>
/// Initializes a new instance of the <see cref="ChunkerCrossValidator"/> class.
/// </summary>
/// <param name="languageCode">The language code.</param>
/// <param name="parameters">The parameters.</param>
/// <param name="factory">The factory.</param>
/// <param name="listeners">The listeners.</param>
public ChunkerCrossValidator(
string languageCode,
TrainingParameters parameters,
ChunkerFactory factory,
params IEvaluationMonitor <ChunkSample>[] listeners)
{
chunkerFactory = factory;
FMeasure = new FMeasure <Span>();
this.languageCode = languageCode;
this.parameters = parameters;
this.listeners = listeners;
}
/// <summary>
/// Initializes a new instance of the <see cref="ChunkerCrossValidator"/> class.
/// </summary>
/// <param name="languageCode">The language code.</param>
/// <param name="parameters">The parameters.</param>
/// <param name="factory">The factory.</param>
/// <param name="listeners">The listeners.</param>
public ChunkerCrossValidator(
string languageCode,
TrainingParameters parameters,
ChunkerFactory factory,
params IEvaluationMonitor<ChunkSample>[] listeners) {
chunkerFactory = factory;
FMeasure = new FMeasure<Span>();
this.languageCode = languageCode;
this.parameters = parameters;
this.listeners = listeners;
}
public override void run(string format, string[] args)
{
base.run(format, args);
mlParams = CmdLineUtil.loadTrainingParameters(@params.Params, false);
if (mlParams == null)
{
mlParams = ModelUtil.createTrainingParameters(@params.Iterations.Value, @params.Cutoff.Value);
}
TokenizerCrossValidator validator;
TokenizerEvaluationMonitor listener = null;
if (@params.Misclassified.Value)
{
listener = new TokenEvaluationErrorListener();
}
try
{
Dictionary dict = TokenizerTrainerTool.loadDict(@params.AbbDict);
TokenizerFactory tokFactory = TokenizerFactory.create(@params.Factory, @params.Lang, dict, @params.AlphaNumOpt.Value, null);
validator = new TokenizerCrossValidator(mlParams, tokFactory, listener);
validator.evaluate(sampleStream, @params.Folds.Value);
}
catch (IOException e)
{
throw new TerminateToolException(-1, "IO error while reading training data or indexing data: " + e.Message, e);
}
finally
{
try
{
sampleStream.close();
}
catch (IOException)
{
// sorry that this can fail
}
}
FMeasure result = validator.FMeasure;
Console.WriteLine(result.ToString());
}
/// <summary>
/// Evaluates the given reference sample object.
/// The implementation has to update the score after every invocation.
/// </summary>
/// <param name="reference">The reference sample.</param>
/// <returns>The predicted sample</returns>
protected override Parse ProcessSample(Parse reference)
{
var sentenceText = reference.Text;
var predictions = ParserTool.ParseLine(sentenceText, parser, 1);
if (predictions == null || predictions.Length == 0)
{
return(null);
}
FMeasure.UpdateScores(GetConstituencySpans(reference), GetConstituencySpans(predictions[0]));
return(predictions[0]);
}
/// <summary>
/// Initializes a new instance of the <see cref="CrossEvaluator{T, P}"/> class.
/// </summary>
protected CrossEvaluator()
{
FMeasure = new FMeasure <P>();
}
public ParserEvaluator(Chunking.Parser parser, params IEvaluationMonitor <Parse>[] listeners)
: base(listeners)
{
this.parser = parser;
FMeasure = new FMeasure <Span>();
}
/// <summary>
/// Initializes a new instance of the <see cref="ChunkerEvaluator"/> class.
/// </summary>
/// <param name="chunker">The chunker.</param>
/// <param name="listeners">The evaluation listeners.</param>
public ChunkerEvaluator(IChunker chunker, params IEvaluationMonitor <ChunkSample>[] listeners) : base(listeners)
{
this.chunker = chunker;
FMeasure = new FMeasure <Span>();
}
public void TestMerge() {
var fm = new FMeasure<object>();
fm.UpdateScores(gold, predicted);
fm.UpdateScores(goldToMerge, predictedToMerge);
var fmMerge = new FMeasure<object>();
fmMerge.UpdateScores(gold, predicted);
var toMerge = new FMeasure<object>();
toMerge.UpdateScores(goldToMerge, predictedToMerge);
fmMerge.MergeInto(toMerge);
double selected1 = predicted.Length;
double target1 = gold.Length;
double tp1 = FMeasure<object>.CountTruePositives(gold, predicted);
double selected2 = predictedToMerge.Length;
double target2 = goldToMerge.Length;
double tp2 = FMeasure<object>.CountTruePositives(goldToMerge, predictedToMerge);
Assert.AreEqual((tp1 + tp2)/(target1 + target2), fm.RecallScore, DELTA);
Assert.AreEqual((tp1 + tp2)/(selected1 + selected2), fm.PrecisionScore, DELTA);
Assert.AreEqual(fm.RecallScore, fmMerge.RecallScore, DELTA);
Assert.AreEqual(fm.PrecisionScore, fmMerge.PrecisionScore, DELTA);
}
public void TestEmpty() {
var fm = new FMeasure<object>();
Assert.AreEqual(-1, fm.Value, DELTA);
Assert.AreEqual(0, fm.RecallScore, DELTA);
Assert.AreEqual(0, fm.PrecisionScore, DELTA);
}
public void TestPerfect() {
var fm = new FMeasure<object>();
fm.UpdateScores(gold, gold);
Assert.AreEqual(1, fm.Value, DELTA);
Assert.AreEqual(1, fm.RecallScore, DELTA);
Assert.AreEqual(1, fm.PrecisionScore, DELTA);
}
