public static void TestEvaluator() {
const string goldParseString =
"(TOP (S (NP (NNS Sales) (NNS executives)) (VP (VBD were) (VP (VBG examing) (NP (DT the) (NNS figures)) (PP (IN with) (NP (JJ great) (NN care))) )) (NP (NN yesterday)) (. .) ))";
const string testParseString =
"(TOP (S (NP (NNS Sales) (NNS executives)) (VP (VBD were) (VP (VBG examing) (NP (DT the) (NNS figures)) (PP (IN with) (NP (JJ great) (NN care) (NN yesterday))) )) (. .) ))";
var gold = ParserEvaluator.GetConstituencySpans(Parse.ParseParse(goldParseString));
var test = ParserEvaluator.GetConstituencySpans(Parse.ParseParse(testParseString));
var measure = new FMeasure<Span>();
measure.UpdateScores(gold, test);
// Java expected output:
// Precision: 0.42857142857142855
// Recall: 0.375
// F-Measure: 0.39999999999999997
// c# expected output - close enough :)
// Precision: 0,428571428571429
// Recall: 0,375
// F-Measure: 0,4
Assert.AreEqual(measure.RecallScore, 0.375d);
Assert.AreEqual(measure.Value, 0,4d);
}
/// <summary>
/// Evaluates the given reference sample object.
/// The implementation has to update the score after every invocation.
/// </summary>
/// <param name="reference">The reference sample.</param>
/// <returns>The predicted sample</returns>
protected override DocumentSample ProcessSample(DocumentSample reference)
{
var document = reference.Text;
var probs = documentCategorizer.Categorize(document, reference.ExtraInformation);
var cat = documentCategorizer.GetBestCategory(probs);
FMeasure.UpdateScores(new[] { reference.Category }, new[] { cat });
return(new DocumentSample(cat, reference.Text));
}
/// <summary>
/// Evaluates the given reference sample object.
/// The implementation has to update the score after every invocation.
/// </summary>
/// <param name="reference">The reference sample.</param>
/// <returns>The predicted sample</returns>
protected override Parse ProcessSample(Parse reference)
{
var sentenceText = reference.Text;
var predictions = ParserTool.ParseLine(sentenceText, parser, 1);
if (predictions == null || predictions.Length == 0)
{
return(null);
}
FMeasure.UpdateScores(GetConstituencySpans(reference), GetConstituencySpans(predictions[0]));
return(predictions[0]);
}
public void TestMerge() {
var fm = new FMeasure<object>();
fm.UpdateScores(gold, predicted);
fm.UpdateScores(goldToMerge, predictedToMerge);
var fmMerge = new FMeasure<object>();
fmMerge.UpdateScores(gold, predicted);
var toMerge = new FMeasure<object>();
toMerge.UpdateScores(goldToMerge, predictedToMerge);
fmMerge.MergeInto(toMerge);
double selected1 = predicted.Length;
double target1 = gold.Length;
double tp1 = FMeasure<object>.CountTruePositives(gold, predicted);
double selected2 = predictedToMerge.Length;
double target2 = goldToMerge.Length;
double tp2 = FMeasure<object>.CountTruePositives(goldToMerge, predictedToMerge);
Assert.AreEqual((tp1 + tp2)/(target1 + target2), fm.RecallScore, DELTA);
Assert.AreEqual((tp1 + tp2)/(selected1 + selected2), fm.PrecisionScore, DELTA);
Assert.AreEqual(fm.RecallScore, fmMerge.RecallScore, DELTA);
Assert.AreEqual(fm.PrecisionScore, fmMerge.PrecisionScore, DELTA);
}
public void TestPerfect() {
var fm = new FMeasure<object>();
fm.UpdateScores(gold, gold);
Assert.AreEqual(1, fm.Value, DELTA);
Assert.AreEqual(1, fm.RecallScore, DELTA);
Assert.AreEqual(1, fm.PrecisionScore, DELTA);
}