public override void PrintResults(PrintWriter pw, IList <ICoreMap> goldStandard, IList <ICoreMap> extractorOutput) { ResultsPrinter.Align(goldStandard, extractorOutput); // the mention factory cannot be null here System.Diagnostics.Debug.Assert(relationMentionFactory != null, "ERROR: RelationExtractorResultsPrinter.relationMentionFactory cannot be null in printResults!"); // Count predicted-actual relation type pairs ICounter <Pair <string, string> > results = new ClassicCounter <Pair <string, string> >(); ClassicCounter <string> labelCount = new ClassicCounter <string>(); // TODO: assumes binary relations for (int goldSentenceIndex = 0; goldSentenceIndex < goldStandard.Count; goldSentenceIndex++) { foreach (RelationMention goldRelation in AnnotationUtils.GetAllRelations(relationMentionFactory, goldStandard[goldSentenceIndex], createUnrelatedRelations)) { ICoreMap extractorSentence = extractorOutput[goldSentenceIndex]; IList <RelationMention> extractorRelations = AnnotationUtils.GetRelations(relationMentionFactory, extractorSentence, goldRelation.GetArg(0), goldRelation.GetArg(1)); labelCount.IncrementCount(goldRelation.GetType()); foreach (RelationMention extractorRelation in extractorRelations) { results.IncrementCount(new Pair <string, string>(extractorRelation.GetType(), goldRelation.GetType())); } } } PrintResultsInternal(pw, results, labelCount); }
public override void PrintResults(PrintWriter pw, IList <ICoreMap> goldStandard, IList <ICoreMap> extractorOutput) { ResultsPrinter.Align(goldStandard, extractorOutput); ICounter <string> correct = new ClassicCounter <string>(); ICounter <string> predicted = new ClassicCounter <string>(); ICounter <string> gold = new ClassicCounter <string>(); for (int i = 0; i < goldStandard.Count; i++) { ICoreMap goldSent = goldStandard[i]; ICoreMap sysSent = extractorOutput[i]; string sysText = sysSent.Get(typeof(CoreAnnotations.TextAnnotation)); string goldText = goldSent.Get(typeof(CoreAnnotations.TextAnnotation)); if (verbose) { log.Info("SCORING THE FOLLOWING SENTENCE:"); log.Info(sysSent.Get(typeof(CoreAnnotations.TokensAnnotation))); } HashSet <string> matchedGolds = new HashSet <string>(); IList <EntityMention> goldEntities = goldSent.Get(typeof(MachineReadingAnnotations.EntityMentionsAnnotation)); if (goldEntities == null) { goldEntities = new List <EntityMention>(); } foreach (EntityMention m in goldEntities) { string label = MakeLabel(m); if (excludedClasses != null && excludedClasses.Contains(label)) { continue; } gold.IncrementCount(label); } IList <EntityMention> sysEntities = sysSent.Get(typeof(MachineReadingAnnotations.EntityMentionsAnnotation)); if (sysEntities == null) { sysEntities = new List <EntityMention>(); } foreach (EntityMention m_1 in sysEntities) { string label = MakeLabel(m_1); if (excludedClasses != null && excludedClasses.Contains(label)) { continue; } predicted.IncrementCount(label); if (verbose) { log.Info("COMPARING PREDICTED MENTION: " + m_1); } bool found = false; foreach (EntityMention gm in goldEntities) { if (matchedGolds.Contains(gm.GetObjectId())) { continue; } if (verbose) { log.Info("\tagainst: " + gm); } if (gm.Equals(m_1, useSubTypes)) { if (verbose) { log.Info("\t\t\tMATCH!"); } found = true; matchedGolds.Add(gm.GetObjectId()); if (verboseInstances) { log.Info("TRUE POSITIVE: " + m_1 + " matched " + gm); log.Info("In sentence: " + sysText); } break; } } if (found) { correct.IncrementCount(label); } else { if (verboseInstances) { log.Info("FALSE POSITIVE: " + m_1.ToString()); log.Info("In sentence: " + sysText); } } } if (verboseInstances) { foreach (EntityMention m_2 in goldEntities) { string label = MakeLabel(m_2); if (!matchedGolds.Contains(m_2.GetObjectId()) && (excludedClasses == null || !excludedClasses.Contains(label))) { log.Info("FALSE NEGATIVE: " + m_2.ToString()); log.Info("In sentence: " + goldText); } } } } double totalCount = 0; double totalCorrect = 0; double totalPredicted = 0; pw.Println("Label\tCorrect\tPredict\tActual\tPrecn\tRecall\tF"); IList <string> labels = new List <string>(gold.KeySet()); labels.Sort(); foreach (string label_1 in labels) { if (excludedClasses != null && excludedClasses.Contains(label_1)) { continue; } double numCorrect = correct.GetCount(label_1); double numPredicted = predicted.GetCount(label_1); double trueCount = gold.GetCount(label_1); double precision = (numPredicted > 0) ? (numCorrect / numPredicted) : 0; double recall = numCorrect / trueCount; double f = (precision + recall > 0) ? 2 * precision * recall / (precision + recall) : 0.0; pw.Println(StringUtils.PadOrTrim(label_1, 21) + "\t" + numCorrect + "\t" + numPredicted + "\t" + trueCount + "\t" + Formatter.Format(precision * 100) + "\t" + Formatter.Format(100 * recall) + "\t" + Formatter.Format(100 * f)); totalCount += trueCount; totalCorrect += numCorrect; totalPredicted += numPredicted; } double precision_1 = (totalPredicted > 0) ? (totalCorrect / totalPredicted) : 0; double recall_1 = totalCorrect / totalCount; double f_1 = (totalPredicted > 0 && totalCorrect > 0) ? 2 * precision_1 * recall_1 / (precision_1 + recall_1) : 0.0; pw.Println("Total\t" + totalCorrect + "\t" + totalPredicted + "\t" + totalCount + "\t" + Formatter.Format(100 * precision_1) + "\t" + Formatter.Format(100 * recall_1) + "\t" + Formatter.Format(100 * f_1)); }