/// <summary> /// Computes the specified tracing results. /// </summary> /// <param name="tracingResults">The tracing results.</param> /// <param name="dataset">The dataset.</param> /// <returns></returns> public override Metric Compute(SingleTracingResults tracingResults, TLDataset dataset) { LineSeries precisionRecallCurve = new LineSeries(MetricName, MetricDescription); //only if tracing results are not null... if (tracingResults != null) { var resultMatrix = tracingResults.ResultMatrix; var answerSet = dataset.AnswerSet; var sourceArtifacts = dataset.SourceArtifacts; TLLinksList resultLinks = resultMatrix.AllLinks; resultLinks.Sort(); int numberOfRelevant = 0; foreach (TLArtifact sourceArtifact in sourceArtifacts.Values) { numberOfRelevant += answerSet.GetCountOfLinksAboveThresholdForSourceArtifact(sourceArtifact.Id); } //add point only if number of relevant and number of retrieved links are greater than 0 //basically don't allow division by 0 if (numberOfRelevant == 0) { m_logger.Warn("Number of relevant links is 0, thus the recall value cannot be computed for the Precision Recall Curve"); } else { int numberOfCorrectlyRetrieved = 0; int numberOfRetrieved = 0; foreach (TLSingleLink link in resultLinks) { numberOfRetrieved++; //check if this is a relevant link if (answerSet.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId)) { numberOfCorrectlyRetrieved++; } double recall = (double)numberOfCorrectlyRetrieved / numberOfRelevant; //don't need to check if number of retrieved is greater than 0 as it is always the case double precision = (double)numberOfCorrectlyRetrieved / numberOfRetrieved; precisionRecallCurve.AddPoint(new Point(recall, precision)); } } } return(precisionRecallCurve); }
/// <summary> /// Generates a summarizing Metric for results analysis /// </summary> /// <returns>Metric object</returns> protected override Metric GenerateSummaryImplementation() { LineSeries data = new LineSeries(_name, _description); for (int i = 1; i <= Results.Count / 2; i++) { double x, y; try { x = Results[String.Format(_recallFormat, i)]; y = Results[String.Format(_precisionFormat, i)]; } catch (KeyNotFoundException e) { throw new DevelopmentKitException("Results in incorrect format.", e); } data.AddPoint(new Point(x, y)); } return(data); }
public void ExperimentResultsRawSerializationTest() { int n = 0; TLExperimentResults expResultsIn = new TLExperimentResults("Technique " + n++); for (int k = 0; k < 5; k++) { DatasetResults dataResults = new DatasetResults("Dataset " + n++); for (int i = 0; i < 10; i++) { LineSeries line = new LineSeries("Line " + i, "Description " + n++); for (int j = 1000 * i; j < 1000; j++) { line.AddPoint(new Point(j, j + 1)); } dataResults.AddMetric(line); BoxSummaryData box = new BoxSummaryData("Box " + i, "Description " + n++); for (int j = 0; j < 100; j++) { box.AddPoint(new BoxPlotPoint(j, j + 1, j + 2, j + 3, j + 4, j + 5, j + 6, j + 7)); } dataResults.AddMetric(box); } expResultsIn.AddDatasetResult(dataResults); } BinaryWriter binWriter = new BinaryWriter(new MemoryStream()); BinaryReader binReader = new BinaryReader(binWriter.BaseStream); expResultsIn.WriteData(binWriter); binReader.BaseStream.Position = 0; TLExperimentResults expResultsOut = (TLExperimentResults)Activator.CreateInstance(typeof(TLExperimentResults), true); expResultsOut.ReadData(binReader); Assert.AreEqual(expResultsIn.TechniqueName, expResultsOut.TechniqueName); Assert.AreEqual(expResultsIn.DatasetsResults.Count(), expResultsOut.DatasetsResults.Count()); foreach (DatasetResults result1 in expResultsIn.DatasetsResults) { bool sameDatasetResultExists = false; DatasetResults result2 = null; foreach (DatasetResults res in expResultsOut.DatasetsResults) { if (res.DatasetName == result1.DatasetName) { sameDatasetResultExists = true; result2 = res; break; } } Assert.IsTrue(sameDatasetResultExists); Assert.AreEqual(result1.DatasetName, result2.DatasetName); Assert.AreEqual(result1.Metrics.Count(), result2.Metrics.Count()); foreach (Metric m1 in result1.Metrics) { bool sameMetricExists = false; Metric m2 = null; foreach (Metric metric in result2.Metrics) { if (m1.MetricName == metric.MetricName) { sameMetricExists = true; m2 = metric; break; } } Assert.IsTrue(sameMetricExists); Assert.AreEqual(m1.Description, m2.Description); if (m1 is LineSeries) { Assert.IsTrue(m2 is LineSeries); LineSeries l1 = (LineSeries)m1; LineSeries l2 = (LineSeries)m2; Assert.AreEqual(l1.Points.Count(), l2.Points.Count()); } else { Assert.IsTrue(m2 is BoxSummaryData); BoxSummaryData b1 = (BoxSummaryData)m1; BoxSummaryData b2 = (BoxSummaryData)m2; Assert.AreEqual(b1.Points.Count(), b2.Points.Count()); } } } }