/// <summary>
/// Computes the specified tracing results.
/// </summary>
/// <param name="tracingResults">The tracing results.</param>
/// <param name="dataset">The dataset.</param>
/// <returns></returns>
public override Metric Compute(SingleTracingResults tracingResults, TLDataset dataset)
{
LineSeries precisionRecallCurve = new LineSeries(MetricName, MetricDescription);
//only if tracing results are not null...
if (tracingResults != null)
{
var resultMatrix = tracingResults.ResultMatrix;
var answerSet = dataset.AnswerSet;
var sourceArtifacts = dataset.SourceArtifacts;
TLLinksList resultLinks = resultMatrix.AllLinks;
resultLinks.Sort();
int numberOfRelevant = 0;
foreach (TLArtifact sourceArtifact in sourceArtifacts.Values)
{
numberOfRelevant += answerSet.GetCountOfLinksAboveThresholdForSourceArtifact(sourceArtifact.Id);
}
//add point only if number of relevant and number of retrieved links are greater than 0
//basically don't allow division by 0
if (numberOfRelevant == 0)
{
m_logger.Warn("Number of relevant links is 0, thus the recall value cannot be computed for the Precision Recall Curve");
}
else
{
int numberOfCorrectlyRetrieved = 0;
int numberOfRetrieved = 0;
foreach (TLSingleLink link in resultLinks)
{
numberOfRetrieved++;
//check if this is a relevant link
if (answerSet.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId))
{
numberOfCorrectlyRetrieved++;
}
double recall = (double)numberOfCorrectlyRetrieved / numberOfRelevant;
//don't need to check if number of retrieved is greater than 0 as it is always the case
double precision = (double)numberOfCorrectlyRetrieved / numberOfRetrieved;
precisionRecallCurve.AddPoint(new Point(recall, precision));
}
}
}
return(precisionRecallCurve);
}
/// <summary>
/// Generates a summarizing Metric for results analysis
/// </summary>
/// <returns>Metric object</returns>
protected override Metric GenerateSummaryImplementation()
{
LineSeries data = new LineSeries(_name, _description);
for (int i = 1; i <= Results.Count / 2; i++)
{
double x, y;
try
{
x = Results[String.Format(_recallFormat, i)];
y = Results[String.Format(_precisionFormat, i)];
}
catch (KeyNotFoundException e)
{
throw new DevelopmentKitException("Results in incorrect format.", e);
}
data.AddPoint(new Point(x, y));
}
return(data);
}
public void ExperimentResultsRawSerializationTest()
{
int n = 0;
TLExperimentResults expResultsIn = new TLExperimentResults("Technique " + n++);
for (int k = 0; k < 5; k++)
{
DatasetResults dataResults = new DatasetResults("Dataset " + n++);
for (int i = 0; i < 10; i++)
{
LineSeries line = new LineSeries("Line " + i, "Description " + n++);
for (int j = 1000 * i; j < 1000; j++)
{
line.AddPoint(new Point(j, j + 1));
}
dataResults.AddMetric(line);
BoxSummaryData box = new BoxSummaryData("Box " + i, "Description " + n++);
for (int j = 0; j < 100; j++)
{
box.AddPoint(new BoxPlotPoint(j, j + 1, j + 2, j + 3, j + 4, j + 5, j + 6, j + 7));
}
dataResults.AddMetric(box);
}
expResultsIn.AddDatasetResult(dataResults);
}
BinaryWriter binWriter = new BinaryWriter(new MemoryStream());
BinaryReader binReader = new BinaryReader(binWriter.BaseStream);
expResultsIn.WriteData(binWriter);
binReader.BaseStream.Position = 0;
TLExperimentResults expResultsOut = (TLExperimentResults)Activator.CreateInstance(typeof(TLExperimentResults), true);
expResultsOut.ReadData(binReader);
Assert.AreEqual(expResultsIn.TechniqueName, expResultsOut.TechniqueName);
Assert.AreEqual(expResultsIn.DatasetsResults.Count(), expResultsOut.DatasetsResults.Count());
foreach (DatasetResults result1 in expResultsIn.DatasetsResults)
{
bool sameDatasetResultExists = false;
DatasetResults result2 = null;
foreach (DatasetResults res in expResultsOut.DatasetsResults)
{
if (res.DatasetName == result1.DatasetName)
{
sameDatasetResultExists = true;
result2 = res;
break;
}
}
Assert.IsTrue(sameDatasetResultExists);
Assert.AreEqual(result1.DatasetName, result2.DatasetName);
Assert.AreEqual(result1.Metrics.Count(), result2.Metrics.Count());
foreach (Metric m1 in result1.Metrics)
{
bool sameMetricExists = false;
Metric m2 = null;
foreach (Metric metric in result2.Metrics)
{
if (m1.MetricName == metric.MetricName)
{
sameMetricExists = true;
m2 = metric;
break;
}
}
Assert.IsTrue(sameMetricExists);
Assert.AreEqual(m1.Description, m2.Description);
if (m1 is LineSeries)
{
Assert.IsTrue(m2 is LineSeries);
LineSeries l1 = (LineSeries)m1;
LineSeries l2 = (LineSeries)m2;
Assert.AreEqual(l1.Points.Count(), l2.Points.Count());
}
else
{
Assert.IsTrue(m2 is BoxSummaryData);
BoxSummaryData b1 = (BoxSummaryData)m1;
BoxSummaryData b2 = (BoxSummaryData)m2;
Assert.AreEqual(b1.Points.Count(), b2.Points.Count());
}
}
}
}
