private Contest BuildContestWithLargeBaseline()
{
byte[] bytes = ReadDummyContestBytes("TraceLab.Core.Test.TestResources.ContestWithLargeBaseline.tbml");
var dummyContest = CreateDummyContest("PublishContestWithLargeBaseline", bytes, "TestLargeContestFile.tbml", "application/xml");
//read baseline results from file
var assembly = System.Reflection.Assembly.GetExecutingAssembly();
var resourceStream = assembly.GetManifestResourceStream("TraceLab.Core.Test.TestResources.ContestWithLargeBaseline.tbml");
using (var reader = System.Xml.XmlReader.Create(resourceStream))
{
XPathDocument doc = new XPathDocument(reader);
XPathNavigator nav = doc.CreateNavigator();
XPathNavigator iter = nav.SelectSingleNode("//TLExperimentResults");
var serializer = TraceLab.Core.Serialization.XmlSerializerFactory.GetSerializer(typeof(TLExperimentResults), null);
TLExperimentResults baseline = (TLExperimentResults)serializer.Deserialize(iter.ReadSubtree());
dummyContest.BaselineResults = TraceLab.Core.Benchmarks.BenchmarkResultsHelper.PrepareBaselineContestRestults(dummyContest.ContestGUID, baseline, "BASELINE", "MOCK BASELINE");
//extract metrics definitions out of selected Experiment Results
List <MetricDefinition> metrics;
List <string> datasets;
TraceLab.Core.Benchmarks.BenchmarkResultsHelper.ExtractDatasetsAndMetricsDefinitions(baseline, out metrics, out datasets);
dummyContest.Metrics = metrics;
dummyContest.Datasets = datasets;
}
return(dummyContest);
}
public override void Compute()
{
Logger.Trace("Starting metrics computation: " + _config.TechniqueName);
TLSimilarityMatrix matrix = (TLSimilarityMatrix)Workspace.Load("CandidateMatrix");
TLSimilarityMatrix oracle = (TLSimilarityMatrix)Workspace.Load("AnswerMatrix");
TLExperimentResults exResults = new TLExperimentResults(_config.TechniqueName);
#region Effectiveness Best Measure
if (_config.EffectivenessBestMeasure)
{
Logger.Trace("Computing effectiveness best measure...");
IMetricComputation computation = new EffectivenessBestMeasureComputation(matrix, oracle);
computation.Compute();
ResultsController.Instance.AddResult(_config.TechniqueName, _config.DatasetName, computation);
}
else
{
Logger.Trace("Skipped effectiveness best measure computation.");
}
#endregion
#region Effectiveness All Measure
if (_config.EffectivenessAllMeasure)
{
Logger.Trace("Computing effectiveness all measure...");
IMetricComputation computation = new EffectivenessAllMeasureComputation(matrix, oracle);
computation.Compute();
ResultsController.Instance.AddResult(_config.TechniqueName, _config.DatasetName, computation);
}
else
{
Logger.Trace("Skipped effectiveness all measure computation.");
}
#endregion
}
public override void Compute()
{
Logger.Trace("Starting metrics computation: " + _config.TechniqueName);
TLSimilarityMatrix matrix = (TLSimilarityMatrix)Workspace.Load("CandidateMatrix");
TLSimilarityMatrix oracle = (TLSimilarityMatrix)Workspace.Load("AnswerMatrix");
TLExperimentResults exResults = new TLExperimentResults(_config.TechniqueName);
#region Precision
if (_config.Precision)
{
Logger.Trace("Computing precision...");
IMetricComputation computation = new PrecisionRanklistComputation(matrix, oracle);
computation.Compute();
ResultsController.Instance.AddResult(_config.TechniqueName, _config.DatasetName, computation);
}
else
{
Logger.Trace("Skipped precision computation.");
}
#endregion
#region Recall
if (_config.Recall)
{
Logger.Trace("Computing recall...");
IMetricComputation computation = new RecallRanklistComputation(matrix, oracle);
computation.Compute();
ResultsController.Instance.AddResult(_config.TechniqueName, _config.DatasetName, computation);
}
else
{
Logger.Trace("Skipped recall computation.");
}
#endregion
#region Average Precision
if (_config.AveragePrecision)
{
Logger.Trace("Computing average precision...");
IMetricComputation computation = new AveragePrecisionRanklistComputation(matrix, oracle);
computation.Compute();
ResultsController.Instance.AddResult(_config.TechniqueName, _config.DatasetName, computation);
}
else
{
Logger.Trace("Skipped average precision computation.");
}
#endregion
#region PR curve
if (_config.PRCurve)
{
Logger.Trace("Computing precision-recall curve...");
IMetricComputation computation = new PrecisionRecallCurveComputation(matrix, oracle);
computation.Compute();
ResultsController.Instance.AddResult(_config.TechniqueName, _config.DatasetName, computation);
}
else
{
Logger.Trace("Skipped precision-recall curve computation.");
}
#endregion
}
/// <summary>
/// Serializes the experiment.
/// </summary>
/// <param name="experiment">The experiment to be serialized.</param>
/// <param name="writer">The writer.</param>
/// <param name="library">The library.</param>
/// <param name="workspaceTypeDirectories">The workspace type directories.</param>
/// <param name="dataRoot">The data root.</param>
public void SerializeExperiment(IExperiment experiment, TLExperimentResults baseline, XmlWriter writer)
{
//use standard factory to serialize the experiment
var factory = new BenchmarkExperimentFactory(baseline);
experiment.SerializeToXml(writer,
(QuickGraph.VertexIdentity <ExperimentNode>)(v => v.ID),
(QuickGraph.EdgeIdentity <ExperimentNode, ExperimentNodeConnection>)(e => e.ID),
"graph", "node", "edge", "",
factory.WriteGraphAttributes,
factory.WriteNodeAttributes,
factory.WriteEdgeAttributes);
}
/// <summary>
/// Generates a TLExperimentsResultsCollection containing summary data of results.
/// This operation is thread-safe.
/// </summary>
/// <returns>Results summaries</returns>
public TLExperimentsResultsCollection GenerateSummaryResults()
{
lock (_lock)
{
TLExperimentsResultsCollection ExperimentsResultsCollection = new TLExperimentsResultsCollection();
// iterate over techniques
foreach (string technique in _techniques)
{
TLExperimentResults TechniqueResults = new TLExperimentResults(technique);
// iterate over datasets
foreach (string dataset in _datasets)
{
// get list of results for technique + dataset
List <IMetricComputation> list = null;
string key = ComputeKey(technique, dataset);
_results.TryGetValue(key, out list);
if (list != null)
{
DatasetResults data = new DatasetResults(dataset);
// add results to dataset
foreach (IMetricComputation computation in list)
{
if (!computation.HasRun)
{
computation.Compute();
}
data.AddMetric(computation.GenerateSummary());
}
// add dataset to technique
if (data.Metrics.Count() > 0)
{
TechniqueResults.AddDatasetResult(data);
}
}
}
// add technique to collection
if (TechniqueResults.DatasetsResults.Count() > 0)
{
ExperimentsResultsCollection.Add(TechniqueResults);
}
}
return(ExperimentsResultsCollection);
}
}
public static TraceLabSDK.Types.Contests.TLExperimentResults ReadBaseline(string benchmarkFile)
{
TLExperimentResults baseline = null;
using (XmlReader reader = XmlReader.Create(benchmarkFile))
{
XPathDocument doc = new XPathDocument(reader);
XPathNavigator nav = doc.CreateNavigator();
XPathNavigator baselineXmlNode = nav.SelectSingleNode("/graph/TLExperimentResults");
if (baselineXmlNode != null)
{
var experimentInfoSerializer = TraceLab.Core.Serialization.XmlSerializerFactory.GetSerializer(typeof(TLExperimentResults), null);
baseline = (TLExperimentResults)experimentInfoSerializer.Deserialize(baselineXmlNode.ReadSubtree());
}
}
return(baseline);
}
private TLExperimentResults ComputeMetricResultsForDataset(GroupOfTracingResults <T> tracingResults)
{
//create experiment results container for this technique
TLExperimentResults experimentResult = new TLExperimentResults(tracingResults.TechniqueName);
foreach (TLDataset dataset in m_datasets)
{
DatasetResults datasetResults = new DatasetResults(dataset.Name);
//iterate through all computation and calculate metric results for this dataset
foreach (MetricComputationForSingleDataset <T> computation in metricComputationsPerDataset)
{
T tracingResult = default(T);
if (tracingResults.Contains(dataset.Name))
{
tracingResult = tracingResults[dataset.Name];
}
var metric = computation.Compute(tracingResult, dataset);
if (metric == null)
{
throw new InvalidOperationException("The metric computation method failed to return the metric. " +
"Even if tracing results are empty computation must return metric with name and description, although it may have empty data");
}
if (computation is IStatisticallyComparableMetric <T> )
{
//collect results, so that statistical comparison can be computed afterwards, if there are two techniques
}
datasetResults.AddMetric(metric);
}
experimentResult.AddDatasetResult(datasetResults);
}
//compute also set of metrics across all datasets combined
//experimentResult.AcrossAllDatasetsResults = ComputeMetricResultsAcrossAllDatasets(tracingResults);
return(experimentResult);
}
public void ExperimentResultsRawSerializationTest()
{
int n = 0;
TLExperimentResults expResultsIn = new TLExperimentResults("Technique " + n++);
for (int k = 0; k < 5; k++)
{
DatasetResults dataResults = new DatasetResults("Dataset " + n++);
for (int i = 0; i < 10; i++)
{
LineSeries line = new LineSeries("Line " + i, "Description " + n++);
for (int j = 1000 * i; j < 1000; j++)
{
line.AddPoint(new Point(j, j + 1));
}
dataResults.AddMetric(line);
BoxSummaryData box = new BoxSummaryData("Box " + i, "Description " + n++);
for (int j = 0; j < 100; j++)
{
box.AddPoint(new BoxPlotPoint(j, j + 1, j + 2, j + 3, j + 4, j + 5, j + 6, j + 7));
}
dataResults.AddMetric(box);
}
expResultsIn.AddDatasetResult(dataResults);
}
BinaryWriter binWriter = new BinaryWriter(new MemoryStream());
BinaryReader binReader = new BinaryReader(binWriter.BaseStream);
expResultsIn.WriteData(binWriter);
binReader.BaseStream.Position = 0;
TLExperimentResults expResultsOut = (TLExperimentResults)Activator.CreateInstance(typeof(TLExperimentResults), true);
expResultsOut.ReadData(binReader);
Assert.AreEqual(expResultsIn.TechniqueName, expResultsOut.TechniqueName);
Assert.AreEqual(expResultsIn.DatasetsResults.Count(), expResultsOut.DatasetsResults.Count());
foreach (DatasetResults result1 in expResultsIn.DatasetsResults)
{
bool sameDatasetResultExists = false;
DatasetResults result2 = null;
foreach (DatasetResults res in expResultsOut.DatasetsResults)
{
if (res.DatasetName == result1.DatasetName)
{
sameDatasetResultExists = true;
result2 = res;
break;
}
}
Assert.IsTrue(sameDatasetResultExists);
Assert.AreEqual(result1.DatasetName, result2.DatasetName);
Assert.AreEqual(result1.Metrics.Count(), result2.Metrics.Count());
foreach (Metric m1 in result1.Metrics)
{
bool sameMetricExists = false;
Metric m2 = null;
foreach (Metric metric in result2.Metrics)
{
if (m1.MetricName == metric.MetricName)
{
sameMetricExists = true;
m2 = metric;
break;
}
}
Assert.IsTrue(sameMetricExists);
Assert.AreEqual(m1.Description, m2.Description);
if (m1 is LineSeries)
{
Assert.IsTrue(m2 is LineSeries);
LineSeries l1 = (LineSeries)m1;
LineSeries l2 = (LineSeries)m2;
Assert.AreEqual(l1.Points.Count(), l2.Points.Count());
}
else
{
Assert.IsTrue(m2 is BoxSummaryData);
BoxSummaryData b1 = (BoxSummaryData)m1;
BoxSummaryData b2 = (BoxSummaryData)m2;
Assert.AreEqual(b1.Points.Count(), b2.Points.Count());
}
}
}
}
public BenchmarkExperimentFactory(TLExperimentResults baselineResults) : base()
{
m_baseline = baselineResults;
}
public TLExperimentResults LoadBaseline()
{
TLExperimentResults baseline = BenchmarkLoader.ReadBaseline(BenchmarkInfo.FilePath);
return(baseline);
}
