/// <summary>
/// Exports an IMetricComputation to disk
/// </summary>
/// <param name="computation">Computation</param>
/// <param name="filename">Destination file</param>
/// <param name="summary">Option to write summary data</param>
public static void Export(IMetricComputation computation, string filename, bool summary = false)
{
// compute file names
string basefilename = (filename.EndsWith(".txt"))
? filename.Substring(0, filename.Length - 4)
: filename;
string longfilename = basefilename + ".csv";
string summaryfilename = basefilename + ".summary.xml";
// write results
TextWriter file = new StreamWriter(longfilename);
file.WriteLine("\"{0}\",\"{1}\"", computation.Name.Replace("\"", "\\\"\\"), computation.Description.Replace("\"", "\\\"\\"));
foreach (KeyValuePair <string, double> result in computation.Results)
{
file.WriteLine("\"{0}\",{1}", result.Key.Replace("\"", "\\\"\\"), result.Value);
}
file.Flush();
file.Close();
// write summary
if (summary)
{
using (XmlWriter writer = XmlWriter.Create(summaryfilename))
{
writer.WriteStartDocument();
writer.WriteStartElement("Metric");
Metric metric = computation.GenerateSummary();
metric.WriteXml(writer);
writer.WriteEndElement();
writer.WriteEndDocument();
}
}
}
public SummaryStatistics(string metricName, string metricDescription, IMetricComputation baseMetricComputation, ComponentLogger logger)
{
m_metricComputation = baseMetricComputation;
m_metricName = metricName;
m_metricDescription = metricDescription;
m_logger = logger;
}
/// <summary>
/// Computes the score for the given current results.
/// </summary>
/// <param name="baselineMatrices">The baseline matrices.</param>
/// <param name="currentResults">The current results.</param>
/// <returns></returns>
public static double ComputeScore(TLSimilarityMatricesCollection baselineMatrices, TLSimilarityMatricesCollection currentResultsMatrices,
TLDatasetsList datasets, MetricComputationComponentConfig config)
{
double score = 0.0;
double sum = 0;
double counts = 0;
IMetricComputation metricComputation = GetMetricComputation(config);
foreach (TLDataset dataset in datasets)
{
TLSimilarityMatrix baseline = baselineMatrices[dataset.Name];
TLSimilarityMatrix current = currentResultsMatrices[dataset.Name];
//score is computed based on delta between two techniques from metric computation
SortedDictionary <string, double> baselineValues = metricComputation.Calculate(baseline, dataset);
SortedDictionary <string, double> currentValues = metricComputation.Calculate(current, dataset);
var deltas = ScoreComputationHelper.Delta(baselineValues, currentValues);
//now compute average of computed deltas, and that's the score
foreach (double delta in deltas.Values)
{
sum += delta;
}
counts += deltas.Count;
}
score = sum / counts;
return(score);
}
/// <summary>
/// Adds the results for the given computation.
/// This operation is thread-safe.
/// </summary>
/// <param name="technique">Technique name</param>
/// <param name="dataset">Dataset name</param>
/// <param name="computation">Computation object</param>
public void AddResult(string technique, string dataset, IMetricComputation computation)
{
if (String.IsNullOrWhiteSpace(technique))
{
throw new ComponentException("Technique name cannot be empty.");
}
if (String.IsNullOrWhiteSpace(dataset))
{
throw new ComponentException("Dataset name cannot be empty.");
}
if (computation == null)
{
throw new ComponentException("Computation cannot be null.");
}
lock (_lock)
{
_techniques.Add(technique);
_datasets.Add(dataset);
string key = ComputeKey(technique, dataset);
if (!_results.ContainsKey(key))
{
_results.Add(key, new List <IMetricComputation>());
}
_results[key].Add(computation);
}
}
public SummaryStatistics(string metricName, string metricDescription, IMetricComputation baseMetricComputation, ComponentLogger logger)
{
m_metricComputation = baseMetricComputation;
m_metricName = metricName;
m_metricDescription = metricDescription;
m_logger = logger;
}
/// <summary>
/// Exports an IMetricComputation to disk
/// </summary>
/// <param name="computation">Computation</param>
/// <param name="filename">Destination file</param>
/// <param name="summary">Option to write summary data</param>
public static void Export(IMetricComputation computation, string filename, bool summary = false)
{
// compute file names
string basefilename = (filename.EndsWith(".txt"))
? filename.Substring(0, filename.Length - 4)
: filename;
string longfilename = basefilename + ".csv";
string summaryfilename = basefilename + ".summary.xml";
// write results
TextWriter file = new StreamWriter(longfilename);
file.WriteLine("\"{0}\",\"{1}\"", computation.Name.Replace("\"", "\\\"\\"), computation.Description.Replace("\"", "\\\"\\"));
foreach (KeyValuePair<string, double> result in computation.Results)
{
file.WriteLine("\"{0}\",{1}", result.Key.Replace("\"", "\\\"\\"), result.Value);
}
file.Flush();
file.Close();
// write summary
if (summary)
{
using (XmlWriter writer = XmlWriter.Create(summaryfilename))
{
writer.WriteStartDocument();
writer.WriteStartElement("Metric");
Metric metric = computation.GenerateSummary();
metric.WriteXml(writer);
writer.WriteEndElement();
writer.WriteEndDocument();
}
}
}
public WilcoxonSignedRankTest(IMetricComputation baseMetricComputation)
{
m_baseMetricComputation = baseMetricComputation;
}
/// <summary>
/// Adds the results for the given computation.
/// This operation is thread-safe.
/// </summary>
/// <param name="technique">Technique name</param>
/// <param name="dataset">Dataset name</param>
/// <param name="computation">Computation object</param>
public void AddResult(string technique, string dataset, IMetricComputation computation)
{
if (String.IsNullOrWhiteSpace(technique))
{
throw new ComponentException("Technique name cannot be empty.");
}
if (String.IsNullOrWhiteSpace(dataset))
{
throw new ComponentException("Dataset name cannot be empty.");
}
if (computation == null)
{
throw new ComponentException("Computation cannot be null.");
}
lock (_lock)
{
_techniques.Add(technique);
_datasets.Add(dataset);
string key = ComputeKey(technique, dataset);
if (!_results.ContainsKey(key))
{
_results.Add(key, new List<IMetricComputation>());
}
_results[key].Add(computation);
}
}
