/// <summary>
/// Computes the score for the given current results.
/// </summary>
/// <param name="baselineMatrices">The baseline matrices.</param>
/// <param name="currentResults">The current results.</param>
/// <returns></returns>
public static double ComputeScore(TLSimilarityMatricesCollection baselineMatrices, TLSimilarityMatricesCollection currentResultsMatrices,
TLDatasetsList datasets, MetricComputationComponentConfig config)
{
double score = 0.0;
double sum = 0;
double counts = 0;
IMetricComputation metricComputation = GetMetricComputation(config);
foreach (TLDataset dataset in datasets)
{
TLSimilarityMatrix baseline = baselineMatrices[dataset.Name];
TLSimilarityMatrix current = currentResultsMatrices[dataset.Name];
//score is computed based on delta between two techniques from metric computation
SortedDictionary<string, double> baselineValues = metricComputation.Calculate(baseline, dataset);
SortedDictionary<string, double> currentValues = metricComputation.Calculate(current, dataset);
var deltas = ScoreComputationHelper.Delta(baselineValues, currentValues);
//now compute average of computed deltas, and that's the score
foreach (double delta in deltas.Values)
{
sum += delta;
}
counts += deltas.Count;
}
score = sum / counts;
return score;
}
public override void Compute()
{
TLDatasetsList listOfDatasets = (TLDatasetsList)Workspace.Load("listOfDatasets");
if (listOfDatasets == null)
{
throw new ComponentException("Received null listOfDatasets");
}
if (listOfDatasets.Count == 0)
{
throw new ComponentException("Received empty listOfDatasets");
}
int index = (int)Workspace.Load("indexOfDataset");
if (index < 0)
{
throw new ComponentException("Received negative indexOfDataset");
}
if (index > listOfDatasets.Count)
{
throw new ComponentException("Received indexOfDataset is higher that number of datasets in the received list.");
}
TLDataset dataset = listOfDatasets[index];
Workspace.Store("sourceArtifacts", dataset.SourceArtifacts);
Workspace.Store("targetArtifacts", dataset.TargetArtifacts);
Workspace.Store("answerSet", dataset.AnswerSet);
Workspace.Store("datasetName", dataset.Name);
Logger.Info(String.Format("Loaded dataset '{0}' source artifacts collection (id:{1}), and target artifacts collection (id:{2})", dataset.Name, dataset.SourceArtifacts.CollectionId, dataset.TargetArtifacts.CollectionId));
}
/// <summary>
/// Computes the score for the given current results.
/// </summary>
/// <param name="baselineMatrices">The baseline matrices.</param>
/// <param name="currentResults">The current results.</param>
/// <returns></returns>
public static double ComputeScore(TLSimilarityMatricesCollection baselineMatrices, TLSimilarityMatricesCollection currentResultsMatrices,
TLDatasetsList datasets, MetricComputationComponentConfig config)
{
double score = 0.0;
double sum = 0;
double counts = 0;
IMetricComputation metricComputation = GetMetricComputation(config);
foreach (TLDataset dataset in datasets)
{
TLSimilarityMatrix baseline = baselineMatrices[dataset.Name];
TLSimilarityMatrix current = currentResultsMatrices[dataset.Name];
//score is computed based on delta between two techniques from metric computation
SortedDictionary <string, double> baselineValues = metricComputation.Calculate(baseline, dataset);
SortedDictionary <string, double> currentValues = metricComputation.Calculate(current, dataset);
var deltas = ScoreComputationHelper.Delta(baselineValues, currentValues);
//now compute average of computed deltas, and that's the score
foreach (double delta in deltas.Values)
{
sum += delta;
}
counts += deltas.Count;
}
score = sum / counts;
return(score);
}
public MetricComputationEngine(TLDatasetsList datasets, ComponentLogger logger, MetricComputationComponentConfig config)
: base(datasets)
{
if (logger == null)
{
throw new ArgumentNullException("logger");
}
m_logger = logger;
InitMetricComputationsPerDataset(config);
}
public override void Compute()
{
// Loading artifacts & datasets from workspace
TLArtifactsCollection sourceArtifacts = (TLArtifactsCollection)Workspace.Load("sourceArtifacts");
TLArtifactsCollection targetArtifacts = (TLArtifactsCollection)Workspace.Load("targetArtifacts");
TLSimilarityMatrix answerMatrix = (TLSimilarityMatrix)Workspace.Load("answerMatrix");
TLSimilarityMatrix similarityMatrix = (TLSimilarityMatrix)Workspace.Load("similarityMatrix");
// Checking for null arguments
if (sourceArtifacts == null)
{
throw new ComponentException("The loaded source artifacts cannot be null!");
}
if (targetArtifacts == null)
{
throw new ComponentException("The loaded target artifacts cannot be null!");
}
if (answerMatrix == null)
{
throw new ComponentException("The loaded answer matrix cannot be null!");
}
if (similarityMatrix == null)
{
throw new ComponentException("The loaded similarity matrix cannot be null!");
}
// Results calculation
TLDatasetsList datasets = new TLDatasetsList();
var dataset = new TLDataset("Experiment results");
dataset.AnswerSet = answerMatrix;
dataset.SourceArtifacts = sourceArtifacts;
dataset.TargetArtifacts = targetArtifacts;
datasets.Add(dataset);
TLSimilarityMatricesCollection similarityMatrices = new TLSimilarityMatricesCollection();
similarityMatrix.Name = "Experiment results";
similarityMatrices.Add(similarityMatrix);
MetricComputationEngine engine = new MetricComputationEngine(datasets, Logger, m_config);
//wrap result similarity matrix into TracingResult
var tracingResults = GroupOfTracingResults <SingleTracingResults> .Adapt(similarityMatrices, "Experiment results");
engine.AddTracingResults(tracingResults);
var results = engine.ComputeResults();
// Store the results in the workspace
Workspace.Store("results", results);
}
public override void Compute()
{
var datasetsLocations = ReadDatasetsLocations();
TLDatasetsList listOfDatasets = new TLDatasetsList();
foreach (DatasetLocation locations in datasetsLocations)
{
TLDataset dataset = new TLDataset(locations.DatasetName);
string error;
//do validation
if (CoestDatasetImporterHelper.ValidatePath(locations.SourceArtifactsLocation, "Source Artifacts File", out error))
{
dataset.SourceArtifacts = CoestDatasetImporterHelper.ImportArtifacts(locations.SourceArtifactsLocation, m_config.TrimElementValues);
Logger.Info(String.Format("Source artifacts imported from {0}.", locations.SourceArtifactsLocation));
}
else
{
throw new ComponentException(error);
}
//do validation
if (CoestDatasetImporterHelper.ValidatePath(locations.SourceArtifactsLocation, "Target Artifacts File", out error))
{
dataset.TargetArtifacts = CoestDatasetImporterHelper.ImportArtifacts(locations.TargetArtifactsLocation, m_config.TrimElementValues);
Logger.Info(String.Format("Target artifacts imported from {0}.", locations.TargetArtifactsLocation));
}
else
{
throw new ComponentException(error);
}
//do validation
if (CoestDatasetImporterHelper.ValidatePath(locations.SourceArtifactsLocation, "Target Artifacts File", out error))
{
dataset.AnswerSet = CoestDatasetImporterHelper.ImportAnswerSet(locations.AnswerSetLocation, dataset.SourceArtifacts, locations.SourceArtifactsLocation, dataset.TargetArtifacts, locations.TargetArtifactsLocation, Logger, m_config.TrimElementValues);
Logger.Info(String.Format("Answer set imported from {0}.", locations.AnswerSetLocation));
}
else
{
throw new ComponentException(error);
}
listOfDatasets.Add(dataset);
}
Workspace.Store("listOfDatasets", listOfDatasets);
Workspace.Store("numberOfDatasets", listOfDatasets.Count);
}
public override void Compute()
{
var datasetsLocations = ReadDatasetsLocations();
TLDatasetsList listOfDatasets = new TLDatasetsList();
foreach (DatasetLocation locations in datasetsLocations)
{
TLDataset dataset = new TLDataset(locations.DatasetName);
string error;
//do validation
if (CoestDatasetImporterHelper.ValidatePath(locations.SourceArtifactsLocation, "Source Artifacts File", out error))
{
dataset.SourceArtifacts = CoestDatasetImporterHelper.ImportArtifacts(locations.SourceArtifactsLocation, m_config.TrimElementValues);
Logger.Info(String.Format("Source artifacts imported from {0}.", locations.SourceArtifactsLocation));
}
else
{
throw new ComponentException(error);
}
//do validation
if (CoestDatasetImporterHelper.ValidatePath(locations.SourceArtifactsLocation, "Target Artifacts File", out error))
{
dataset.TargetArtifacts = CoestDatasetImporterHelper.ImportArtifacts(locations.TargetArtifactsLocation, m_config.TrimElementValues);
Logger.Info(String.Format("Target artifacts imported from {0}.", locations.TargetArtifactsLocation));
}
else
{
throw new ComponentException(error);
}
//do validation
if (CoestDatasetImporterHelper.ValidatePath(locations.SourceArtifactsLocation, "Target Artifacts File", out error))
{
dataset.AnswerSet = CoestDatasetImporterHelper.ImportAnswerSet(locations.AnswerSetLocation, dataset.SourceArtifacts, locations.SourceArtifactsLocation, dataset.TargetArtifacts, locations.TargetArtifactsLocation, Logger, m_config.TrimElementValues);
Logger.Info(String.Format("Answer set imported from {0}.", locations.AnswerSetLocation));
}
else
{
throw new ComponentException(error);
}
listOfDatasets.Add(dataset);
}
Workspace.Store("listOfDatasets", listOfDatasets);
Workspace.Store("numberOfDatasets", listOfDatasets.Count);
}
public override void Compute()
{
// Loading artifacts & datasets from workspace
TLArtifactsCollection sourceArtifacts = (TLArtifactsCollection)Workspace.Load("sourceArtifacts");
TLArtifactsCollection targetArtifacts = (TLArtifactsCollection)Workspace.Load("targetArtifacts");
TLSimilarityMatrix answerMatrix = (TLSimilarityMatrix)Workspace.Load("answerMatrix");
TLSimilarityMatrix similarityMatrix = (TLSimilarityMatrix)Workspace.Load("similarityMatrix");
// Checking for null arguments
if (sourceArtifacts == null)
{
throw new ComponentException("The loaded source artifacts cannot be null!");
}
if (targetArtifacts == null)
{
throw new ComponentException("The loaded target artifacts cannot be null!");
}
if (answerMatrix == null)
{
throw new ComponentException("The loaded answer matrix cannot be null!");
}
if (similarityMatrix == null)
{
throw new ComponentException("The loaded similarity matrix cannot be null!");
}
// Results calculation
TLDatasetsList datasets = new TLDatasetsList();
var dataset = new TLDataset("Experiment results");
dataset.AnswerSet = answerMatrix;
dataset.SourceArtifacts = sourceArtifacts;
dataset.TargetArtifacts = targetArtifacts;
datasets.Add(dataset);
TLSimilarityMatricesCollection similarityMatrices = new TLSimilarityMatricesCollection();
similarityMatrix.Name = "Experiment results";
similarityMatrices.Add(similarityMatrix);
MetricComputationEngine engine = new MetricComputationEngine(datasets, Logger, m_config);
//wrap result similarity matrix into TracingResult
var tracingResults = GroupOfTracingResults<SingleTracingResults>.Adapt(similarityMatrices, "Experiment results");
engine.AddTracingResults(tracingResults);
var results = engine.ComputeResults();
// Store the results in the workspace
Workspace.Store("results", results);
}
public override void Compute()
{
//load dataset from workspace
TLDatasetsList datasets = (TLDatasetsList)Workspace.Load("datasets");
TLSimilarityMatricesCollection resultSimilarityMatrices = (TLSimilarityMatricesCollection)Workspace.Load("resultSimilarityMatrices");
//wrap result similarity matrix into TracingResult
var tracingResults = Adapt(resultSimilarityMatrices, "Current technique");
MetricComputationEngine engine = new MetricComputationEngine(datasets, Logger, m_config);
engine.AddTracingResults(tracingResults);
TLExperimentsResultsCollection allExperimentResults = engine.ComputeResults();
//set base data from which all metrics were computed from
allExperimentResults["Current technique"].BaseData = resultSimilarityMatrices;
//check if baseline results are in the workspace
var baseline = (TLExperimentResults)Workspace.Load("BASELINE");
if (baseline != null)
{
//if so, add them to collection of experiment results, so that it can be viewed by GUI component
allExperimentResults.Add(baseline);
TLSimilarityMatricesCollection baselineMatrices = baseline.BaseData as TLSimilarityMatricesCollection;
//compute the score
allExperimentResults["Current technique"].Score = ScoreComputation.ComputeScore(baselineMatrices, resultSimilarityMatrices, datasets, m_config);
}
else
{
allExperimentResults["Current technique"].Score = 0.0;
}
Workspace.Store("comparedResults", allExperimentResults);
Workspace.Store("currentResults", allExperimentResults["Current technique"]);
}
public MetricComputationEngineService(TLDatasetsList datasets)
{
m_datasets = datasets;
m_allTracingResults = new List <GroupOfTracingResults <T> >();
}
