public static void ComputeMetrics(TLSimilarityMatrix sims, TLSimilarityMatrix oracle, RecallLevel level,
out TLKeyValuePairsList precision, out TLKeyValuePairsList recall, out TLKeyValuePairsList avgPrecision, out TLKeyValuePairsList meanAvgPrecision)
{
TLLinksList links = MetricsUtil.GetLinksAtRecall(sims, oracle, level);
int numCorrect = 0;
int totalRead = 0;
double totalAvgPrecision = 0.0;
foreach (TLSingleLink link in links)
{
totalRead++;
if (oracle.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId))
{
numCorrect++;
totalAvgPrecision += numCorrect / (double)totalRead;
}
}
// temporary
precision = new TLKeyValuePairsList();
precision.Add(new KeyValuePair <string, double>("#TOTAL", numCorrect / Convert.ToDouble(links.Count)));
recall = new TLKeyValuePairsList();
recall.Add(new KeyValuePair <string, double>("#TOTAL", Math.Ceiling(oracle.Count * RecallLevelUtil.RecallValue(level)) / oracle.Count));
avgPrecision = new TLKeyValuePairsList();
avgPrecision.Add(new KeyValuePair <string, double>("#TOTAL", totalAvgPrecision / oracle.Count));
meanAvgPrecision = new TLKeyValuePairsList();
meanAvgPrecision.Add(new KeyValuePair <string, double>("#TOTAL", MeanAveragePrecision.Compute(Similarities.CreateMatrix(links), oracle)));
}
public static DataSetPairs Compute(TLSimilarityMatrix sims, TLSimilarityMatrix oracle, RecallLevel recall)
{
TLSimilarityMatrix matrix = Similarities.CreateMatrix(MetricsUtil.GetLinksAtRecall(sims, oracle, recall));
matrix.Threshold = double.MinValue;
DataSetPairs pairs = new DataSetPairs();
foreach (string sourceArtifact in oracle.SourceArtifactsIds)
{
TLLinksList links = matrix.GetLinksAboveThresholdForSourceArtifact(sourceArtifact);
links.Sort();
int totalCorrect = oracle.GetLinksAboveThresholdForSourceArtifact(sourceArtifact).Count;
int numCorrect = 0;
int totalRead = 0;
double totalAvgPrecision = 0.0;
foreach (TLSingleLink link in links)
{
totalRead++;
if (oracle.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId))
{
numCorrect++;
totalAvgPrecision += numCorrect / (double)totalRead;
}
}
pairs.PrecisionData.Add(new KeyValuePair <string, double>(sourceArtifact, numCorrect / Convert.ToDouble(links.Count)));
pairs.RecallData.Add(new KeyValuePair <string, double>(sourceArtifact, Convert.ToDouble(numCorrect) / totalCorrect));
pairs.AveragePrecisionData.Add(new KeyValuePair <string, double>(sourceArtifact, totalAvgPrecision / totalCorrect));
}
pairs.MeanAveragePrecisionData.Add(new KeyValuePair <string, double>("#TOTAL", DataSetPairsCollection.CalculateAverage(pairs.AveragePrecisionData)));
return(pairs);
}
/// <summary>
/// Returns links for the desired recall level.
/// </summary>
/// <param name="matrix">Candidate matrix</param>
/// <param name="answerMatrix">Answer matrix</param>
/// <param name="level">Desired recall level</param>
/// <returns>List of links at desired recall</returns>
public static TLLinksList GetLinksAtRecall(TLSimilarityMatrix matrix, TLSimilarityMatrix answerMatrix, double level)
{
if (level <= 0.0 || level > 1.0)
{
throw new DevelopmentKitException("Recall level must be between 0 and 1.");
}
double totalCorrect = answerMatrix.Count * level;
int numCorrect = 0;
TLLinksList links = matrix.AllLinks;
links.Sort();
TLLinksList newLinks = new TLLinksList();
while (links.Count > 0 && numCorrect < totalCorrect)
{
TLSingleLink link = links[0];
if (answerMatrix.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId))
{
numCorrect++;
}
newLinks.Add(link);
links.RemoveAt(0);
}
return(newLinks);
}
private static void ReadSimilarityMatrixToExcelWorksheet(TLSimilarityMatrix similarityMatrix, TLSimilarityMatrix answerMatrix, Excel.Worksheet xlWorkSheet)
{
//header
int row = 1;
xlWorkSheet.Cells[row, 1] = "Source Artifact Id";
xlWorkSheet.Cells[row, 2] = "Target Artifact Id";
xlWorkSheet.Cells[row, 3] = "Probability";
xlWorkSheet.Cells[row, 4] = "Is correct";
row++;
foreach (string sourceArtifact in similarityMatrix.SourceArtifactsIds)
{
var traceLinks = similarityMatrix.GetLinksAboveThresholdForSourceArtifact(sourceArtifact);
traceLinks.Sort();
foreach (TLSingleLink link in traceLinks)
{
xlWorkSheet.Cells[row, 1] = link.SourceArtifactId;
xlWorkSheet.Cells[row, 2] = link.TargetArtifactId;
xlWorkSheet.Cells[row, 3] = link.Score;
xlWorkSheet.Cells[row, 4] = (answerMatrix.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId)) ? "1" : "0";
row++;
}
}
}
internal static void CompareResults(TLSimilarityMatrix oracle, TLSimilarityMatrix results, IEnumerable <string> rawMethods)
{
Console.WriteLine("Comparing results...");
Assert.AreEqual(oracle.Count, results.Count);
foreach (string oracleMethod in oracle.GetSetOfTargetArtifactIdsAboveThresholdForSourceArtifact("trace"))
{
string rawMethod = rawMethods.ElementAt(Convert.ToInt32(oracleMethod) - 1);
string method = rawMethod.Substring(0, rawMethod.IndexOf('('));
//Console.WriteLine(oracleMethod + ": " + method);
Assert.IsTrue(results.IsLinkAboveThreshold("trace", method));
Assert.AreEqual(oracle.GetScoreForLink("trace", oracleMethod), results.GetScoreForLink("trace", method), Settings.Default.DoublePrecision);
}
}
/// <summary>
/// Computes the recall of the given similarity matrix using the answer matrix provided.
/// </summary>
protected override void ComputeImplementation()
{
_oracle.Threshold = 0;
int correct = 0;
foreach (TLSingleLink link in _matrix.AllLinks)
{
if (_oracle.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId))
{
correct++;
}
}
Results = new TraceLabSDK.SerializableDictionary <string, double>();
Results.Add("Recall", correct / (double)_oracle.Count);
}
/// <summary>
/// Computes the effectiveness all measure of the given similarity matrix using the answer matrix provided.
/// </summary>
protected override void ComputeImplementation()
{
_oracle.Threshold = 0;
Results = new SerializableDictionary <string, double>();
foreach (string query in _oracle.SourceArtifactsIds)
{
TLLinksList links = _matrix.GetLinksAboveThresholdForSourceArtifact(query);
links.Sort();
for (int i = 0; i < links.Count; i++)
{
if (_oracle.IsLinkAboveThreshold(query, links[i].TargetArtifactId))
{
Results.Add(String.Format("{0}_{1}", query, links[i].TargetArtifactId), i);
}
}
}
}
public static TLSimilarityMatrix Compute(TLSimilarityMatrix sims, TLSimilarityMatrix relationships, TLSimilarityMatrix feedback)
{
// new matrix
TLSimilarityMatrix newMatrix = new TLSimilarityMatrix();
#if UseDelta
// compute delta
double delta = SharedUtils.ComputeDelta(sims);
#endif
// make sure the entire list is sorted
TLLinksList links = sims.AllLinks;
links.Sort();
// end condition
int correct = 0;
// iterate over each source-target pair
while (links.Count > 0 && correct < feedback.Count)
{
// get link at top of list
TLSingleLink link = links[0];
// check feedback
if (feedback.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId))
{
correct++;
// update related links
for (int i = 1; i < links.Count; i++)
{
if (link.SourceArtifactId.Equals(links[i].SourceArtifactId) &&
relationships.IsLinkAboveThreshold(link.TargetArtifactId, links[i].TargetArtifactId))
{
#if UseDelta
links[i].Score += links[i].Score * delta;
#else
links[i].Score += links[i].Score * 0.1;
#endif
}
}
}
// remove link
newMatrix.AddLink(link.SourceArtifactId, link.TargetArtifactId, link.Score);
links.RemoveAt(0);
// reorder links
links.Sort();
}
return(newMatrix);
}
/// <summary>
/// Computes the precision-recall curve of the given similarity matrix using the answer matrix provided.
/// </summary>
protected override void ComputeImplementation()
{
_oracle.Threshold = 0;
int correct = 0;
TLLinksList links = _matrix.AllLinks;
links.Sort();
Results = new SerializableDictionary <string, double>();
for (int linkNumber = 1; linkNumber <= links.Count; linkNumber++)
{
if (_oracle.IsLinkAboveThreshold(links[linkNumber - 1].SourceArtifactId, links[linkNumber - 1].TargetArtifactId))
{
correct++;
}
Results.Add(String.Format(_precisionFormat, linkNumber), correct / (double)linkNumber);
Results.Add(String.Format(_recallFormat, linkNumber), correct / (double)_oracle.Count);
}
}
/// <summary>
/// Computes the recall of each source artifact in the similarity matrix using the answer matrix provided.
/// </summary>
protected override void ComputeImplementation()
{
SerializableDictionary <string, double> sourceRecall = new SerializableDictionary <string, double>();
_oracle.Threshold = 0;
foreach (string sourceArtifact in _oracle.SourceArtifactsIds)
{
TLLinksList links = _matrix.GetLinksAboveThresholdForSourceArtifact(sourceArtifact);
int correct = 0;
foreach (TLSingleLink link in links)
{
if (_oracle.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId))
{
correct++;
}
}
sourceRecall.Add(sourceArtifact, correct / (double)_oracle.GetCountOfLinksAboveThresholdForSourceArtifact(sourceArtifact));
}
Results = sourceRecall;
}
private static void WriteSims(ref Info info, CSMR13DataSet dataset, TLSimilarityMatrix oracle, string model)
{
TextWriter Output = File.CreateText(info.OutputDirectory + @"\CheckLinkOrder\" + SharedUtils.CleanFileName(dataset.Name) + "." + model + ".txt");
TLSimilarityMatrix sims = Similarities.Import(info.ResultsDirectory.FullName + @"\" + SharedUtils.CleanFileName(dataset.Name) + @"\sims\" + model + ".sims");
TLLinksList simList = sims.AllLinks;
simList.Sort();
int pos = 1;
foreach (TLSingleLink link in simList)
{
if (oracle.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId))
{
Output.WriteLine("[{0}]\t{1}\t{2}\t{3}", pos, link.SourceArtifactId, link.TargetArtifactId, link.Score);
}
pos++;
}
Output.Flush();
Output.Close();
}
/// <summary>
/// Called from MetricComputation
/// </summary>
protected override void ComputeImplementation()
{
Results = new SerializableDictionary <string, double>();
double sumOfPrecisions = 0.0;
int currentLink = 0;
int correctSoFar = 0;
TLLinksList links = _matrix.AllLinks;
links.Sort();
foreach (TLSingleLink link in links)
{
currentLink++;
if (_oracle.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId))
{
correctSoFar++;
sumOfPrecisions += correctSoFar / (double)currentLink;
}
}
Results.Add("AveragePrecision", sumOfPrecisions / _oracle.AllLinks.Count);
}
public static TLLinksList GetLinksAtRecall(TLSimilarityMatrix sims, TLSimilarityMatrix oracle, RecallLevel level)
{
double totalCorrect = oracle.Count * RecallLevelUtil.RecallValue(level);
int numCorrect = 0;
TLLinksList list = new TLLinksList();
TLLinksList links = sims.AllLinks;
links.Sort();
while (links.Count > 0 && numCorrect < totalCorrect)
{
TLSingleLink link = links[0];
if (oracle.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId))
{
numCorrect++;
}
list.Add(link);
links.RemoveAt(0);
}
return(list);
}
/// <summary>
/// Called from MetricComputation
/// </summary>
protected override void ComputeImplementation()
{
Results = new SerializableDictionary <string, double>();
foreach (string sourceID in _oracle.SourceArtifactsIds)
{
double sumOfPrecisions = 0.0;
int currentLink = 0;
int correctSoFar = 0;
TLLinksList links = _matrix.GetLinksAboveThresholdForSourceArtifact(sourceID);
links.Sort();
foreach (TLSingleLink link in links)
{
currentLink++;
if (_oracle.IsLinkAboveThreshold(sourceID, link.TargetArtifactId))
{
correctSoFar++;
sumOfPrecisions += correctSoFar / (double)currentLink;
}
}
Results.Add(sourceID, sumOfPrecisions / _oracle.GetCountOfLinksAboveThresholdForSourceArtifact(sourceID));
}
}
private static double Calculate(string sourceArtifactId, TLLinksList resultList, TLSimilarityMatrix answerMatrix)
{
resultList.Sort();
int correct = 0;
Double totalAvgPrecision = 0.0;
int totalDocumentsRead = 0;
foreach (TLSingleLink link in resultList)
{
totalDocumentsRead++;
//check if this is relevant link
if (answerMatrix.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId))
{
correct++;
Double precisionAtCurrentIteration = (double)correct / totalDocumentsRead;
totalAvgPrecision += precisionAtCurrentIteration;
}
}
//int numberOfRelevant = answerMatrix.GetCountOfLinksAboveThresholdForSourceArtifact(sourceArtifactId);
return(totalAvgPrecision);
}
private static void ReadSimilarityMatrixToFile(TLSimilarityMatrix similarityMatrix, TLSimilarityMatrix answerMatrix, System.IO.TextWriter writeFile)
{
//header
writeFile.WriteLine("Source Artifact Id,Target Artifact Id,Probability,Is correct");
foreach (string sourceArtifact in similarityMatrix.SourceArtifactsIds)
{
var traceLinks = similarityMatrix.GetLinksAboveThresholdForSourceArtifact(sourceArtifact);
traceLinks.Sort();
foreach (TLSingleLink link in traceLinks)
{
writeFile.WriteLine("{0},{1},{2},{3}", link.SourceArtifactId, link.TargetArtifactId, link.Score, (answerMatrix.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId)) ? "1" : "0");
}
}
}
public static DatasetResults Calculate(ref TLSimilarityMatrix sims, ref TLSimilarityMatrix goldset, Dictionary <int, string> qmap, string ModelName)
{
TLKeyValuePairsList allall = new TLKeyValuePairsList();
TLKeyValuePairsList allbest = new TLKeyValuePairsList();
TLKeyValuePairsList bugall = new TLKeyValuePairsList();
TLKeyValuePairsList bugbest = new TLKeyValuePairsList();
TLKeyValuePairsList featall = new TLKeyValuePairsList();
TLKeyValuePairsList featbest = new TLKeyValuePairsList();
TLKeyValuePairsList patchall = new TLKeyValuePairsList();
TLKeyValuePairsList patchbest = new TLKeyValuePairsList();
sims.Threshold = Double.MinValue;
foreach (KeyValuePair <int, string> qmapKVP in qmap)
{
TLLinksList simList = sims.GetLinksAboveThresholdForSourceArtifact(qmapKVP.Key.ToString());
simList.Sort();
bool best = false;
for (int i = 0; i < simList.Count; i++)
{
if (goldset.IsLinkAboveThreshold(simList[i].SourceArtifactId, simList[i].TargetArtifactId))
{
KeyValuePair <string, double> recovered = new KeyValuePair <string, double>(simList[i].SourceArtifactId + "_" + simList[i].TargetArtifactId, i);
allall.Add(recovered);
if (!best)
{
allbest.Add(recovered);
best = true;
if (qmapKVP.Value == Trace.GetFeatureSetType(FeatureSet.Bugs))
{
bugbest.Add(recovered);
}
else if (qmapKVP.Value == Trace.GetFeatureSetType(FeatureSet.Features))
{
featbest.Add(recovered);
}
else if (qmapKVP.Value == Trace.GetFeatureSetType(FeatureSet.Patch))
{
patchbest.Add(recovered);
}
}
if (qmapKVP.Value == Trace.GetFeatureSetType(FeatureSet.Bugs))
{
bugall.Add(recovered);
}
else if (qmapKVP.Value == Trace.GetFeatureSetType(FeatureSet.Features))
{
featall.Add(recovered);
}
else if (qmapKVP.Value == Trace.GetFeatureSetType(FeatureSet.Patch))
{
patchall.Add(recovered);
}
}
}
}
List <SummaryData> alldata = new List <SummaryData>();
alldata.Add(CreateSummaryData(allall, "All (all)"));
alldata.Add(CreateSummaryData(bugall, "Bugs (all)"));
alldata.Add(CreateSummaryData(featall, "Features (all)"));
alldata.Add(CreateSummaryData(patchall, "Patches (all)"));
List <SummaryData> bestdata = new List <SummaryData>();
bestdata.Add(CreateSummaryData(allbest, "All (best)"));
bestdata.Add(CreateSummaryData(bugbest, "Bugs (best)"));
bestdata.Add(CreateSummaryData(featbest, "Features (best)"));
bestdata.Add(CreateSummaryData(patchbest, "Patches (best)"));
List <Metric> data = new List <Metric>();
data.Add(new EffectivenessMetric(alldata, 0.0, "none", ModelName + " all"));
data.Add(new EffectivenessMetric(bestdata, 0.0, "none", ModelName + " best"));
return(new DatasetResults("", data));
}
/// <summary>
/// Private method to export a TLSimilarityMatrix to CSV with an additional column for correct links
/// 0 - incorrect link
/// 1 - correct link
/// </summary>
/// <param name="similarityMatrix">Candidate Matrix</param>
/// <param name="answerMatrix">Answer Matrix</param>
/// <param name="writeFile">Open TextWriter stream</param>
private static void WriteMatrixCSVWithCorrectness(TLSimilarityMatrix similarityMatrix, TLSimilarityMatrix answerMatrix, System.IO.TextWriter writeFile)
{
//header
writeFile.WriteLine("Source Artifact Id,Target Artifact Id,Probability,Is correct");
TLLinksList traceLinks = similarityMatrix.AllLinks;
traceLinks.Sort();
foreach (TLSingleLink link in traceLinks)
{
writeFile.WriteLine("{0},{1},{2},{3}", link.SourceArtifactId, link.TargetArtifactId, link.Score, (answerMatrix.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId)) ? "1" : "0");
}
}