public override void Compute()
{
TLKeyValuePairsList precision = (TLKeyValuePairsList)Workspace.Load("PrecisionData");
TLKeyValuePairsList recall = (TLKeyValuePairsList)Workspace.Load("RecallData");
TLKeyValuePairsList avgprecision = (TLKeyValuePairsList)Workspace.Load("AvgPrecisionData");
StringBuilder sb = new StringBuilder();
sb.AppendFormat("{0}\n", _config.Title);
sb.AppendLine("--------------------------------------------------");
sb.AppendFormat("Precision Data:\nMin:\t{0}\nMax:\t{1}\nAvg:\t{2}\n",
GetMin(precision),
GetMax(precision),
GetAvg(precision)
);
sb.AppendLine("--------------------------------------------------");
sb.AppendFormat("Recall Data:\nMin:\t{0}\nMax:\t{1}\nAvg:\t{2}\n",
GetMin(recall),
GetMax(recall),
GetAvg(recall)
);
sb.AppendLine("--------------------------------------------------");
sb.AppendFormat("Avg Precision Data:\nMin:\t{0}\nMax:\t{1}\nAvg:\t{2}\n",
GetMin(avgprecision),
GetMax(avgprecision),
GetAvg(avgprecision)
);
sb.AppendLine("--------------------------------------------------");
/*
* SimpleResultsWindow window = new SimpleResultsWindow();
* window.PrintToScreen(sb.ToString());
* window.ShowDialog();
*/
Logger.Info(sb.ToString());
}
public static void ComputeMetrics(TLSimilarityMatrix sims, TLSimilarityMatrix oracle, RecallLevel level,
out TLKeyValuePairsList precision, out TLKeyValuePairsList recall, out TLKeyValuePairsList avgPrecision, out TLKeyValuePairsList meanAvgPrecision)
{
TLLinksList links = MetricsUtil.GetLinksAtRecall(sims, oracle, level);
int numCorrect = 0;
int totalRead = 0;
double totalAvgPrecision = 0.0;
foreach (TLSingleLink link in links)
{
totalRead++;
if (oracle.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId))
{
numCorrect++;
totalAvgPrecision += numCorrect / (double) totalRead;
}
}
// temporary
precision = new TLKeyValuePairsList();
precision.Add(new KeyValuePair<string, double>("#TOTAL", numCorrect / Convert.ToDouble(links.Count)));
recall = new TLKeyValuePairsList();
recall.Add(new KeyValuePair<string, double>("#TOTAL", Math.Ceiling(oracle.Count * RecallLevelUtil.RecallValue(level)) / oracle.Count));
avgPrecision = new TLKeyValuePairsList();
avgPrecision.Add(new KeyValuePair<string, double>("#TOTAL", totalAvgPrecision / oracle.Count));
meanAvgPrecision = new TLKeyValuePairsList();
meanAvgPrecision.Add(new KeyValuePair<string,double>("#TOTAL", MeanAveragePrecision.Compute(Similarities.CreateMatrix(links), oracle)));
}
public static void ComputeMetrics(TLSimilarityMatrix sims, TLSimilarityMatrix oracle, RecallLevel level,
out TLKeyValuePairsList precision, out TLKeyValuePairsList recall, out TLKeyValuePairsList avgPrecision, out TLKeyValuePairsList meanAvgPrecision)
{
TLLinksList links = MetricsUtil.GetLinksAtRecall(sims, oracle, level);
int numCorrect = 0;
int totalRead = 0;
double totalAvgPrecision = 0.0;
foreach (TLSingleLink link in links)
{
totalRead++;
if (oracle.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId))
{
numCorrect++;
totalAvgPrecision += numCorrect / (double)totalRead;
}
}
// temporary
precision = new TLKeyValuePairsList();
precision.Add(new KeyValuePair <string, double>("#TOTAL", numCorrect / Convert.ToDouble(links.Count)));
recall = new TLKeyValuePairsList();
recall.Add(new KeyValuePair <string, double>("#TOTAL", Math.Ceiling(oracle.Count * RecallLevelUtil.RecallValue(level)) / oracle.Count));
avgPrecision = new TLKeyValuePairsList();
avgPrecision.Add(new KeyValuePair <string, double>("#TOTAL", totalAvgPrecision / oracle.Count));
meanAvgPrecision = new TLKeyValuePairsList();
meanAvgPrecision.Add(new KeyValuePair <string, double>("#TOTAL", MeanAveragePrecision.Compute(Similarities.CreateMatrix(links), oracle)));
}
public DataSetPairs()
{
Name = String.Empty;
PrecisionData = new TLKeyValuePairsList();
RecallData = new TLKeyValuePairsList();
AveragePrecisionData = new TLKeyValuePairsList();
MeanAveragePrecisionData = new TLKeyValuePairsList();
}
public DataSetPairs()
{
Name = String.Empty;
PrecisionData = new TLKeyValuePairsList();
RecallData = new TLKeyValuePairsList();
AveragePrecisionData = new TLKeyValuePairsList();
MeanAveragePrecisionData = new TLKeyValuePairsList();
}
private double GetAvg(TLKeyValuePairsList list)
{
double avg = 0.0;
foreach (KeyValuePair<string, double> val in list)
{
if (!Double.IsNaN(val.Value))
avg += val.Value;
}
return avg / list.Count;
}
private SummaryData CreateSummaryData(TLKeyValuePairsList results, string modelName)
{
List <double> data = new List <double>();
foreach (KeyValuePair <string, double> kvp in results)
{
data.Add(kvp.Value);
}
return(new SummaryData(modelName, data.ToArray()));
}
private double GetMin(TLKeyValuePairsList list)
{
double min = Double.MaxValue;
foreach (KeyValuePair <string, double> val in list)
{
if (!Double.IsNaN(val.Value) && val.Value < min)
{
min = val.Value;
}
}
return(min);
}
private double[] ExtractDataArray(TLKeyValuePairsList pairs)
{
List <double> array = new List <double>();
foreach (KeyValuePair <string, double> pair in pairs)
{
if (!Double.IsNaN(pair.Value))
{
array.Add(pair.Value);
}
}
return(array.ToArray());
}
private double GetAvg(TLKeyValuePairsList list)
{
double avg = 0.0;
foreach (KeyValuePair <string, double> val in list)
{
if (!Double.IsNaN(val.Value))
{
avg += val.Value;
}
}
return(avg / list.Count);
}
private double GetMax(TLKeyValuePairsList list)
{
double max = Double.MinValue;
foreach (KeyValuePair <string, double> val in list)
{
if (!Double.IsNaN(val.Value) && val.Value > max)
{
max = val.Value;
}
}
return(max);
}
/// <summary>
/// Computes the traceability between source and target artifacts using dictionary and American Corpus Term weigths.
/// </summary>
/// <param name="sourceArtifacts">The source artifacts.</param>
/// <param name="targetArtifacts">The target artifacts.</param>
/// <param name="dict">The dict.</param>
/// <param name="ancTermsWeights">The anc terms weights.</param>
/// <param name="config">The config.</param>
/// <returns>Similarity matrix with links between source and target artifacts</returns>
private static TLSimilarityMatrix ComputeTraceability(TLArtifactsCollection sourceArtifacts,
TLArtifactsCollection targetArtifacts,
TLDictionaryIndex dict,
TLKeyValuePairsList ancTermsWeights,
TracerConfig config)
{
if (sourceArtifacts == null)
{
throw new ComponentException("Received source artifacts are null!");
}
if (targetArtifacts == null)
{
throw new ComponentException("Received target artifacts are null!");
}
if (dict == null)
{
throw new ComponentException("Received dictionary index is null!");
}
if (ancTermsWeights == null)
{
throw new ComponentException("Received 'ancTermsWeights' is null!");
}
TLSimilarityMatrix similarityMatrix = new TLSimilarityMatrix();
ANCSearcher searcher = new ANCSearcher(SimilarityMetricFactory.GetSimiliarityMetric(config.SimilarityMetric));
// Iterates over all the source artifacts to determine the probabilities to target artifacts - by executing a search
foreach (TLArtifact sourceArtifact in sourceArtifacts.Values)
{
String query = sourceArtifact.Text;
// Executes the query
List<Result> results;
results = searcher.search(query, dict, PrepareANCData(ancTermsWeights));
// Iterates over the results and stores them in the matrix
foreach (Result r in results)
{
string targetArtifactId = r.ArtifactId;
similarityMatrix.AddLink(sourceArtifact.Id, targetArtifactId, r.Ranking);
}
}
return similarityMatrix;
}
/// <summary>
/// Computes the traceability between source and target artifacts using dictionary and American Corpus Term weigths.
/// </summary>
/// <param name="sourceArtifacts">The source artifacts.</param>
/// <param name="targetArtifacts">The target artifacts.</param>
/// <param name="dict">The dict.</param>
/// <param name="ancTermsWeights">The anc terms weights.</param>
/// <param name="config">The config.</param>
/// <returns>Similarity matrix with links between source and target artifacts</returns>
private static TLSimilarityMatrix ComputeTraceability(TLArtifactsCollection sourceArtifacts,
TLArtifactsCollection targetArtifacts,
TLDictionaryIndex dict,
TLKeyValuePairsList ancTermsWeights,
TracerConfig config)
{
if (sourceArtifacts == null)
{
throw new ComponentException("Received source artifacts are null!");
}
if (targetArtifacts == null)
{
throw new ComponentException("Received target artifacts are null!");
}
if (dict == null)
{
throw new ComponentException("Received dictionary index is null!");
}
if (ancTermsWeights == null)
{
throw new ComponentException("Received 'ancTermsWeights' is null!");
}
TLSimilarityMatrix similarityMatrix = new TLSimilarityMatrix();
ANCSearcher searcher = new ANCSearcher(SimilarityMetricFactory.GetSimiliarityMetric(config.SimilarityMetric));
// Iterates over all the source artifacts to determine the probabilities to target artifacts - by executing a search
foreach (TLArtifact sourceArtifact in sourceArtifacts.Values)
{
String query = sourceArtifact.Text;
// Executes the query
List <Result> results;
results = searcher.search(query, dict, PrepareANCData(ancTermsWeights));
// Iterates over the results and stores them in the matrix
foreach (Result r in results)
{
string targetArtifactId = r.ArtifactId;
similarityMatrix.AddLink(sourceArtifact.Id, targetArtifactId, r.Ranking);
}
}
return(similarityMatrix);
}
public override void Compute()
{
Logger.Trace("Start component ANCTracerComponent");
TLArtifactsCollection sourceArtifacts = (TLArtifactsCollection)Workspace.Load("sourceArtifacts");
TLArtifactsCollection targetArtifacts = (TLArtifactsCollection)Workspace.Load("targetArtifacts");
TLDictionaryIndex dict = (TLDictionaryIndex)Workspace.Load("dictionaryIndex");
TLKeyValuePairsList ancTermsWeights = (TLKeyValuePairsList)Workspace.Load("ancTermsWeights");
TracerConfig config = (TracerConfig)this.Configuration;
TLSimilarityMatrix similarityMatrix = ComputeTraceability(sourceArtifacts, targetArtifacts, dict, ancTermsWeights, config);
Workspace.Store("similarityMatrix", similarityMatrix);
Logger.Trace("Completed component ANCTracerComponent");
}
public static double CalculateAverage(TLKeyValuePairsList metrics)
{
double val = 0.0;
double non = 0.0;
foreach (KeyValuePair <string, double> metric in metrics)
{
if (!Double.IsNaN(metric.Value))
{
val += metric.Value;
}
else
{
non++;
}
}
return(val / (metrics.Count - non));
}
private static Dictionary <string, double> PrepareANCData(TLKeyValuePairsList metricContainer)
{
Dictionary <string, double> retCol = new Dictionary <string, double>();
foreach (KeyValuePair <string, double> metric in metricContainer)
{
if (retCol.ContainsKey(metric.Key) == true)
{
if (retCol[metric.Key] < metric.Value)
{
retCol[metric.Key] = metric.Value;
}
}
else
{
retCol.Add(metric.Key, metric.Value);
}
}
return(retCol);
}
private double GetMin(TLKeyValuePairsList list)
{
double min = Double.MaxValue;
foreach (KeyValuePair<string, double> val in list)
{
if (!Double.IsNaN(val.Value) && val.Value < min)
min = val.Value;
}
return min;
}
public static DatasetResults Calculate(ref TLSimilarityMatrix sims, ref TLSimilarityMatrix goldset, Dictionary <int, string> qmap, string ModelName)
{
TLKeyValuePairsList allall = new TLKeyValuePairsList();
TLKeyValuePairsList allbest = new TLKeyValuePairsList();
TLKeyValuePairsList bugall = new TLKeyValuePairsList();
TLKeyValuePairsList bugbest = new TLKeyValuePairsList();
TLKeyValuePairsList featall = new TLKeyValuePairsList();
TLKeyValuePairsList featbest = new TLKeyValuePairsList();
TLKeyValuePairsList patchall = new TLKeyValuePairsList();
TLKeyValuePairsList patchbest = new TLKeyValuePairsList();
sims.Threshold = Double.MinValue;
foreach (KeyValuePair <int, string> qmapKVP in qmap)
{
TLLinksList simList = sims.GetLinksAboveThresholdForSourceArtifact(qmapKVP.Key.ToString());
simList.Sort();
bool best = false;
for (int i = 0; i < simList.Count; i++)
{
if (goldset.IsLinkAboveThreshold(simList[i].SourceArtifactId, simList[i].TargetArtifactId))
{
KeyValuePair <string, double> recovered = new KeyValuePair <string, double>(simList[i].SourceArtifactId + "_" + simList[i].TargetArtifactId, i);
allall.Add(recovered);
if (!best)
{
allbest.Add(recovered);
best = true;
if (qmapKVP.Value == Trace.GetFeatureSetType(FeatureSet.Bugs))
{
bugbest.Add(recovered);
}
else if (qmapKVP.Value == Trace.GetFeatureSetType(FeatureSet.Features))
{
featbest.Add(recovered);
}
else if (qmapKVP.Value == Trace.GetFeatureSetType(FeatureSet.Patch))
{
patchbest.Add(recovered);
}
}
if (qmapKVP.Value == Trace.GetFeatureSetType(FeatureSet.Bugs))
{
bugall.Add(recovered);
}
else if (qmapKVP.Value == Trace.GetFeatureSetType(FeatureSet.Features))
{
featall.Add(recovered);
}
else if (qmapKVP.Value == Trace.GetFeatureSetType(FeatureSet.Patch))
{
patchall.Add(recovered);
}
}
}
}
List <SummaryData> alldata = new List <SummaryData>();
alldata.Add(CreateSummaryData(allall, "All (all)"));
alldata.Add(CreateSummaryData(bugall, "Bugs (all)"));
alldata.Add(CreateSummaryData(featall, "Features (all)"));
alldata.Add(CreateSummaryData(patchall, "Patches (all)"));
List <SummaryData> bestdata = new List <SummaryData>();
bestdata.Add(CreateSummaryData(allbest, "All (best)"));
bestdata.Add(CreateSummaryData(bugbest, "Bugs (best)"));
bestdata.Add(CreateSummaryData(featbest, "Features (best)"));
bestdata.Add(CreateSummaryData(patchbest, "Patches (best)"));
List <Metric> data = new List <Metric>();
data.Add(new EffectivenessMetric(alldata, 0.0, "none", ModelName + " all"));
data.Add(new EffectivenessMetric(bestdata, 0.0, "none", ModelName + " best"));
return(new DatasetResults("", data));
}
private double[] ExtractDataArray(TLKeyValuePairsList pairs)
{
List<double> array = new List<double>();
foreach (KeyValuePair<string, double> pair in pairs)
{
if (!Double.IsNaN(pair.Value))
{
array.Add(pair.Value);
}
}
return array.ToArray();
}
private static Dictionary<string, double> PrepareANCData(TLKeyValuePairsList metricContainer)
{
Dictionary<string, double> retCol = new Dictionary<string, double>();
foreach (KeyValuePair<string, double> metric in metricContainer)
{
if (retCol.ContainsKey(metric.Key) == true)
{
if (retCol[metric.Key] < metric.Value)
{
retCol[metric.Key] = metric.Value;
}
}
else
{
retCol.Add(metric.Key, metric.Value);
}
}
return retCol;
}
private SummaryData CreateSummaryData(TLKeyValuePairsList results, string modelName)
{
List<double> data = new List<double>();
foreach (KeyValuePair<string, double> kvp in results)
{
data.Add(kvp.Value);
}
return new SummaryData(modelName, data.ToArray());
}
private double GetMax(TLKeyValuePairsList list)
{
double max = Double.MinValue;
foreach (KeyValuePair<string, double> val in list)
{
if (!Double.IsNaN(val.Value) && val.Value > max)
max = val.Value;
}
return max;
}
public static DatasetResults Calculate(ref TLSimilarityMatrix sims, ref TLSimilarityMatrix goldset, Dictionary<int, string> qmap, string ModelName)
{
TLKeyValuePairsList allall = new TLKeyValuePairsList();
TLKeyValuePairsList allbest = new TLKeyValuePairsList();
TLKeyValuePairsList bugall = new TLKeyValuePairsList();
TLKeyValuePairsList bugbest = new TLKeyValuePairsList();
TLKeyValuePairsList featall = new TLKeyValuePairsList();
TLKeyValuePairsList featbest = new TLKeyValuePairsList();
TLKeyValuePairsList patchall = new TLKeyValuePairsList();
TLKeyValuePairsList patchbest = new TLKeyValuePairsList();
sims.Threshold = Double.MinValue;
foreach (KeyValuePair<int, string> qmapKVP in qmap)
{
TLLinksList simList = sims.GetLinksAboveThresholdForSourceArtifact(qmapKVP.Key.ToString());
simList.Sort();
bool best = false;
for (int i = 0; i < simList.Count; i++)
{
if (goldset.IsLinkAboveThreshold(simList[i].SourceArtifactId, simList[i].TargetArtifactId))
{
KeyValuePair<string, double> recovered = new KeyValuePair<string, double>(simList[i].SourceArtifactId + "_" + simList[i].TargetArtifactId, i);
allall.Add(recovered);
if (!best)
{
allbest.Add(recovered);
best = true;
if (qmapKVP.Value == Trace.GetFeatureSetType(FeatureSet.Bugs))
{
bugbest.Add(recovered);
}
else if (qmapKVP.Value == Trace.GetFeatureSetType(FeatureSet.Features))
{
featbest.Add(recovered);
}
else if (qmapKVP.Value == Trace.GetFeatureSetType(FeatureSet.Patch))
{
patchbest.Add(recovered);
}
}
if (qmapKVP.Value == Trace.GetFeatureSetType(FeatureSet.Bugs))
{
bugall.Add(recovered);
}
else if (qmapKVP.Value == Trace.GetFeatureSetType(FeatureSet.Features))
{
featall.Add(recovered);
}
else if (qmapKVP.Value == Trace.GetFeatureSetType(FeatureSet.Patch))
{
patchall.Add(recovered);
}
}
}
}
List<SummaryData> alldata = new List<SummaryData>();
alldata.Add(CreateSummaryData(allall, "All (all)"));
alldata.Add(CreateSummaryData(bugall, "Bugs (all)"));
alldata.Add(CreateSummaryData(featall, "Features (all)"));
alldata.Add(CreateSummaryData(patchall, "Patches (all)"));
List<SummaryData> bestdata = new List<SummaryData>();
bestdata.Add(CreateSummaryData(allbest, "All (best)"));
bestdata.Add(CreateSummaryData(bugbest, "Bugs (best)"));
bestdata.Add(CreateSummaryData(featbest, "Features (best)"));
bestdata.Add(CreateSummaryData(patchbest, "Patches (best)"));
List<Metric> data = new List<Metric>();
data.Add(new EffectivenessMetric(alldata, 0.0, "none", ModelName + " all"));
data.Add(new EffectivenessMetric(bestdata, 0.0, "none", ModelName + " best"));
return new DatasetResults("", data);
}
