/// <summary>
/// Runs a model.
/// </summary>
/// <param name="runner">
/// The runner.
/// </param>
/// <param name="modelName">
/// The results file name.
/// </param>
/// <param name="numIterations">
/// The number of iterations.
/// </param>
/// <param name="useGoldLabels">Use gold labels in training.</param>
/// <returns>
/// The dictionary of metrics.
/// </returns>
public static Dictionary <string, object> RunModel(RunnerBase runner, string modelName, int numIterations = 20,
bool useGoldLabels = false)
{
Console.WriteLine($@"Running {modelName}");
var metrics = runner.RunModel(numIterations, useGoldLabels);
return(metrics.ToDictionary(kvp => kvp.Key.ToString(), kvp => kvp.Value));
}
/// <summary>
/// Gets <see cref="RunnerBase" /> results.
/// </summary>
/// <param name="runner">
/// The runner.
/// </param>
/// <returns>
/// The dictionary of results.
/// </returns>
public static Dictionary <string, object> GetTrainingResults(RunnerBase runner)
{
var result = new Dictionary <string, object>();
var confusionMatrix = PlotData.GetConfusionMatrix(runner.ConfusionMatrix, runner.DataMapping,
PlotData.ConfusionMatrixRowLabel, PlotData.ConfusionMatrixColLabel2);
result["ConfusionMatrix"] = confusionMatrix;
var confusionMatrixPercentage = PlotData.GetConfusionMatrix(runner.ConfusionMatrix, runner.DataMapping,
PlotData.ConfusionMatrixRowLabel, PlotData.ConfusionMatrixColLabel2, true);
result["ConfusionMatrixPercentage"] = confusionMatrixPercentage;
var tweetMatrix = PlotData.GetTweetMatrix(runner.TweetMatrix, runner.DataMapping,
PlotData.ConfusionMatrixRowLabel, "Inferred True");
result["Accuracy"] = runner.Accuracy;
result["AverageRecall"] = runner.AverageRecall;
return(result);
}
/// <summary>
/// Gets the worker metrics for a data set
/// </summary>
/// <param name="data">
/// The data.
/// </param>
/// <param name="runner">
/// The runner.
/// </param>
/// <param name="maxNumberWorkers">
/// The maximum number of workers for which to get metrics.
/// </param>
/// <returns>
/// The metrics.
/// </returns>
public static Dictionary <string, Dictionary <string, object> > GetWorkerMetrics(CrowdData data,
RunnerBase runner, int maxNumberWorkers = 20)
{
var metricsForWorkers = new List <Metric> {
Metric.Count, Metric.Accuracy, Metric.ConfusionMatrix
};
var result = new Dictionary <string, Dictionary <string, object> >();
var mapping = runner != null ? runner.DataMapping : new CrowdDataMapping(data, LabelValuesToString);
foreach (var metric in metricsForWorkers)
{
result[metric.ToString()] = new Dictionary <string, object>();
}
var confusionMatrixKey = Metric.ConfusionMatrix.ToString();
var confusionMatrixPercentageKey = $"{confusionMatrixKey}Percentage";
result[confusionMatrixPercentageKey] = new Dictionary <string, object>();
var labelsGroupedByWorker = data.CrowdLabels.GroupBy(cd => cd.WorkerId);
var trueLabels = runner?.Predictions;
const string RowLabel = PlotData.ConfusionMatrixRowLabel;
const string ColumnLabel = PlotData.ConfusionMatrixColLabel;
foreach (var worker in labelsGroupedByWorker)
{
var workerId = worker.Key;
var workerLabels = worker.Distinct(CrowdData.WorkerTweetEqualityComparer.Instance)
.ToDictionary(lab => lab.TweetId, lab => lab.WorkerLabel);
var workerMetrics = GetMetrics(mapping, workerLabels, trueLabels);
foreach (var metric in metricsForWorkers)
{
if (metric == Metric.ConfusionMatrix)
{
var mat = workerMetrics[metric];
var confMat = PlotData.GetConfusionMatrix((double[, ])mat, mapping, RowLabel, ColumnLabel);
result[confusionMatrixKey][workerId] = confMat;
var matPerc =
PlotData.GetConfusionMatrix((double[, ])mat, mapping, RowLabel, ColumnLabel, true);
result[confusionMatrixPercentageKey][workerId] = matPerc;
}
else
{
result[metric.ToString()][workerId] = workerMetrics[metric];
}
}
}
// Limit the confusion matrices to the more prominent workers.
var prominentWorkers = new HashSet <string>(result[Metric.Count.ToString()]
.OrderByDescending(kvp => (int)kvp.Value).Take(maxNumberWorkers).Select(kvp => kvp.Key));
result[confusionMatrixKey] = result[confusionMatrixKey].Where(kvp1 => prominentWorkers.Contains(kvp1.Key))
.ToDictionary(kvp1 => kvp1.Key, kvp1 => kvp1.Value);
result[confusionMatrixPercentageKey] = result[confusionMatrixPercentageKey]
.Where(kvp1 => prominentWorkers.Contains(kvp1.Key))
.ToDictionary(kvp1 => kvp1.Key, kvp1 => kvp1.Value);
return(result);
}
