/// <summary>
/// Gets the <see cref="BiasedCommunityModelRunner" /> results.
/// </summary>
/// <param name="runner">
/// The runner.
/// </param>
/// <returns>
/// The dictionary of results.
/// </returns>
public static Dictionary <string, object> GetBiasedCommunityTrainingResults(ModelRunnerBase runner)
{
var result = GetModelTrainingResults(runner);
if (runner is BiasedCommunityModelRunner biasedCommunityRunner)
{
var communityCpts = new Dictionary <string, Dirichlet[]>();
for (var i = 0; i < biasedCommunityRunner.CommunityCpt.Count; i++)
{
communityCpts[i.ToString()] = biasedCommunityRunner.CommunityCpt[i];
}
var plottableCommunityCpts = PlotData.GetWorkerCpts(
communityCpts,
runner.DataMapping);
var communityCptsPerc = PlotData.GetWorkerCpts(
communityCpts,
runner.DataMapping,
true);
result["CommunityCpts"] = plottableCommunityCpts;
result["CommunityCptsPerc"] = communityCptsPerc;
result["WorkerCommunities"] = biasedCommunityRunner.WorkerCommunities;
result["WorkerCommunityCounts"] = biasedCommunityRunner.WorkerCommunityCounts;
}
return(result);
}
/// <summary>
/// Gets the <see cref="BiasedWorkerModelRunner" /> results.
/// </summary>
/// <param name="runner">
/// The runner.
/// </param>
/// <param name="maximumNumberWorkers">
/// The maximum number of workers for whom to return results..
/// </param>
/// <returns>
/// The dictionary of results.
/// </returns>
public static Dictionary <string, object> GetBiasedWorkerTrainingResults(ModelRunnerBase runner,
int maximumNumberWorkers = 20)
{
var result = GetModelTrainingResults(runner);
if (runner is BiasedWorkerModelRunner biasedWorkerRunner)
{
var prominentWorkers = GetProminentWorkers(runner.DataMapping.Data, maximumNumberWorkers);
var prominentWorkerCpts = biasedWorkerRunner.WorkerCpt
.Where(kvp => prominentWorkers.Contains(kvp.Key)).ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
var workerCpts = PlotData.GetWorkerCpts(
prominentWorkerCpts,
runner.DataMapping);
var workerCptsPerc = PlotData.GetWorkerCpts(
prominentWorkerCpts,
runner.DataMapping,
true);
result["ProminentWorkerCpts"] = workerCpts;
result["ProminentWorkerCpts"] = workerCptsPerc;
}
return(result);
}
/// <summary>
/// Gets the <see cref="HonestWorkerRunner" /> results.
/// </summary>
/// <param name="runner">
/// The runner.
/// </param>
/// <returns>
/// The dictionary of results.
/// </returns>
public static Dictionary <string, object> GetHonestWorkerTrainingResults(ModelRunnerBase runner)
{
var result = GetModelTrainingResults(runner);
var abilityHistogram = PlotData.GetWorkerAbilities((HonestWorkerRunner)runner);
result["AbilityHistogram"] = abilityHistogram;
return(result);
}
/// <summary>
/// Gets <see cref="ModelRunnerBase" /> results.
/// </summary>
/// <param name="runner">
/// The runner.
/// </param>
/// <returns>
/// The dictionary of results.
/// </returns>
private static Dictionary <string, object> GetModelTrainingResults(ModelRunnerBase runner)
{
var result = GetTrainingResults(runner);
result["Evidence"] = runner.ModelEvidence;
result["BackgroundLabelProb"] = runner.BackgroundLabelProb.PseudoCount.ToArray();
result["Evidence"] = runner.AverageLogProbability;
return(result);
}
/// <summary>
/// Gets the validation results for the honest worker model.
/// </summary>
/// <param name="runner">
/// The runner.
/// </param>
/// <param name="validationMetrics">
/// The validation metrics.
/// </param>
/// <param name="maxNumberWorkers">
/// The maximum number of workers for whom to return validation results.
/// </param>
/// <returns>
/// The validation results.
/// </returns>
public static Dictionary <string, object> GetHonestWorkerValidationResults(ModelRunnerBase runner,
Dictionary <string, object> validationMetrics, int maxNumberWorkers)
{
var results = GetValidationResults(runner, validationMetrics);
// Scatter plot for ability versus accuracy
var workerMetrics = GetWorkerMetrics(runner.DataMapping.Data, null, maxNumberWorkers);
var accuracies = workerMetrics[Metric.Accuracy.ToString()];
var abilities = ((HonestWorkerRunner)runner.TrainingRunner).WorkerAbility;
var validationWorkersWithAbilities =
accuracies.Keys.Where(wid => abilities.ContainsKey(wid)).ToList();
var scatterPlot = validationWorkersWithAbilities
.Select(wid => new Point((double)accuracies[wid], abilities[wid].GetMean())).ToList();
results["Accuracy Ability"] = scatterPlot;
return(results);
}
/// <summary>
/// Gets the validation results.
/// </summary>
/// <param name="runner">
/// The runner.
/// </param>
/// <param name="validationMetrics">
/// The validation metrics.
/// </param>
/// <returns>
/// The validation results.
/// </returns>
public static Dictionary <string, object> GetValidationResults(ModelRunnerBase runner,
Dictionary <string, object> validationMetrics)
{
var results = new Dictionary <string, object>();
foreach (var kvp in validationMetrics)
{
var key = kvp.Key;
var value = kvp.Value;
if (kvp.Key == Metric.ConfusionMatrix.ToString())
{
value = PlotData.GetConfusionMatrix((double[, ])kvp.Value, runner.DataMapping,
PlotData.ConfusionMatrixRowLabel, PlotData.ConfusionMatrixColLabel2);
var valuePerc = PlotData.GetConfusionMatrix((double[, ])kvp.Value, runner.DataMapping,
PlotData.ConfusionMatrixRowLabel, PlotData.ConfusionMatrixColLabel2, true);
results[key + "Percentage"] = valuePerc;
}
results[key] = value;
}
return(results);
}
/// <summary>
/// Initializes a new instance of the <see cref="BiasedWorkerModelRunner"/> class.
/// </summary>
/// <param name="dataMapping">
/// The data mapping.
/// </param>
/// <param name="model">
/// The model.
/// </param>
/// <param name="trainingRunner">
/// The training runner.
/// </param>
public BiasedWorkerModelRunner(CrowdDataMapping dataMapping, BiasedWorkerModel model, ModelRunnerBase trainingRunner = null)
: base(dataMapping, model, trainingRunner)
{
}
/// <inheritdoc />
/// <param name="dataMapping">
/// The data mapping
/// </param>
/// <param name="model">
/// The model.
/// </param>
/// <param name="trainingRunner">
/// The training Runner. This should be null if
/// (a) we are running training, or
/// (b) we are running validation and the model has no training
/// </param>
protected ModelRunnerBase(CrowdDataMapping dataMapping, ModelBase model, ModelRunnerBase trainingRunner = null)
: base(dataMapping)
{
this.Model = model;
this.TrainingRunner = trainingRunner;
}
/// <summary>
/// Gets the <see cref="BiasedCommunityWordsRunner" /> results.
/// </summary>
/// <param name="runner">
/// The runner.
/// </param>
/// <returns>
/// The dictionary of results.
/// </returns>
public static Dictionary <string, object> GetBiasedCommunityWordsRunnerResults(ModelRunnerBase runner)
{
var result = GetBiasedCommunityTrainingResults(runner);
if (runner is BiasedCommunityWordsRunner biasedCommunityWordsRunner)
{
result["BackgroundLogProbWord"] = biasedCommunityWordsRunner.BackgroundLogProbWord;
var logProbWord = biasedCommunityWordsRunner.LogProbWord.ToDictionary(
kvp => runner.DataMapping.LabelValueToString[kvp.Key],
kvp => kvp.Value);
result["LogProbWord"] = logProbWord;
var relativeLogProbWord = biasedCommunityWordsRunner.RelativeLogProbWord.ToDictionary(
kvp => runner.DataMapping.LabelValueToString[kvp.Key],
kvp => kvp.Value);
result["RelativeLogProbWord"] = relativeLogProbWord;
var orderedLogProbWord = biasedCommunityWordsRunner.OrderedLogProbWord.ToDictionary(
kvp => runner.DataMapping.LabelValueToString[kvp.Key],
kvp => kvp.Value);
result["OrderedLogProbWord"] = orderedLogProbWord;
var orderedRelativeLogProbWord = biasedCommunityWordsRunner.OrderedRelativeLogProbWord.ToDictionary(
kvp => runner.DataMapping.LabelValueToString[kvp.Key],
kvp => kvp.Value.ToDictionary(kvp1 => kvp1.Key, kvp1 => kvp1.Value));
result["OrderedRelativeLogProbWord"] = orderedRelativeLogProbWord;
result["TopTenWords"] = orderedRelativeLogProbWord.ToDictionary(
kvp => kvp.Key,
kvp => kvp.Value.OrderByDescending(kvp1 => kvp1.Value).Take(10).Select(kvp1 => Regex.Replace(kvp1.Key, @"[\d-]", string.Empty)).ToList());
}
return(result);
}
/// <summary>
/// Initializes a new instance of the <see cref="HonestWorkerRunner"/> class.
/// </summary>
/// <param name="dataMapping">
/// The data mapping.
/// </param>
/// <param name="model">
/// The model.
/// </param>
/// <param name="trainingRunner">
/// The training runner.
/// </param>
public HonestWorkerRunner(CrowdDataMapping dataMapping, HonestWorkerModel model, ModelRunnerBase trainingRunner = null)
: base(dataMapping, model, trainingRunner)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="BiasedCommunityWordsRunner"/> class.
/// </summary>
/// <param name="dataMapping">
/// The data mapping.
/// </param>
/// <param name="model">
/// The model.
/// </param>
/// <param name="trainingRunner">
/// The training runner.
/// </param>
public BiasedCommunityWordsRunner(CrowdDataWithTextMapping dataMapping, BiasedCommunityWordsModel model, ModelRunnerBase trainingRunner = null)
: base(dataMapping, model, trainingRunner)
{
}