/// <summary> /// Gets the <see cref="BiasedCommunityModelRunner" /> results. /// </summary> /// <param name="runner"> /// The runner. /// </param> /// <returns> /// The dictionary of results. /// </returns> public static Dictionary <string, object> GetBiasedCommunityTrainingResults(ModelRunnerBase runner) { var result = GetModelTrainingResults(runner); if (runner is BiasedCommunityModelRunner biasedCommunityRunner) { var communityCpts = new Dictionary <string, Dirichlet[]>(); for (var i = 0; i < biasedCommunityRunner.CommunityCpt.Count; i++) { communityCpts[i.ToString()] = biasedCommunityRunner.CommunityCpt[i]; } var plottableCommunityCpts = PlotData.GetWorkerCpts( communityCpts, runner.DataMapping); var communityCptsPerc = PlotData.GetWorkerCpts( communityCpts, runner.DataMapping, true); result["CommunityCpts"] = plottableCommunityCpts; result["CommunityCptsPerc"] = communityCptsPerc; result["WorkerCommunities"] = biasedCommunityRunner.WorkerCommunities; result["WorkerCommunityCounts"] = biasedCommunityRunner.WorkerCommunityCounts; } return(result); }
/// <summary> /// Gets the <see cref="BiasedWorkerModelRunner" /> results. /// </summary> /// <param name="runner"> /// The runner. /// </param> /// <param name="maximumNumberWorkers"> /// The maximum number of workers for whom to return results.. /// </param> /// <returns> /// The dictionary of results. /// </returns> public static Dictionary <string, object> GetBiasedWorkerTrainingResults(ModelRunnerBase runner, int maximumNumberWorkers = 20) { var result = GetModelTrainingResults(runner); if (runner is BiasedWorkerModelRunner biasedWorkerRunner) { var prominentWorkers = GetProminentWorkers(runner.DataMapping.Data, maximumNumberWorkers); var prominentWorkerCpts = biasedWorkerRunner.WorkerCpt .Where(kvp => prominentWorkers.Contains(kvp.Key)).ToDictionary(kvp => kvp.Key, kvp => kvp.Value); var workerCpts = PlotData.GetWorkerCpts( prominentWorkerCpts, runner.DataMapping); var workerCptsPerc = PlotData.GetWorkerCpts( prominentWorkerCpts, runner.DataMapping, true); result["ProminentWorkerCpts"] = workerCpts; result["ProminentWorkerCpts"] = workerCptsPerc; } return(result); }
/// <summary> /// Gets the <see cref="HonestWorkerRunner" /> results. /// </summary> /// <param name="runner"> /// The runner. /// </param> /// <returns> /// The dictionary of results. /// </returns> public static Dictionary <string, object> GetHonestWorkerTrainingResults(ModelRunnerBase runner) { var result = GetModelTrainingResults(runner); var abilityHistogram = PlotData.GetWorkerAbilities((HonestWorkerRunner)runner); result["AbilityHistogram"] = abilityHistogram; return(result); }
/// <summary> /// Gets <see cref="ModelRunnerBase" /> results. /// </summary> /// <param name="runner"> /// The runner. /// </param> /// <returns> /// The dictionary of results. /// </returns> private static Dictionary <string, object> GetModelTrainingResults(ModelRunnerBase runner) { var result = GetTrainingResults(runner); result["Evidence"] = runner.ModelEvidence; result["BackgroundLabelProb"] = runner.BackgroundLabelProb.PseudoCount.ToArray(); result["Evidence"] = runner.AverageLogProbability; return(result); }
/// <summary> /// Gets the validation results for the honest worker model. /// </summary> /// <param name="runner"> /// The runner. /// </param> /// <param name="validationMetrics"> /// The validation metrics. /// </param> /// <param name="maxNumberWorkers"> /// The maximum number of workers for whom to return validation results. /// </param> /// <returns> /// The validation results. /// </returns> public static Dictionary <string, object> GetHonestWorkerValidationResults(ModelRunnerBase runner, Dictionary <string, object> validationMetrics, int maxNumberWorkers) { var results = GetValidationResults(runner, validationMetrics); // Scatter plot for ability versus accuracy var workerMetrics = GetWorkerMetrics(runner.DataMapping.Data, null, maxNumberWorkers); var accuracies = workerMetrics[Metric.Accuracy.ToString()]; var abilities = ((HonestWorkerRunner)runner.TrainingRunner).WorkerAbility; var validationWorkersWithAbilities = accuracies.Keys.Where(wid => abilities.ContainsKey(wid)).ToList(); var scatterPlot = validationWorkersWithAbilities .Select(wid => new Point((double)accuracies[wid], abilities[wid].GetMean())).ToList(); results["Accuracy Ability"] = scatterPlot; return(results); }
/// <summary> /// Gets the validation results. /// </summary> /// <param name="runner"> /// The runner. /// </param> /// <param name="validationMetrics"> /// The validation metrics. /// </param> /// <returns> /// The validation results. /// </returns> public static Dictionary <string, object> GetValidationResults(ModelRunnerBase runner, Dictionary <string, object> validationMetrics) { var results = new Dictionary <string, object>(); foreach (var kvp in validationMetrics) { var key = kvp.Key; var value = kvp.Value; if (kvp.Key == Metric.ConfusionMatrix.ToString()) { value = PlotData.GetConfusionMatrix((double[, ])kvp.Value, runner.DataMapping, PlotData.ConfusionMatrixRowLabel, PlotData.ConfusionMatrixColLabel2); var valuePerc = PlotData.GetConfusionMatrix((double[, ])kvp.Value, runner.DataMapping, PlotData.ConfusionMatrixRowLabel, PlotData.ConfusionMatrixColLabel2, true); results[key + "Percentage"] = valuePerc; } results[key] = value; } return(results); }
/// <summary> /// Initializes a new instance of the <see cref="BiasedWorkerModelRunner"/> class. /// </summary> /// <param name="dataMapping"> /// The data mapping. /// </param> /// <param name="model"> /// The model. /// </param> /// <param name="trainingRunner"> /// The training runner. /// </param> public BiasedWorkerModelRunner(CrowdDataMapping dataMapping, BiasedWorkerModel model, ModelRunnerBase trainingRunner = null) : base(dataMapping, model, trainingRunner) { }
/// <inheritdoc /> /// <param name="dataMapping"> /// The data mapping /// </param> /// <param name="model"> /// The model. /// </param> /// <param name="trainingRunner"> /// The training Runner. This should be null if /// (a) we are running training, or /// (b) we are running validation and the model has no training /// </param> protected ModelRunnerBase(CrowdDataMapping dataMapping, ModelBase model, ModelRunnerBase trainingRunner = null) : base(dataMapping) { this.Model = model; this.TrainingRunner = trainingRunner; }
/// <summary> /// Gets the <see cref="BiasedCommunityWordsRunner" /> results. /// </summary> /// <param name="runner"> /// The runner. /// </param> /// <returns> /// The dictionary of results. /// </returns> public static Dictionary <string, object> GetBiasedCommunityWordsRunnerResults(ModelRunnerBase runner) { var result = GetBiasedCommunityTrainingResults(runner); if (runner is BiasedCommunityWordsRunner biasedCommunityWordsRunner) { result["BackgroundLogProbWord"] = biasedCommunityWordsRunner.BackgroundLogProbWord; var logProbWord = biasedCommunityWordsRunner.LogProbWord.ToDictionary( kvp => runner.DataMapping.LabelValueToString[kvp.Key], kvp => kvp.Value); result["LogProbWord"] = logProbWord; var relativeLogProbWord = biasedCommunityWordsRunner.RelativeLogProbWord.ToDictionary( kvp => runner.DataMapping.LabelValueToString[kvp.Key], kvp => kvp.Value); result["RelativeLogProbWord"] = relativeLogProbWord; var orderedLogProbWord = biasedCommunityWordsRunner.OrderedLogProbWord.ToDictionary( kvp => runner.DataMapping.LabelValueToString[kvp.Key], kvp => kvp.Value); result["OrderedLogProbWord"] = orderedLogProbWord; var orderedRelativeLogProbWord = biasedCommunityWordsRunner.OrderedRelativeLogProbWord.ToDictionary( kvp => runner.DataMapping.LabelValueToString[kvp.Key], kvp => kvp.Value.ToDictionary(kvp1 => kvp1.Key, kvp1 => kvp1.Value)); result["OrderedRelativeLogProbWord"] = orderedRelativeLogProbWord; result["TopTenWords"] = orderedRelativeLogProbWord.ToDictionary( kvp => kvp.Key, kvp => kvp.Value.OrderByDescending(kvp1 => kvp1.Value).Take(10).Select(kvp1 => Regex.Replace(kvp1.Key, @"[\d-]", string.Empty)).ToList()); } return(result); }
/// <summary> /// Initializes a new instance of the <see cref="HonestWorkerRunner"/> class. /// </summary> /// <param name="dataMapping"> /// The data mapping. /// </param> /// <param name="model"> /// The model. /// </param> /// <param name="trainingRunner"> /// The training runner. /// </param> public HonestWorkerRunner(CrowdDataMapping dataMapping, HonestWorkerModel model, ModelRunnerBase trainingRunner = null) : base(dataMapping, model, trainingRunner) { }
/// <summary> /// Initializes a new instance of the <see cref="BiasedCommunityWordsRunner"/> class. /// </summary> /// <param name="dataMapping"> /// The data mapping. /// </param> /// <param name="model"> /// The model. /// </param> /// <param name="trainingRunner"> /// The training runner. /// </param> public BiasedCommunityWordsRunner(CrowdDataWithTextMapping dataMapping, BiasedCommunityWordsModel model, ModelRunnerBase trainingRunner = null) : base(dataMapping, model, trainingRunner) { }