Exemple #1
0
        /// <summary>
        ///     Gets the <see cref="BiasedCommunityModelRunner" /> results.
        /// </summary>
        /// <param name="runner">
        ///     The runner.
        /// </param>
        /// <returns>
        ///     The dictionary of results.
        /// </returns>
        public static Dictionary <string, object> GetBiasedCommunityTrainingResults(ModelRunnerBase runner)
        {
            var result = GetModelTrainingResults(runner);

            if (runner is BiasedCommunityModelRunner biasedCommunityRunner)
            {
                var communityCpts = new Dictionary <string, Dirichlet[]>();
                for (var i = 0; i < biasedCommunityRunner.CommunityCpt.Count; i++)
                {
                    communityCpts[i.ToString()] = biasedCommunityRunner.CommunityCpt[i];
                }

                var plottableCommunityCpts = PlotData.GetWorkerCpts(
                    communityCpts,
                    runner.DataMapping);

                var communityCptsPerc = PlotData.GetWorkerCpts(
                    communityCpts,
                    runner.DataMapping,
                    true);

                result["CommunityCpts"]         = plottableCommunityCpts;
                result["CommunityCptsPerc"]     = communityCptsPerc;
                result["WorkerCommunities"]     = biasedCommunityRunner.WorkerCommunities;
                result["WorkerCommunityCounts"] = biasedCommunityRunner.WorkerCommunityCounts;
            }

            return(result);
        }
Exemple #2
0
        /// <summary>
        ///     Gets the <see cref="BiasedWorkerModelRunner" /> results.
        /// </summary>
        /// <param name="runner">
        ///     The runner.
        /// </param>
        /// <param name="maximumNumberWorkers">
        ///     The maximum number of workers for whom to return results..
        /// </param>
        /// <returns>
        ///     The dictionary of results.
        /// </returns>
        public static Dictionary <string, object> GetBiasedWorkerTrainingResults(ModelRunnerBase runner,
                                                                                 int maximumNumberWorkers = 20)
        {
            var result = GetModelTrainingResults(runner);

            if (runner is BiasedWorkerModelRunner biasedWorkerRunner)
            {
                var prominentWorkers = GetProminentWorkers(runner.DataMapping.Data, maximumNumberWorkers);

                var prominentWorkerCpts = biasedWorkerRunner.WorkerCpt
                                          .Where(kvp => prominentWorkers.Contains(kvp.Key)).ToDictionary(kvp => kvp.Key, kvp => kvp.Value);

                var workerCpts = PlotData.GetWorkerCpts(
                    prominentWorkerCpts,
                    runner.DataMapping);

                var workerCptsPerc = PlotData.GetWorkerCpts(
                    prominentWorkerCpts,
                    runner.DataMapping,
                    true);

                result["ProminentWorkerCpts"] = workerCpts;
                result["ProminentWorkerCpts"] = workerCptsPerc;
            }

            return(result);
        }
Exemple #3
0
        /// <summary>
        ///     Gets the <see cref="HonestWorkerRunner" /> results.
        /// </summary>
        /// <param name="runner">
        ///     The runner.
        /// </param>
        /// <returns>
        ///     The dictionary of results.
        /// </returns>
        public static Dictionary <string, object> GetHonestWorkerTrainingResults(ModelRunnerBase runner)
        {
            var result           = GetModelTrainingResults(runner);
            var abilityHistogram = PlotData.GetWorkerAbilities((HonestWorkerRunner)runner);

            result["AbilityHistogram"] = abilityHistogram;

            return(result);
        }
Exemple #4
0
        /// <summary>
        ///     Gets <see cref="ModelRunnerBase" /> results.
        /// </summary>
        /// <param name="runner">
        ///     The runner.
        /// </param>
        /// <returns>
        ///     The dictionary of results.
        /// </returns>
        private static Dictionary <string, object> GetModelTrainingResults(ModelRunnerBase runner)
        {
            var result = GetTrainingResults(runner);

            result["Evidence"]            = runner.ModelEvidence;
            result["BackgroundLabelProb"] = runner.BackgroundLabelProb.PseudoCount.ToArray();
            result["Evidence"]            = runner.AverageLogProbability;
            return(result);
        }
Exemple #5
0
        /// <summary>
        ///     Gets the validation results for the honest worker model.
        /// </summary>
        /// <param name="runner">
        ///     The runner.
        /// </param>
        /// <param name="validationMetrics">
        ///     The validation metrics.
        /// </param>
        /// <param name="maxNumberWorkers">
        ///     The maximum number of workers for whom to return validation results.
        /// </param>
        /// <returns>
        ///     The validation results.
        /// </returns>
        public static Dictionary <string, object> GetHonestWorkerValidationResults(ModelRunnerBase runner,
                                                                                   Dictionary <string, object> validationMetrics, int maxNumberWorkers)
        {
            var results = GetValidationResults(runner, validationMetrics);

            // Scatter plot for ability versus accuracy
            var workerMetrics = GetWorkerMetrics(runner.DataMapping.Data, null, maxNumberWorkers);
            var accuracies    = workerMetrics[Metric.Accuracy.ToString()];
            var abilities     = ((HonestWorkerRunner)runner.TrainingRunner).WorkerAbility;
            var validationWorkersWithAbilities =
                accuracies.Keys.Where(wid => abilities.ContainsKey(wid)).ToList();
            var scatterPlot = validationWorkersWithAbilities
                              .Select(wid => new Point((double)accuracies[wid], abilities[wid].GetMean())).ToList();

            results["Accuracy Ability"] = scatterPlot;
            return(results);
        }
Exemple #6
0
        /// <summary>
        ///     Gets the validation results.
        /// </summary>
        /// <param name="runner">
        ///     The runner.
        /// </param>
        /// <param name="validationMetrics">
        ///     The validation metrics.
        /// </param>
        /// <returns>
        ///     The validation results.
        /// </returns>
        public static Dictionary <string, object> GetValidationResults(ModelRunnerBase runner,
                                                                       Dictionary <string, object> validationMetrics)
        {
            var results = new Dictionary <string, object>();

            foreach (var kvp in validationMetrics)
            {
                var key   = kvp.Key;
                var value = kvp.Value;
                if (kvp.Key == Metric.ConfusionMatrix.ToString())
                {
                    value = PlotData.GetConfusionMatrix((double[, ])kvp.Value, runner.DataMapping,
                                                        PlotData.ConfusionMatrixRowLabel, PlotData.ConfusionMatrixColLabel2);
                    var valuePerc = PlotData.GetConfusionMatrix((double[, ])kvp.Value, runner.DataMapping,
                                                                PlotData.ConfusionMatrixRowLabel, PlotData.ConfusionMatrixColLabel2, true);
                    results[key + "Percentage"] = valuePerc;
                }

                results[key] = value;
            }

            return(results);
        }
 /// <summary>
 /// Initializes a new instance of the <see cref="BiasedWorkerModelRunner"/> class.
 /// </summary>
 /// <param name="dataMapping">
 /// The data mapping.
 /// </param>
 /// <param name="model">
 /// The model.
 /// </param>
 /// <param name="trainingRunner">
 /// The training runner.
 /// </param>
 public BiasedWorkerModelRunner(CrowdDataMapping dataMapping, BiasedWorkerModel model, ModelRunnerBase trainingRunner = null)
     : base(dataMapping, model, trainingRunner)
 {
 }
 /// <inheritdoc />
 /// <param name="dataMapping">
 /// The data mapping
 /// </param>
 /// <param name="model">
 /// The model.
 /// </param>
 /// <param name="trainingRunner">
 /// The training Runner. This should be null if
 /// (a) we are running training, or
 /// (b) we are running validation and the model has no training
 /// </param>
 protected ModelRunnerBase(CrowdDataMapping dataMapping, ModelBase model, ModelRunnerBase trainingRunner = null)
     : base(dataMapping)
 {
     this.Model          = model;
     this.TrainingRunner = trainingRunner;
 }
Exemple #9
0
        /// <summary>
        ///     Gets the <see cref="BiasedCommunityWordsRunner" /> results.
        /// </summary>
        /// <param name="runner">
        ///     The runner.
        /// </param>
        /// <returns>
        ///     The dictionary of results.
        /// </returns>
        public static Dictionary <string, object> GetBiasedCommunityWordsRunnerResults(ModelRunnerBase runner)
        {
            var result = GetBiasedCommunityTrainingResults(runner);

            if (runner is BiasedCommunityWordsRunner biasedCommunityWordsRunner)
            {
                result["BackgroundLogProbWord"] = biasedCommunityWordsRunner.BackgroundLogProbWord;

                var logProbWord = biasedCommunityWordsRunner.LogProbWord.ToDictionary(
                    kvp => runner.DataMapping.LabelValueToString[kvp.Key],
                    kvp => kvp.Value);
                result["LogProbWord"] = logProbWord;

                var relativeLogProbWord = biasedCommunityWordsRunner.RelativeLogProbWord.ToDictionary(
                    kvp => runner.DataMapping.LabelValueToString[kvp.Key],
                    kvp => kvp.Value);
                result["RelativeLogProbWord"] = relativeLogProbWord;

                var orderedLogProbWord = biasedCommunityWordsRunner.OrderedLogProbWord.ToDictionary(
                    kvp => runner.DataMapping.LabelValueToString[kvp.Key],
                    kvp => kvp.Value);
                result["OrderedLogProbWord"] = orderedLogProbWord;

                var orderedRelativeLogProbWord = biasedCommunityWordsRunner.OrderedRelativeLogProbWord.ToDictionary(
                    kvp => runner.DataMapping.LabelValueToString[kvp.Key],
                    kvp => kvp.Value.ToDictionary(kvp1 => kvp1.Key, kvp1 => kvp1.Value));
                result["OrderedRelativeLogProbWord"] = orderedRelativeLogProbWord;

                result["TopTenWords"] = orderedRelativeLogProbWord.ToDictionary(
                    kvp => kvp.Key,
                    kvp => kvp.Value.OrderByDescending(kvp1 => kvp1.Value).Take(10).Select(kvp1 => Regex.Replace(kvp1.Key, @"[\d-]", string.Empty)).ToList());
            }

            return(result);
        }
 /// <summary>
 /// Initializes a new instance of the <see cref="HonestWorkerRunner"/> class.
 /// </summary>
 /// <param name="dataMapping">
 /// The data mapping.
 /// </param>
 /// <param name="model">
 /// The model.
 /// </param>
 /// <param name="trainingRunner">
 /// The training runner.
 /// </param>
 public HonestWorkerRunner(CrowdDataMapping dataMapping, HonestWorkerModel model, ModelRunnerBase trainingRunner = null)
     : base(dataMapping, model, trainingRunner)
 {
 }
Exemple #11
0
 /// <summary>
 /// Initializes a new instance of the <see cref="BiasedCommunityWordsRunner"/> class.
 /// </summary>
 /// <param name="dataMapping">
 /// The data mapping.
 /// </param>
 /// <param name="model">
 /// The model.
 /// </param>
 /// <param name="trainingRunner">
 /// The training runner.
 /// </param>
 public BiasedCommunityWordsRunner(CrowdDataWithTextMapping dataMapping, BiasedCommunityWordsModel model, ModelRunnerBase trainingRunner = null)
     : base(dataMapping, model, trainingRunner)
 {
 }