Ejemplo n.º 1
0
        /// <summary>
        ///     Gets the <see cref="BiasedCommunityModelRunner" /> results.
        /// </summary>
        /// <param name="runner">
        ///     The runner.
        /// </param>
        /// <returns>
        ///     The dictionary of results.
        /// </returns>
        public static Dictionary <string, object> GetBiasedCommunityTrainingResults(ModelRunnerBase runner)
        {
            var result = GetModelTrainingResults(runner);

            if (runner is BiasedCommunityModelRunner biasedCommunityRunner)
            {
                var communityCpts = new Dictionary <string, Dirichlet[]>();
                for (var i = 0; i < biasedCommunityRunner.CommunityCpt.Count; i++)
                {
                    communityCpts[i.ToString()] = biasedCommunityRunner.CommunityCpt[i];
                }

                var plottableCommunityCpts = PlotData.GetWorkerCpts(
                    communityCpts,
                    runner.DataMapping);

                var communityCptsPerc = PlotData.GetWorkerCpts(
                    communityCpts,
                    runner.DataMapping,
                    true);

                result["CommunityCpts"]         = plottableCommunityCpts;
                result["CommunityCptsPerc"]     = communityCptsPerc;
                result["WorkerCommunities"]     = biasedCommunityRunner.WorkerCommunities;
                result["WorkerCommunityCounts"] = biasedCommunityRunner.WorkerCommunityCounts;
            }

            return(result);
        }
Ejemplo n.º 2
0
        /// <summary>
        ///     Gets the <see cref="BiasedWorkerModelRunner" /> results.
        /// </summary>
        /// <param name="runner">
        ///     The runner.
        /// </param>
        /// <param name="maximumNumberWorkers">
        ///     The maximum number of workers for whom to return results..
        /// </param>
        /// <returns>
        ///     The dictionary of results.
        /// </returns>
        public static Dictionary <string, object> GetBiasedWorkerTrainingResults(ModelRunnerBase runner,
                                                                                 int maximumNumberWorkers = 20)
        {
            var result = GetModelTrainingResults(runner);

            if (runner is BiasedWorkerModelRunner biasedWorkerRunner)
            {
                var prominentWorkers = GetProminentWorkers(runner.DataMapping.Data, maximumNumberWorkers);

                var prominentWorkerCpts = biasedWorkerRunner.WorkerCpt
                                          .Where(kvp => prominentWorkers.Contains(kvp.Key)).ToDictionary(kvp => kvp.Key, kvp => kvp.Value);

                var workerCpts = PlotData.GetWorkerCpts(
                    prominentWorkerCpts,
                    runner.DataMapping);

                var workerCptsPerc = PlotData.GetWorkerCpts(
                    prominentWorkerCpts,
                    runner.DataMapping,
                    true);

                result["ProminentWorkerCpts"] = workerCpts;
                result["ProminentWorkerCpts"] = workerCptsPerc;
            }

            return(result);
        }
Ejemplo n.º 3
0
        /// <summary>
        ///     Gets the <see cref="HonestWorkerRunner" /> results.
        /// </summary>
        /// <param name="runner">
        ///     The runner.
        /// </param>
        /// <returns>
        ///     The dictionary of results.
        /// </returns>
        public static Dictionary <string, object> GetHonestWorkerTrainingResults(ModelRunnerBase runner)
        {
            var result           = GetModelTrainingResults(runner);
            var abilityHistogram = PlotData.GetWorkerAbilities((HonestWorkerRunner)runner);

            result["AbilityHistogram"] = abilityHistogram;

            return(result);
        }
Ejemplo n.º 4
0
        /// <summary>
        ///     Gets <see cref="RunnerBase" /> results.
        /// </summary>
        /// <param name="runner">
        ///     The runner.
        /// </param>
        /// <returns>
        ///     The dictionary of results.
        /// </returns>
        public static Dictionary <string, object> GetTrainingResults(RunnerBase runner)
        {
            var result          = new Dictionary <string, object>();
            var confusionMatrix = PlotData.GetConfusionMatrix(runner.ConfusionMatrix, runner.DataMapping,
                                                              PlotData.ConfusionMatrixRowLabel, PlotData.ConfusionMatrixColLabel2);

            result["ConfusionMatrix"] = confusionMatrix;
            var confusionMatrixPercentage = PlotData.GetConfusionMatrix(runner.ConfusionMatrix, runner.DataMapping,
                                                                        PlotData.ConfusionMatrixRowLabel, PlotData.ConfusionMatrixColLabel2, true);

            result["ConfusionMatrixPercentage"] = confusionMatrixPercentage;
            var tweetMatrix = PlotData.GetTweetMatrix(runner.TweetMatrix, runner.DataMapping,
                                                      PlotData.ConfusionMatrixRowLabel, "Inferred True");

            result["Accuracy"]      = runner.Accuracy;
            result["AverageRecall"] = runner.AverageRecall;

            return(result);
        }
Ejemplo n.º 5
0
        /// <summary>
        ///     Gets the validation results.
        /// </summary>
        /// <param name="runner">
        ///     The runner.
        /// </param>
        /// <param name="validationMetrics">
        ///     The validation metrics.
        /// </param>
        /// <returns>
        ///     The validation results.
        /// </returns>
        public static Dictionary <string, object> GetValidationResults(ModelRunnerBase runner,
                                                                       Dictionary <string, object> validationMetrics)
        {
            var results = new Dictionary <string, object>();

            foreach (var kvp in validationMetrics)
            {
                var key   = kvp.Key;
                var value = kvp.Value;
                if (kvp.Key == Metric.ConfusionMatrix.ToString())
                {
                    value = PlotData.GetConfusionMatrix((double[, ])kvp.Value, runner.DataMapping,
                                                        PlotData.ConfusionMatrixRowLabel, PlotData.ConfusionMatrixColLabel2);
                    var valuePerc = PlotData.GetConfusionMatrix((double[, ])kvp.Value, runner.DataMapping,
                                                                PlotData.ConfusionMatrixRowLabel, PlotData.ConfusionMatrixColLabel2, true);
                    results[key + "Percentage"] = valuePerc;
                }

                results[key] = value;
            }

            return(results);
        }
Ejemplo n.º 6
0
        /// <summary>
        ///     Gets the worker metrics for a data set
        /// </summary>
        /// <param name="data">
        ///     The data.
        /// </param>
        /// <param name="runner">
        ///     The runner.
        /// </param>
        /// <param name="maxNumberWorkers">
        ///     The maximum number of workers for which to get metrics.
        /// </param>
        /// <returns>
        ///     The metrics.
        /// </returns>
        public static Dictionary <string, Dictionary <string, object> > GetWorkerMetrics(CrowdData data,
                                                                                         RunnerBase runner, int maxNumberWorkers = 20)
        {
            var metricsForWorkers = new List <Metric> {
                Metric.Count, Metric.Accuracy, Metric.ConfusionMatrix
            };
            var result  = new Dictionary <string, Dictionary <string, object> >();
            var mapping = runner != null ? runner.DataMapping : new CrowdDataMapping(data, LabelValuesToString);

            foreach (var metric in metricsForWorkers)
            {
                result[metric.ToString()] = new Dictionary <string, object>();
            }

            var confusionMatrixKey           = Metric.ConfusionMatrix.ToString();
            var confusionMatrixPercentageKey = $"{confusionMatrixKey}Percentage";

            result[confusionMatrixPercentageKey] = new Dictionary <string, object>();

            var          labelsGroupedByWorker = data.CrowdLabels.GroupBy(cd => cd.WorkerId);
            var          trueLabels            = runner?.Predictions;
            const string RowLabel    = PlotData.ConfusionMatrixRowLabel;
            const string ColumnLabel = PlotData.ConfusionMatrixColLabel;

            foreach (var worker in labelsGroupedByWorker)
            {
                var workerId     = worker.Key;
                var workerLabels = worker.Distinct(CrowdData.WorkerTweetEqualityComparer.Instance)
                                   .ToDictionary(lab => lab.TweetId, lab => lab.WorkerLabel);
                var workerMetrics = GetMetrics(mapping, workerLabels, trueLabels);

                foreach (var metric in metricsForWorkers)
                {
                    if (metric == Metric.ConfusionMatrix)
                    {
                        var mat     = workerMetrics[metric];
                        var confMat = PlotData.GetConfusionMatrix((double[, ])mat, mapping, RowLabel, ColumnLabel);
                        result[confusionMatrixKey][workerId] = confMat;
                        var matPerc =
                            PlotData.GetConfusionMatrix((double[, ])mat, mapping, RowLabel, ColumnLabel, true);
                        result[confusionMatrixPercentageKey][workerId] = matPerc;
                    }
                    else
                    {
                        result[metric.ToString()][workerId] = workerMetrics[metric];
                    }
                }
            }

            // Limit the confusion matrices to the more prominent workers.
            var prominentWorkers = new HashSet <string>(result[Metric.Count.ToString()]
                                                        .OrderByDescending(kvp => (int)kvp.Value).Take(maxNumberWorkers).Select(kvp => kvp.Key));

            result[confusionMatrixKey] = result[confusionMatrixKey].Where(kvp1 => prominentWorkers.Contains(kvp1.Key))
                                         .ToDictionary(kvp1 => kvp1.Key, kvp1 => kvp1.Value);
            result[confusionMatrixPercentageKey] = result[confusionMatrixPercentageKey]
                                                   .Where(kvp1 => prominentWorkers.Contains(kvp1.Key))
                                                   .ToDictionary(kvp1 => kvp1.Key, kvp1 => kvp1.Value);

            return(result);
        }