Exemple #1
0
 /// <summary>
 ///   Constructs a new Receiver Operating Characteristic point.
 /// </summary>
 internal Point(ReceiverOperatingCharacteristic curve, double cutoff,
                int truePositives, int trueNegatives, int falsePositives, int falseNegatives)
     : base(truePositives, trueNegatives, falsePositives, falseNegatives)
 {
     this.Cutoff = cutoff;
 }
        /// <summary>
        /// Updates the accuracy using the current results.
        /// </summary>
        protected virtual void UpdateAccuracy()
        {
            double nlpdThreshold   = -Math.Log(0.001);
            int    labelCount      = TrueLabel.First(kvp => kvp.Value != null).Value.Dimension;
            var    confusionMatrix = Util.ArrayInit(labelCount, labelCount, (i, j) => 0.0);
            int    correct         = 0;
            double logProb         = 0.0;

            int goldX = 0;

            // Only for binary labels
            if (Mapping.LabelCount == 2)
            {
                trueBinaryLabel     = new List <double>();
                probTrueBinaryLabel = new List <double>();
            }

            foreach (var kvp in GoldLabels)
            {
                if (kvp.Value == null)
                {
                    continue;
                }

                // We have a gold label
                goldX++;

                Discrete trueLabel = null;
                if (TrueLabel.ContainsKey(kvp.Key))
                {
                    trueLabel = TrueLabel[kvp.Key];
                }

                if (trueLabel == null)
                {
                    trueLabel = Discrete.Uniform(Mapping.LabelCount);
                    //continue;  // No inferred label
                }

                var    probs           = trueLabel.GetProbs();
                double max             = probs.Max();
                var    predictedLabels = probs.Select((p, i) => new
                {
                    prob = p,
                    idx  = i
                }).Where(a => a.prob == max).Select(a => a.idx).ToArray();

                int predictedLabel = predictedLabels.Length == 1 ? predictedLabels[0] : predictedLabels[Rand.Int(predictedLabels.Length)];

                this.PredictedLabel[kvp.Key] = predictedLabel;

                int goldLabel = kvp.Value.Value;

                confusionMatrix[goldLabel, predictedLabel] = confusionMatrix[goldLabel, predictedLabel] + 1.0;

                var nlp = -trueLabel.GetLogProb(goldLabel);
                if (nlp > nlpdThreshold)
                {
                    nlp = nlpdThreshold;
                }
                logProb += nlp;

                if (trueBinaryLabel != null)
                {
                    trueBinaryLabel.Add(goldLabel);
                    probTrueBinaryLabel.Add(probs[goldLabel]);
                }
            }

            Accuracy             = correct / (double)goldX;
            NegativeLogProb      = logProb / goldX;
            ModelConfusionMatrix = confusionMatrix;

            // Average recall
            double sumRec = 0;

            for (int i = 0; i < labelCount; i++)
            {
                double classSum = 0;
                for (int j = 0; j < labelCount; j++)
                {
                    classSum += confusionMatrix[i, j];
                }

                sumRec += confusionMatrix[i, i] / classSum;
            }
            AvgRecall = sumRec / labelCount;

            // WorkerLabelAccuracy: Perc. agreement between worker label and gold label
            int sumAcc    = 0;
            var LabelSet  = Mapping.DataWithGold;
            int numLabels = LabelSet.Count();

            foreach (var datum in LabelSet)
            {
                sumAcc += datum.WorkerLabel == datum.GoldLabel ? 1 : 0;
            }
            WorkerLabelAccuracy = sumAcc / (double)numLabels;

            if (trueBinaryLabel != null && trueBinaryLabel.Count > 0)
            {
                RocCurve = new ReceiverOperatingCharacteristic(trueBinaryLabel.ToArray(), probTrueBinaryLabel.ToArray());
                RocCurve.Compute(0.001);
                BinaryConfusionMatrix = new ReceiverOperatingCharacteristic.ConfusionMatrix((int)confusionMatrix[1, 1], (int)confusionMatrix[0, 0], (int)confusionMatrix[0, 1], (int)confusionMatrix[1, 0]);
            }
        }