/// <summary> /// Constructs a new Receiver Operating Characteristic point. /// </summary> internal Point(ReceiverOperatingCharacteristic curve, double cutoff, int truePositives, int trueNegatives, int falsePositives, int falseNegatives) : base(truePositives, trueNegatives, falsePositives, falseNegatives) { this.Cutoff = cutoff; }
/// <summary> /// Updates the accuracy using the current results. /// </summary> protected virtual void UpdateAccuracy() { double nlpdThreshold = -Math.Log(0.001); int labelCount = TrueLabel.First(kvp => kvp.Value != null).Value.Dimension; var confusionMatrix = Util.ArrayInit(labelCount, labelCount, (i, j) => 0.0); int correct = 0; double logProb = 0.0; int goldX = 0; // Only for binary labels if (Mapping.LabelCount == 2) { trueBinaryLabel = new List <double>(); probTrueBinaryLabel = new List <double>(); } foreach (var kvp in GoldLabels) { if (kvp.Value == null) { continue; } // We have a gold label goldX++; Discrete trueLabel = null; if (TrueLabel.ContainsKey(kvp.Key)) { trueLabel = TrueLabel[kvp.Key]; } if (trueLabel == null) { trueLabel = Discrete.Uniform(Mapping.LabelCount); //continue; // No inferred label } var probs = trueLabel.GetProbs(); double max = probs.Max(); var predictedLabels = probs.Select((p, i) => new { prob = p, idx = i }).Where(a => a.prob == max).Select(a => a.idx).ToArray(); int predictedLabel = predictedLabels.Length == 1 ? predictedLabels[0] : predictedLabels[Rand.Int(predictedLabels.Length)]; this.PredictedLabel[kvp.Key] = predictedLabel; int goldLabel = kvp.Value.Value; confusionMatrix[goldLabel, predictedLabel] = confusionMatrix[goldLabel, predictedLabel] + 1.0; var nlp = -trueLabel.GetLogProb(goldLabel); if (nlp > nlpdThreshold) { nlp = nlpdThreshold; } logProb += nlp; if (trueBinaryLabel != null) { trueBinaryLabel.Add(goldLabel); probTrueBinaryLabel.Add(probs[goldLabel]); } } Accuracy = correct / (double)goldX; NegativeLogProb = logProb / goldX; ModelConfusionMatrix = confusionMatrix; // Average recall double sumRec = 0; for (int i = 0; i < labelCount; i++) { double classSum = 0; for (int j = 0; j < labelCount; j++) { classSum += confusionMatrix[i, j]; } sumRec += confusionMatrix[i, i] / classSum; } AvgRecall = sumRec / labelCount; // WorkerLabelAccuracy: Perc. agreement between worker label and gold label int sumAcc = 0; var LabelSet = Mapping.DataWithGold; int numLabels = LabelSet.Count(); foreach (var datum in LabelSet) { sumAcc += datum.WorkerLabel == datum.GoldLabel ? 1 : 0; } WorkerLabelAccuracy = sumAcc / (double)numLabels; if (trueBinaryLabel != null && trueBinaryLabel.Count > 0) { RocCurve = new ReceiverOperatingCharacteristic(trueBinaryLabel.ToArray(), probTrueBinaryLabel.ToArray()); RocCurve.Compute(0.001); BinaryConfusionMatrix = new ReceiverOperatingCharacteristic.ConfusionMatrix((int)confusionMatrix[1, 1], (int)confusionMatrix[0, 0], (int)confusionMatrix[0, 1], (int)confusionMatrix[1, 0]); } }