public void TestIncorrectFractionSums()
{
Assert.Throws <ArgumentException>(() => TestSplittingHelper(1, 1, 1, 0.5, 0.25, 0.5, 0, 0.1, 0, false));
Assert.Throws <ArgumentException>(() => TestSplittingHelper(1, 1, 1, 0.5, 0.25, 0.6, 0, 0, 0, false));
Assert.Throws <ArgumentException>(() => TestSplittingHelper(1, 1, 1, 0.5, 0.25, 0, 0.5, 0, 0.6, false));
}
public void TestPrecisionRecallCurve()
{
// General
var expectedCurve = new[]
{
new PrecisionRecall(1, 0),
new PrecisionRecall(0, 0),
new PrecisionRecall(0.5, 0.5),
new PrecisionRecall(2 / (double)3, 1),
new PrecisionRecall(0.5, 1)
};
var computedCurve = Metrics.PrecisionRecallCurve(new[] { 1, 2 }, new Dictionary <int, double> {
{ 3, 1 }, { 1, 0.5 }, { 2, 0.25 }, { 4, 0 }
}).ToArray();
Assert.Equal(expectedCurve.Length, computedCurve.Length);
for (int i = 0; i < expectedCurve.Length; i++)
{
Assert.Equal(expectedCurve[i], computedCurve[i]);
}
// No instance scores
expectedCurve = new[] { new PrecisionRecall(1, 0), };
computedCurve = Metrics.PrecisionRecallCurve(new[] { 1 }, new Dictionary <int, double>()).ToArray();
Assert.Equal(expectedCurve.Length, computedCurve.Length);
for (int i = 0; i < expectedCurve.Length; i++)
{
Assert.Equal(expectedCurve[i], computedCurve[i]);
}
// No negative instance scores
expectedCurve = new[] { new PrecisionRecall(1, 0), new PrecisionRecall(1, 1) };
computedCurve = Metrics.PrecisionRecallCurve(new[] { 1 }, new Dictionary <int, double> {
{ 1, 1 }
}).ToArray();
Assert.Equal(expectedCurve.Length, computedCurve.Length);
for (int i = 0; i < expectedCurve.Length; i++)
{
Assert.Equal(expectedCurve[i], computedCurve[i]);
}
// Duplicate positive instances
computedCurve = Metrics.PrecisionRecallCurve(new[] { 1, 1 }, new Dictionary <int, double> {
{ 1, 1 }
}).ToArray();
Assert.Equal(expectedCurve.Length, computedCurve.Length);
for (int i = 0; i < expectedCurve.Length; i++)
{
Assert.Equal(expectedCurve[i], computedCurve[i]);
}
// No positive instance scores
Assert.Throws <ArgumentException>(() => Metrics.PrecisionRecallCurve(new int[] { }, new Dictionary <int, double> {
{ 1, 1 }
}));
// Null checks
Assert.Throws <ArgumentNullException>(() => Metrics.PrecisionRecallCurve(null, new Dictionary <int, double> {
{ 1, 1 }
}));
Assert.Throws <ArgumentNullException>(() => Metrics.PrecisionRecallCurve(new[] { 1 }, null));
}
public void AreaUnderRocCurveTest()
{
// AUC for perfect predictions
// Per-label AUC
Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[0], this.groundTruth, this.groundTruth));
Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[1], this.groundTruth, this.groundTruth));
Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[2], this.groundTruth, this.groundTruth));
// M-measure
IDictionary <string, IDictionary <string, double> > computedAucMatrix;
Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.groundTruth, out computedAucMatrix));
Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.groundTruth));
// Pairwise AUC (upper triangle)
Assert.Equal(1.0, computedAucMatrix[LabelSet[0]][LabelSet[1]]);
Assert.Equal(1.0, computedAucMatrix[LabelSet[0]][LabelSet[2]]);
Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[2]]);
// Pairwise AUC (diagnonal)
foreach (string label in LabelSet)
{
Assert.Equal(double.NaN, computedAucMatrix[label][label]); // undefined result
}
// Pairwise AUC (lower triangle)
Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[0]]);
Assert.Equal(1.0, computedAucMatrix[LabelSet[2]][LabelSet[0]]);
Assert.Equal(1.0, computedAucMatrix[LabelSet[2]][LabelSet[1]]);
// AUC for imperfect predictions
// Per-label AUC
Assert.Equal(5 / 12.0, this.evaluator.AreaUnderRocCurve(LabelSet[0], this.groundTruth, this.predictions)); // matches ROC curve
Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[1], this.groundTruth, this.predictions));
Assert.Equal(1 / 8.0, this.evaluator.AreaUnderRocCurve(LabelSet[2], this.groundTruth, this.predictions));
// M-measure
Assert.Equal(5 / 9.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.predictions, out computedAucMatrix));
Assert.Equal(5 / 9.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.predictions));
// Pairwise AUC (upper triangle)
Assert.Equal(5 / 6.0, computedAucMatrix[LabelSet[0]][LabelSet[1]]); // matches ROC curve
Assert.Equal(0.0, computedAucMatrix[LabelSet[0]][LabelSet[2]]);
Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[2]]);
// Pairwise AUC (diagnonal)
foreach (string label in LabelSet)
{
Assert.Equal(double.NaN, computedAucMatrix[label][label]); // undefined result
}
// Pairwise AUC (lower triangle)
Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[0]]);
Assert.Equal(0.0, computedAucMatrix[LabelSet[2]][LabelSet[0]]);
Assert.Equal(0.5, computedAucMatrix[LabelSet[2]][LabelSet[1]]);
// Test code path for symmetric two-class case
var binaryGroundTruth = new LabelDistribution[2];
binaryGroundTruth[0] = new Dictionary <string, double> {
{ LabelSet[0], 1 }, { LabelSet[1], 0 }
};
binaryGroundTruth[1] = new Dictionary <string, double> {
{ LabelSet[0], 0 }, { LabelSet[1], 1 }
};
var binaryPredictions = new LabelDistribution[2];
binaryPredictions[0] = new Dictionary <string, double> {
{ LabelSet[0], 0.9 }, { LabelSet[1], 0.1 }
};
binaryPredictions[1] = new Dictionary <string, double> {
{ LabelSet[0], 0.8 }, { LabelSet[1], 0.2 }
};
Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(binaryGroundTruth, binaryPredictions, out computedAucMatrix));
Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(binaryGroundTruth, binaryPredictions));
Assert.Equal(double.NaN, computedAucMatrix[LabelSet[0]][LabelSet[0]]); // undefined result
Assert.Equal(1.0, computedAucMatrix[LabelSet[0]][LabelSet[1]]);
Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[0]]);
Assert.Equal(double.NaN, computedAucMatrix[LabelSet[1]][LabelSet[1]]); // undefined result
// No positive or negative class labels
var actualLabelDistribution = new LabelDistribution[1];
actualLabelDistribution[0] = new Dictionary <string, double> {
{ LabelSet[0], 1 }, { LabelSet[1], 0 }, { LabelSet[2], 0 }
};
// One-versus-rest
Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[0], actualLabelDistribution, actualLabelDistribution));
Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[1], actualLabelDistribution, actualLabelDistribution));
Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[2], actualLabelDistribution, actualLabelDistribution));
// One-versus-another
Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[0], LabelSet[2], actualLabelDistribution, actualLabelDistribution));
Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[1], LabelSet[0], actualLabelDistribution, actualLabelDistribution));
Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[2], LabelSet[1], actualLabelDistribution, actualLabelDistribution));
// Positive and negative class labels are identical
Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[0], LabelSet[0], actualLabelDistribution, actualLabelDistribution));
}
public void TestNormalizedDcgBest()
{
double[] gains = { 7, 5, 4, 4, 4, 1 };
Assert.Equal(1, Metrics.Ndcg(gains, gains, Metrics.LinearDiscountFunc), Tolerance);
Assert.Equal(1, Metrics.Ndcg(gains, gains, Metrics.LogarithmicDiscountFunc), Tolerance);
}