public void BhapkarTestConstructorTest()
{
// Bhapkar, V.P. (1966). A note on the equivalence of two test criteria
// for hypotheses in categorical data. Journal of the American Statistical
// Association, 61, 228-235.
int[,] vision =
{
{ 1520, 266, 124, 66 },
{ 234, 1512, 432, 78 },
{ 117, 362, 1772, 205 },
{ 36, 82, 179, 492 },
};
GeneralConfusionMatrix a = new GeneralConfusionMatrix(vision);
BhapkarTest target = new BhapkarTest(a);
Assert.AreEqual(11.97572, target.Statistic, 1e-5);
Assert.AreEqual(0.00746679746972, target.PValue, 1e-6);
Assert.AreEqual(3, target.DegreesOfFreedom);
Assert.IsFalse(double.IsNaN(target.Statistic));
Assert.IsFalse(double.IsNaN(target.PValue));
}
public PerfViewModel()
{
if (System.ComponentModel.LicenseManager.UsageMode == LicenseUsageMode.Designtime)
{
ConfusionMatrix = new GeneralConfusionMatrix(10, new int[100], new int[100]);
ConfusionView = new ConfusionMatrixView(ConfusionMatrix);
}
}
/// <summary>
/// Creates a new multiple table Kappa test.
/// </summary>
///
/// <param name="matrices">The contingency tables.</param>
///
public AverageKappaTest(GeneralConfusionMatrix[] matrices)
{
double[] kappas = new double[matrices.Length];
double[] variances = new double[matrices.Length];
for (int i = 0; i < matrices.Length; i++)
{
kappas[i] = matrices[i].Kappa;
variances[i] = matrices[i].Variance;
}
this.Compute(kappas, variances);
}
public void GeneralConfusionMatrixConstructorTest2()
{
int[,] matrix =
{
{ 4, 0, 0 },
{ 0, 4, 4 },
{ 0, 0, 0 },
};
GeneralConfusionMatrix target = new GeneralConfusionMatrix(matrix);
Assert.AreEqual(3, target.Classes);
Assert.AreEqual(12, target.Samples);
Assert.AreEqual(matrix, target.Matrix);
Assert.AreEqual(0, target.GeometricAgreement);
}
public void KappaTestConstructorTest()
{
// Example from http://vassarstats.net/kappa.html
// Checked against http://graphpad.com/quickcalcs/Kappa2.cfm (OK)
int[,] matrix =
{
{ 44, 5, 1 },
{ 7, 20, 3 },
{ 9, 5, 6 },
};
GeneralConfusionMatrix a = new GeneralConfusionMatrix(matrix);
Assert.AreEqual(a.RowTotals[0], 50);
Assert.AreEqual(a.RowTotals[1], 30);
Assert.AreEqual(a.RowTotals[2], 20);
Assert.AreEqual(a.ColumnTotals[0], 60);
Assert.AreEqual(a.ColumnTotals[1], 30);
Assert.AreEqual(a.ColumnTotals[2], 10);
Assert.AreEqual(0.4915, a.Kappa, 1e-4);
Assert.IsFalse(double.IsNaN(a.Kappa));
double var = a.Variance;
double var0 = a.VarianceUnderNull;
double varD = Accord.Statistics.Testing.KappaTest.DeltaMethodKappaVariance(a);
double se = System.Math.Sqrt(var);
double se0 = System.Math.Sqrt(var0);
double seD = System.Math.Sqrt(varD);
Assert.AreEqual(0.072, a.StandardError, 0.0005);
// Create a test of the null hypothesis (actual k = 0)
KappaTest target = new KappaTest(a, hypothesizedKappa: 0);
// Std. Error is computed differently under the null hypothesis:
Assert.AreEqual(0.073509316753225237, target.StandardError, 1e-5);
Assert.IsFalse(double.IsNaN(target.StandardError));
}
/// <summary>
/// Creates a new Stuart-Maxwell test.
/// </summary>
///
/// <param name="matrix">The contingency table to test.</param>
///
public StuartMaxwellTest(GeneralConfusionMatrix matrix)
{
int classes = matrix.Classes;
int samples = matrix.Samples;
int df = classes - 1;
int[] rowMarginals = matrix.RowTotals;
int[] colMarginals = matrix.ColumnTotals;
d = new double[df];
for (int i = 0; i < d.Length; i++)
d[i] = rowMarginals[i] - colMarginals[i];
S = new double[df, df];
for (int i = 0; i < df; i++)
{
for (int j = 0; j < df; j++)
{
if (i == j)
{
double u = (rowMarginals[i] - colMarginals[i]);
double pii = matrix.Matrix[i, i];
S[i, i] = rowMarginals[i] + colMarginals[i] - 2.0 * pii;
}
else
{
double pij = matrix.Matrix[i, j];
double pji = matrix.Matrix[j, i];
S[i, j] = -(pij + pji);
}
}
}
invS = S.PseudoInverse();
double chiSquare = d.Multiply(invS).InnerProduct(d);
Compute(chiSquare, df);
}
public void BowkerTestConstructorTest()
{
// Example from Bortz, Lienert and Klaus. Boehnke Verteilungsfreie Methoden in Der Biostatistik, pg 166
// http://books.google.com.br/books?id=chxDIA-x3WIC&printsec=frontcover&source=gbs_atb#v=onepage&q&f=false
int[,] matrix =
{
{ 14, 7, 9 },
{ 5, 26, 19 },
{ 1, 7, 12 },
};
GeneralConfusionMatrix a = new GeneralConfusionMatrix(matrix);
BowkerTest target = new BowkerTest(a);
Assert.AreEqual(12.27, target.Statistic, 1e-2);
Assert.IsFalse(Double.IsNaN(target.Statistic));
Assert.AreEqual(3, target.DegreesOfFreedom);
}
/// <summary>
/// Creates a new Bowker test.
/// </summary>
///
/// <param name="matrix">The contingency table to test.</param>
///
public BowkerTest(GeneralConfusionMatrix matrix)
{
int classes = matrix.Classes;
int[,] n = matrix.Matrix;
double Qb = 0;
for (int j = 0; j < classes; j++)
{
for (int i = 0; i < j; i++)
{
double q = (n[i, j] - n[j, i]);
Qb += (q * q) / (n[i, j] + n[j, i]);
}
}
int df = (classes * (classes - 1)) / 2;
Compute(Qb, df);
}
private static void PrintMatrix(TextWriter writer, GeneralConfusionMatrix matrix, string[] mostCommonLanguagesArray)
{
const int padding = 5;
var langs = mostCommonLanguagesArray.Select(l => l.Length > 5 ? l.Remove(5) : l).ToArray();
writer.Write("act->".PadRight(padding));
foreach (var lang in langs)
{
writer.Write(lang.PadRight(padding));
}
writer.WriteLine();
for (int i = 0; i < matrix.Matrix.GetLength(0); i++)
{
writer.Write(langs[i].PadRight(padding));
for (int j = 0; j < matrix.Matrix.GetLength(1); j++)
{
writer.Write(matrix.Matrix[i, j].ToString().PadRight(padding));
}
writer.WriteLine();
}
}
public void StuartMaxwellTestConstructorTest()
{
// Example from http://www.john-uebersax.com/stat/mcnemar.htm#stuart
int[,] matrix =
{
{ 20, 10, 5 },
{ 3, 30, 15 },
{ 0, 5, 40 },
};
GeneralConfusionMatrix a = new GeneralConfusionMatrix(matrix);
StuartMaxwellTest target = new StuartMaxwellTest(a);
Assert.AreEqual(13.76, target.Statistic, 0.01);
Assert.AreEqual(2, target.DegreesOfFreedom);
Assert.AreEqual(0.001, target.PValue, 1e-4);
Assert.IsFalse(double.IsNaN(target.Statistic));
Assert.IsFalse(double.IsNaN(target.PValue));
}
public void GeneralConfusionMatrixConstructorTest()
{
int classes = 3;
int[] expected = { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2 };
int[] predicted = { 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 };
GeneralConfusionMatrix target = new GeneralConfusionMatrix(classes, expected, predicted);
Assert.AreEqual(3, target.Classes);
Assert.AreEqual(12, target.Samples);
int[,] expectedMatrix =
{
{ 4, 0, 0 },
{ 0, 4, 0 },
{ 0, 4, 0 },
};
int[,] actualMatrix = target.Matrix;
Assert.IsTrue(expectedMatrix.IsEqual(actualMatrix));
}
public void buildModel()
{
IGeometryBag geoBag = new GeometryBagClass();
IGeometryCollection geoColl = (IGeometryCollection)geoBag;
IFeatureCursor ftCur = projectArea.Search(null, false);
IFeature ftr = ftCur.NextFeature();
while (ftr != null)
{
geoColl.AddGeometry(ftr.Shape);
ftr = ftCur.NextFeature();
}
dgc = new dataGeneralConfusionMatirx();
dgc.getXTable(oModel);
olabels = dgc.Labels.ToList();
xTable = dgc.XTable;
oClmProp = dgc.GeneralConfusionMatrix.ColumnProportions;
nCnts = new double[oClmProp.Length];
adjustXTable((IGeometry)geoBag);
gc = new GeneralConfusionMatrix(xTable);
kappa = gc.Kappa;
ste = gc.StandardError;
writeXTable();
}
/// <summary>
/// Computes the asymptotic variance for Fleiss's Kappa variance using the formulae
/// by (Fleiss et al, 1969) when the underlying Kappa is assumed different from zero.
/// </summary>
///
/// <param name="matrix">A <see cref="GeneralConfusionMatrix"/> representing the ratings.</param>
///
/// <returns>Kappa's variance.</returns>
///
public static double AsymptoticKappaVariance(GeneralConfusionMatrix matrix)
{
double stdDev;
return AsymptoticKappaVariance(matrix, out stdDev);
}
/// <summary>
/// Computes the asymptotic variance for Fleiss's Kappa variance using the formulae
/// by (Fleiss et al, 1969). If <paramref name="nullHypothesis"/> is set to true, the
/// method will return the variance under the null hypothesis.
/// </summary>
///
/// <param name="matrix">A <see cref="GeneralConfusionMatrix"/> representing the ratings.</param>
/// <param name="stdDev">Kappa's standard deviation.</param>
/// <param name="nullHypothesis">True to compute Kappa's variance when the null hypothesis
/// is true (i.e. that the underlying kappa is zer). False otherwise. Default is false.</param>
///
/// <returns>Kappa's variance.</returns>
///
public static double AsymptoticKappaVariance(GeneralConfusionMatrix matrix, out double stdDev,
bool nullHypothesis = false)
{
double n = matrix.Samples;
double k = matrix.Kappa;
double[,] p = matrix.ProportionMatrix;
double[] colMarginal = matrix.ColumnProportions;
double[] rowMarginal = matrix.RowProportions;
double Pe = 0;
for (int i = 0; i < rowMarginal.Length; i++)
Pe += rowMarginal[i] * colMarginal[i];
double variance;
if (!nullHypothesis)
{
// References: Statistical Methods for Rates and Proportions, pg 606.
// Fleiss calculations on page 607 are done with a rounded Kappa of:
//
// k = 0.68
// Compute A (eq. 18.16)
double A = 0;
for (int i = 0; i < rowMarginal.Length; i++)
{
double pii = p[i, i];
double pid = rowMarginal[i];
double pdi = colMarginal[i];
A += pii * square(1.0 - (pid + pdi) * (1 - k));
}
// Compute B (eq. 18.17)
double sum = 0;
for (int i = 0; i < colMarginal.Length; i++)
{
for (int j = 0; j < rowMarginal.Length; j++)
{
if (i != j)
{
double pij = p[i, j];
double pdi = colMarginal[i];
double pjd = rowMarginal[j];
sum += pij * square(pdi + pjd);
}
}
}
double B = square(1.0 - k) * sum;
// Compute C
double C = square(k - Pe * (1 - k));
// Compute variance and standard error using A, B and C
variance = (A + B - C) / (square(1.0 - Pe) * n);
stdDev = Math.Sqrt(A + B - C) / ((1.0 - Pe) * Math.Sqrt(n));
}
else
{
double sum = 0;
for (int i = 0; i < rowMarginal.Length; i++)
sum += colMarginal[i] * rowMarginal[i] * (colMarginal[i] + rowMarginal[i]);
variance = (1.0 / (square(1.0 - Pe) * n)) * (Pe + Pe * Pe - sum);
stdDev = (1.0 / ((1.0 - Pe) * Math.Sqrt(n))) * Math.Sqrt(Pe + Pe * Pe - sum);
}
System.Diagnostics.Debug.Assert(!(Math.Abs(variance - stdDev * stdDev) > 1e-10 * variance));
return variance;
}
/// <summary>
/// Compute Cohen's Kappa variance using the large sample approximation
/// given by Congalton, which is common in the remote sensing literature.
/// </summary>
///
/// <param name="matrix">A <see cref="GeneralConfusionMatrix"/> representing the ratings.</param>
///
/// <returns>Kappa's variance.</returns>
///
public static double DeltaMethodKappaVariance(GeneralConfusionMatrix matrix)
{
double stdDev = 0;
return DeltaMethodKappaVariance(matrix, out stdDev);
}
/// <summary>
/// Compute Cohen's Kappa variance using the large sample approximation
/// given by Congalton, which is common in the remote sensing literature.
/// </summary>
///
/// <param name="matrix">A <see cref="GeneralConfusionMatrix"/> representing the ratings.</param>
/// <param name="stdDev">Kappa's standard deviation.</param>
///
/// <returns>Kappa's variance.</returns>
///
public static double DeltaMethodKappaVariance(GeneralConfusionMatrix matrix, out double stdDev)
{
int n = matrix.Samples;
double sum;
double θ1 = (1.0 / n) * matrix.Diagonal.Sum(); // observed agreement, po
sum = 0;
for (int i = 0; i < matrix.RowTotals.Length; i++)
sum += matrix.RowTotals[i] * matrix.ColumnTotals[i];
double θ2 = (1.0 / (n * n)) * sum; // expected agreement, pe
sum = 0;
for (int i = 0; i < matrix.RowTotals.Length; i++)
sum += matrix.Diagonal[i] * (matrix.RowTotals[i] + matrix.ColumnTotals[i]);
double θ3 = (1.0 / (n * n)) * sum;
sum = 0;
for (int i = 0; i < matrix.RowTotals.Length; i++)
for (int j = 0; j < matrix.ColumnTotals.Length; j++)
sum += matrix.Matrix[i, j] * Math.Pow(matrix.RowTotals[i] + matrix.ColumnTotals[j], 2);
double θ4 = (1.0 / (n * n * n)) * sum;
double A = (θ1 * (1 - θ1)) / ((1 - θ2) * (1 - θ2));
double B = (2 * (1 - θ1) * (2 * θ1 * θ2 - θ3)) / ((1 - θ2) * (1 - θ2) * (1 - θ2));
double C = ((1 - θ1) * (1 - θ1) * (θ4 - 4 * θ2 * θ2)) / ((1 - θ2) * (1 - θ2) * (1 - θ2) * (1 - θ2));
double var = (1.0 / n) * (A + B + C);
stdDev = Math.Sqrt(var);
return var;
}
public void ChiSquareTest2()
{
int[,] matrix =
{
{ 296, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
{ 0, 293, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 1, 0, 0, 1 },
{ 1, 0, 274, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20 },
{ 1, 0, 0, 278, 0, 1, 0, 0, 7, 4, 2, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 2 },
{ 0, 1, 0, 0, 290, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7 },
{ 0, 4, 1, 7, 2, 263, 0, 0, 0, 2, 0, 3, 0, 0, 0, 1, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4 },
{ 5, 7, 1, 29, 1, 20, 0, 0, 10, 10, 49, 28, 1, 6, 0, 4, 1, 29, 0, 21, 15, 9, 3, 4, 0, 32, 15 },
{ 0, 7, 0, 23, 9, 37, 0, 0, 0, 19, 34, 4, 8, 2, 0, 1, 6, 13, 0, 13, 53, 8, 6, 1, 0, 46, 10 },
{ 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0 },
{ 2, 1, 0, 2, 2, 0, 0, 0, 11, 250, 0, 0, 5, 2, 0, 3, 4, 0, 0, 1, 1, 0, 0, 5, 0, 11, 0 },
{ 1, 3, 0, 0, 2, 3, 0, 0, 0, 1, 251, 4, 0, 0, 0, 0, 0, 1, 2, 2, 1, 11, 10, 0, 0, 6, 2 },
{ 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 2, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0 },
{ 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 278, 10, 0, 0, 5, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0 },
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 279, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0 },
{ 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 292, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 1 },
{ 0, 0, 2, 3, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 289, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0 },
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 3, 0, 0, 289, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0 },
{ 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 2, 2, 0, 0, 0, 0, 0, 276, 0, 2, 10, 0, 0, 0, 0, 4, 1 },
{ 4, 0, 0, 0, 7, 2, 0, 0, 1, 2, 0, 0, 3, 0, 2, 0, 0, 0, 274, 0, 0, 0, 0, 2, 0, 2, 1 },
{ 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8, 0, 0, 0, 262, 0, 0, 2, 0, 0, 1, 1 },
{ 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 10, 0, 2, 278, 2, 0, 0, 0, 3, 0 },
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 273, 4, 0, 0, 2, 1 },
{ 0, 1, 1, 0, 1, 0, 0, 0, 0, 3, 7, 1, 1, 0, 0, 0, 0, 0, 0, 1, 2, 2, 277, 0, 0, 2, 1 },
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7, 5, 0, 6, 4, 0, 0, 0, 1, 0, 0, 259, 0, 10, 0 },
{ 14, 13, 0, 14, 0, 28, 0, 0, 6, 94, 10, 2, 1, 10, 0, 8, 7, 0, 0, 16, 0, 16, 24, 5, 0, 32, 0 },
{ 0, 2, 1, 9, 0, 1, 0, 0, 0, 22, 7, 3, 11, 0, 1, 5, 1, 6, 0, 2, 4, 1, 2, 7, 0, 214, 1 },
{ 0, 1, 13, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 3, 1, 1, 0, 0, 0, 0, 0, 272 },
};
GeneralConfusionMatrix target = new GeneralConfusionMatrix(matrix);
double actual = target.ChiSquare;
Assert.IsTrue(Double.IsNaN(actual));
Assert.AreEqual(0, target.GeometricAgreement);
Assert.AreEqual(matrix.Diagonal().Sum() / (double)target.Samples, target.OverallAgreement);
}
/// <summary>
/// Creates a new Kappa test.
/// </summary>
///
/// <param name="matrix">The contingency table to test.</param>
/// <param name="alternate">The alternative hypothesis (research hypothesis) to test. If the
/// hypothesized kappa is left unspecified, a one-tailed test will be used. Otherwise, the
/// default is to use a two-sided test.</param>
/// <param name="hypothesizedKappa">The hypothesized value for the Kappa statistic. If the test
/// is being used to assert independency between two raters (i.e. testing the null hypothesis
/// that the underlying Kappa is zero), then the <see cref="AsymptoticKappaVariance(GeneralConfusionMatrix)">
/// standard error will be computed with the null hypothesis parameter set to true</see>.</param>
///
public KappaTest(GeneralConfusionMatrix matrix, double hypothesizedKappa,
OneSampleHypothesis alternate = OneSampleHypothesis.ValueIsDifferentFromHypothesis)
{
if (hypothesizedKappa == 0)
{
// Use the null hypothesis variance
Compute(matrix.Kappa, hypothesizedKappa, matrix.StandardErrorUnderNull, alternate);
Variance = matrix.VarianceUnderNull;
}
else
{
// Use the default variance
Compute(matrix.Kappa, hypothesizedKappa, matrix.StandardError, alternate);
Variance = matrix.Variance;
}
}
public void TotalTest()
{
int[,] matrix =
{
{ 1, 2, 3 },
{ 4, 5, 6 },
{ 7, 8, 9 },
};
GeneralConfusionMatrix target = new GeneralConfusionMatrix(matrix);
int[] colTotals = target.ColumnTotals;
int[] rowTotals = target.RowTotals;
Assert.AreEqual(1 + 2 + 3, rowTotals[0]);
Assert.AreEqual(4 + 5 + 6, rowTotals[1]);
Assert.AreEqual(7 + 8 + 9, rowTotals[2]);
Assert.AreEqual(1 + 4 + 7, colTotals[0]);
Assert.AreEqual(2 + 5 + 8, colTotals[1]);
Assert.AreEqual(3 + 6 + 9, colTotals[2]);
}
public PerfViewModel(GeneralConfusionMatrix matrix)
{
ConfusionMatrix = matrix;
ConfusionView = new ConfusionMatrixView(matrix);
}
public void GeometricAgreementTest()
{
int[,] matrix =
{
{ 462, 241 },
{ 28, 59 },
};
GeneralConfusionMatrix target = new GeneralConfusionMatrix(matrix);
double actual = target.GeometricAgreement;
double expected = Math.Sqrt(462 * 59);
Assert.AreEqual(expected, actual, 1e-10);
Assert.IsFalse(Double.IsNaN(actual));
}
public void KappaTestConstructorTest5()
{
// Example from Statistical Methods for Rates and Proportions
// for kappa variance under the null hypothesis
// Checked against Statistical Methods for Rates and Proportions (OK)
int[,] matrix = // (pg 599)
{
{ 75, 1, 4 },
{ 5, 4, 1 },
{ 0, 0, 10 },
};
GeneralConfusionMatrix a = new GeneralConfusionMatrix(matrix);
Assert.AreEqual(100, a.Samples);
Assert.AreEqual(80, a.RowTotals[0]);
Assert.AreEqual(10, a.RowTotals[1]);
Assert.AreEqual(10, a.RowTotals[2]);
Assert.AreEqual(80, a.ColumnTotals[0]);
Assert.AreEqual(5, a.ColumnTotals[1]);
Assert.AreEqual(15, a.ColumnTotals[2]);
double[,] proportions = // (pg 599)
{
{ 0.75, 0.01, 0.04 },
{ 0.05, 0.04, 0.01 },
{ 0.00, 0.00, 0.10 },
};
Assert.IsTrue(proportions.IsEqual(a.ProportionMatrix));
// Test under null hypothesis
KappaTest target = new KappaTest(a, hypothesizedKappa: 0,
alternate: OneSampleHypothesis.ValueIsGreaterThanHypothesis);
Assert.AreEqual(0.68, target.EstimatedValue, 0.01); // pg 605
Assert.AreEqual(a.Kappa, target.EstimatedValue);
Assert.IsFalse(double.IsNaN(target.EstimatedValue));
Assert.AreEqual(0.076, target.StandardError, 0.001);
Assert.IsFalse(double.IsNaN(target.StandardError));
Assert.AreEqual(8.95, target.Statistic, 0.08);
Assert.IsTrue(target.Significant);
}
public void buildModel()
{
IGeometryBag geoBag = new GeometryBagClass();
IGeometryCollection geoColl = (IGeometryCollection)geoBag;
IFeatureCursor ftCur = projectArea.Search(null, false);
IFeature ftr = ftCur.NextFeature();
while (ftr != null)
{
geoColl.AddGeometry(ftr.Shape);
ftr = ftCur.NextFeature();
}
dgc = new dataGeneralConfusionMatirx();
dgc.getXTable(oModel);
olabels = dgc.Labels.ToList();
xTable = dgc.XTable;
oClmProp = dgc.GeneralConfusionMatrix.ColumnProportions;
nCnts = new double[oClmProp.Length];
adjustXTable((IGeometry)geoBag);
gc = new GeneralConfusionMatrix(xTable);
kappa = gc.Kappa;
ste = gc.StandardError;
writeXTable();
}
public void ChiSquareTest()
{
int[,] matrix =
{
{ 10, 9, 5, 7, 8 },
{ 1, 2, 0, 1, 2 },
{ 0, 0, 1, 0, 1 },
{ 1, 0, 0, 3, 0 },
{ 0, 2, 0, 0, 2 },
};
GeneralConfusionMatrix target = new GeneralConfusionMatrix(matrix);
double actual = target.ChiSquare;
Assert.AreEqual(19.43, actual, 0.01);
Assert.IsFalse(Double.IsNaN(actual));
}
/// <summary>
/// Creates a new Two-Table Kappa test.
/// </summary>
///
/// <param name="matrix1">The first contingency table to test.</param>
/// <param name="matrix2">The second contingency table to test.</param>
/// <param name="hypothesizedDifference">The hypothesized difference between the two Kappa values.</param>
/// <param name="alternate">The alternative hypothesis (research hypothesis) to test.</param>
///
public TwoMatrixKappaTest(GeneralConfusionMatrix matrix1, GeneralConfusionMatrix matrix2, double hypothesizedDifference = 0,
TwoSampleHypothesis alternate = TwoSampleHypothesis.ValuesAreDifferent)
{
this.EstimatedValue1 = matrix1.Kappa;
this.EstimatedValue2 = matrix2.Kappa;
this.Variance1 = matrix1.Variance;
this.Variance2 = matrix2.Variance;
this.OverallVariance = Variance1 + Variance2;
double diff = Math.Abs(EstimatedValue1 - EstimatedValue2);
double stdError = Math.Sqrt(OverallVariance);
Compute(diff, hypothesizedDifference, stdError, alternate);
}
/// <summary>
/// Creates a new Two-Table Mean Kappa test.
/// </summary>
///
/// <param name="matrices1">The first group of contingency tables.</param>
/// <param name="matrices2">The second group of contingency tables.</param>
/// <param name="assumeEqualVariances">True to assume equal variances, false otherwise. Default is true.</param>
/// <param name="hypothesizedDifference">The hypothesized difference between the two average Kappa values.</param>
/// <param name="alternate">The alternative hypothesis (research hypothesis) to test.</param>
///
public TwoAverageKappaTest(GeneralConfusionMatrix[] matrices1, GeneralConfusionMatrix[] matrices2,
double hypothesizedDifference = 0, bool assumeEqualVariances = true,
TwoSampleHypothesis alternate = TwoSampleHypothesis.ValuesAreDifferent)
{
double[] kappas1 = new double[matrices1.Length];
for (int i = 0; i < matrices1.Length; i++)
kappas1[i] = matrices1[i].Kappa;
double[] kappas2 = new double[matrices1.Length];
for (int i = 0; i < matrices2.Length; i++)
kappas2[i] = matrices2[i].Kappa;
double meanKappa1 = kappas1.Mean();
double meanKappa2 = kappas2.Mean();
Variance1 = kappas1.Variance(meanKappa1);
Variance2 = kappas2.Variance(meanKappa2);
int kappaSamples1 = matrices1.Length;
int kappaSamples2 = matrices2.Length;
base.Compute(
meanKappa1, Variance1, kappaSamples1,
meanKappa2, Variance2, kappaSamples2,
hypothesizedDifference, assumeEqualVariances, alternate);
}
public void KappaTestConstructorTest2()
{
// Example from: M. Reichenheim (2004). Confidence intervals for the
// kappa statistic. The Stata Journal (2004) 4, Number 4, pp. 421–428.
// http://www.stata-journal.com/sjpdf.html?articlenum=st0076
int[,] matrix = // (pg 599)
{
{ 48, 12 },
{ 16, 160 },
};
GeneralConfusionMatrix a = new GeneralConfusionMatrix(matrix);
// Reichenheim's paper shows:
// Agreement | Expected Agreement | Kappa | Std. Error | Z
// 88.14% | 61.25% | 0.6938 | 0.0650 | 10.67
Assert.AreEqual(88.14, a.OverallAgreement * 100, 1e-2);
Assert.AreEqual(61.25, a.ChanceAgreement * 100, 1e-2);
Assert.AreEqual(0.6938, a.Kappa, 1e-4);
Assert.AreEqual(0.0650, a.StandardErrorUnderNull, 1e-4);
KappaTest target = new KappaTest(a);
Assert.AreEqual(OneSampleHypothesis.ValueIsGreaterThanHypothesis, target.Hypothesis);
Assert.AreEqual(10.67, target.Statistic, 1e-3);
Assert.AreEqual(7.0849733798130419E-27, target.PValue);
}
public void KappaVarianceTest3()
{
// Example from J. L. Fleiss, J. Cohen, B. S. Everitt, "Large sample
// standard errors of kappa and weighted kappa" Psychological Bulletin (1969)
// Volume: 72, Issue: 5, American Psychological Association, Pages: 323-327
// This was the paper which presented the finally correct
// large sample variance for Kappa after so many attempts.
double[,] matrix =
{
{ 0.53, 0.05, 0.02 },
{ 0.11, 0.14, 0.05 },
{ 0.01, 0.06, 0.03 },
};
GeneralConfusionMatrix a = new GeneralConfusionMatrix(matrix, 200);
Assert.AreEqual(a.RowProportions[0], .60, 1e-10);
Assert.AreEqual(a.RowProportions[1], .30, 1e-10);
Assert.AreEqual(a.RowProportions[2], .10, 1e-10);
Assert.AreEqual(a.ColumnProportions[0], .65, 1e-10);
Assert.AreEqual(a.ColumnProportions[1], .25, 1e-10);
Assert.AreEqual(a.ColumnProportions[2], .10, 1e-10);
Assert.AreEqual(0.429, a.Kappa, 1e-3);
Assert.IsFalse(double.IsNaN(a.Kappa));
Assert.AreEqual(0.002885, a.Variance, 1e-6);
Assert.AreEqual(0.003082, a.VarianceUnderNull, 1e-6);
Assert.IsFalse(double.IsNaN(a.Variance));
Assert.IsFalse(double.IsNaN(a.VarianceUnderNull));
}
/// <summary>
/// Creates a new Kappa test.
/// </summary>
///
/// <param name="matrix">The contingency table to test.</param>
/// <param name="alternate">The alternative hypothesis (research hypothesis) to test. If the
/// hypothesized kappa is left unspecified, a one-tailed test will be used. Otherwise, the
/// default is to use a two-sided test.</param>
///
public KappaTest(GeneralConfusionMatrix matrix,
OneSampleHypothesis alternate = OneSampleHypothesis.ValueIsGreaterThanHypothesis)
: this(matrix, 0, alternate)
{
}
public void KappaTestConstructorTest4()
{
// Example from Statistical Methods for Rates and Proportions
// Checked against http://graphpad.com/quickcalcs/Kappa2.cfm (OK)
// Checked against Statistical Methods for Rates and Proportions (OK)
// Checked against http://vassarstats.net/kappa.html (FAIL)
int[,] matrix = // (pg 599)
{
{ 75, 1, 4 },
{ 5, 4, 1 },
{ 0, 0, 10 },
};
GeneralConfusionMatrix a = new GeneralConfusionMatrix(matrix);
Assert.AreEqual(100, a.Samples);
Assert.AreEqual(80, a.RowTotals[0]);
Assert.AreEqual(10, a.RowTotals[1]);
Assert.AreEqual(10, a.RowTotals[2]);
Assert.AreEqual(80, a.ColumnTotals[0]);
Assert.AreEqual(5, a.ColumnTotals[1]);
Assert.AreEqual(15, a.ColumnTotals[2]);
double[,] proportions = // (pg 599)
{
{ 0.75, 0.01, 0.04 },
{ 0.05, 0.04, 0.01 },
{ 0.00, 0.00, 0.10 },
};
Assert.IsTrue(proportions.IsEqual(a.ProportionMatrix));
double expectedVar = a.Variance;
// Test against non-null hypothesis
KappaTest target = new KappaTest(a, hypothesizedKappa: 0.8);
// Fleiss reports 0.68 (page 606)
// Graphpad reports 0.676
Assert.AreEqual(0.68, target.EstimatedValue, 0.01);
Assert.AreEqual(a.Kappa, target.EstimatedValue);
Assert.IsFalse(double.IsNaN(target.EstimatedValue));
// Fleiss reports 0.087 (page 607)
// Graphpad reports 0.088
Assert.AreEqual(0.087, target.StandardError, 0.001);
Assert.IsFalse(double.IsNaN(target.StandardError));
Assert.AreEqual(-1.38, target.Statistic, 0.029);
Assert.AreEqual(0.1589, target.PValue, 0.0001);
Assert.IsFalse(target.Significant);
}
public void KappaTest()
{
int[,] matrix =
{
{ 29, 6, 5 },
{ 8, 20, 7 },
{ 1, 2, 22 },
};
GeneralConfusionMatrix target = new GeneralConfusionMatrix(matrix);
Assert.AreEqual(3, target.Classes);
Assert.AreEqual(100, target.Samples);
Assert.AreEqual(0.563, target.Kappa, 1e-3);
Assert.AreEqual(23.367749664961245, target.GeometricAgreement);
}
