/**
* Computes the Fisher scores for all the permutations in a single pass.
* The algorithm works by starting on one side (min a) and moving to the other side (max a).
* We compute all the probabilities that we encounter on the way incrementally.
* Then we sort the probabilities in increasing order and sum them up in that direction.
* The result is a mapping between the permutation probabilities and p-values.
* TODO - we may want to cache the resulting list in case of multiple calls
* */
public double[,] computeAllPermutationsScores()
{
int cPermutations = getMaxPossibleA() - getMinPossibleA() + 1;
List <double> alProbabilities = new List <double>();
double[,] adScores = new double[cPermutations, 2];
//We start from the table with the maximal value to avoid numeric computation problems
ContingencyTable ctMaxValue = getMaxValueTable();
ContingencyTable ctIterator = ctMaxValue;
double pStart = ctIterator.getHypergeometricProbability();
double pCurrent = pStart, dSum = 0.0;
int iCurrent = 0;
//iterate to the right side
while (ctIterator != null)
{
//Add the probability of the current permutation to the list
alProbabilities.Add(pCurrent);
//Increment the probability
pCurrent = ctIterator.incrementalHypergeometricProbability(pCurrent);
//Increment the table - will return null once a exceeds the max value
ctIterator = ctIterator.next();
}
//iterate to the left side
ctIterator = ctMaxValue;
pCurrent = ctIterator.decrementalHypergeometricProbability(pStart);
ctIterator = ctIterator.previous();
while (ctIterator != null)
{
//Add the probability of the current permutation to the list
alProbabilities.Add(pCurrent);
//Decrement the probability
pCurrent = ctIterator.decrementalHypergeometricProbability(pCurrent);
//Decrement the table - will return null once a drops below the min value
ctIterator = ctIterator.previous();
}
//Sort the observed probabilities in increasing order
alProbabilities.Sort();
//BUGBUG - suspecting that we do not handle well identical entries. Not sure if this bug is occuring.
dSum = 0.0;
//Sum the probabilities in increasing order, computing two sided p-values
//BUGBUG - Not sure how to make this work for one sided tests.
for (iCurrent = 0; iCurrent < alProbabilities.Count; iCurrent++)
{
pCurrent = (double)alProbabilities[iCurrent];
dSum += pCurrent;
if (dSum > 1.0)
{
dSum = 1.0;
}
adScores[iCurrent, 0] = pCurrent;
adScores[iCurrent, 1] = dSum;
}
return(adScores);
}
/**
* Computes the Fisher 2 sided p-value.
* We iterate over all the premutations and sum the ones that have a lower probability (more extreme).
* We compute from scratch only a single hypergeometric probability - the probability of the "real" table.
* Then we iterate by incrementing the table and the probability (right side) and by decrementing the table and the probability (left side).
* The algorithm has the complexity of O(n), but usually runs much faster.
* Adding another possible optimization - when the p-value exceeds a cutoff - return 1. This is useful when we only need to know whether one value is larger than the other.
* When the values are too small to be represented by a double (less than 1-E302) the computation returns an upper bound on the real value.
* */
private double computeFisher2TailPermutationTest(double dObservedTableProbability, double dCutoff)
{
double p0 = dObservedTableProbability;
double p0Epsilon = p0 * 0.00001;
double p = p0, pt = 0, pAbove = 0.0, pLeft = p0, pRight = 0.0;
ContingencyTable ctIterator = null;
int cPermutations = 0, cRemiaingPermutations = getMaxPossibleA() - getMinPossibleA();
m_dMinimalPValue = p0;
if (p0 == double.Epsilon)
{
return(p0 * cRemiaingPermutations); //an upper bound estimation
}
//Iterate to the right side - increasing values of a
if (g_ttTestType == TestType.Right || g_ttTestType == TestType.TwoSided)
{
ctIterator = next();
pt = incrementalHypergeometricProbability(p0);
while (ctIterator != null)
{
if (pt < m_dMinimalPValue)
{
m_dMinimalPValue = pt;
}
cPermutations++;
if (p0 + p0Epsilon >= pt)
{
p = p + pt;
pLeft += pt;
if (p > dCutoff)
{
return(1.0);
}
}
else
{
pAbove += pt;
}
pt = ctIterator.incrementalHypergeometricProbability(pt);
ctIterator = ctIterator.next();
if ((ctIterator != null) && (pt <= p0Epsilon))
{
pt *= (getMaxPossibleA() - ctIterator.getA() + 1);
p += pt;
pLeft += pt;
ctIterator = null;
}
}
}
//Iterate to the left side - decreasing values of a
if (g_ttTestType == TestType.Left || g_ttTestType == TestType.TwoSided)
{
ctIterator = previous();
pt = decrementalHypergeometricProbability(p0);
double dBackward = pt;
while (ctIterator != null)
{
if (pt < m_dMinimalPValue)
{
m_dMinimalPValue = pt;
}
cPermutations++;
if (p0 + p0Epsilon >= pt)
{
p = p + pt;
pRight += pt;
if (p > dCutoff)
{
return(1.0);
}
}
else
{
pAbove += pt;
}
double dBefore = pt;
pt = ctIterator.decrementalHypergeometricProbability(pt);
ctIterator = ctIterator.previous();
if ((ctIterator != null) && (pt <= p0Epsilon))
{
pt *= (ctIterator.getA() - getMinPossibleA());
p += pt;
pRight += pt;
ctIterator = null;
}
}
}
m_cComputedFisherScores++;
return(p);
}
public double[] computeAllFisherStatistics(double[] adResults, ref bool bApproximated)
{
double p0 = getHypergeometricProbability();// probability of seeing the actual data
double p0Epsilon = p0 * 0.00001;
double p = p0, pt = 0, ptMax = 0.0, pLeft = p0, pRight = p0;
ContingencyTable ctMaxTable = getMaxValueTable(), ctIterator = ctMaxTable;
int iMaxA = getMaxPossibleA(), iMinA = getMinPossibleA();
int cPermutations = 0, cRemiaingPermutations = iMaxA - iMinA;
int iCurrentA = 0;
double[] adMapping = new double[iMaxA + 1];
adResults[0] = p0;
ptMax = ctIterator.getHypergeometricProbability();
pt = ptMax;
while (ctIterator != null)
{
cPermutations++;
iCurrentA = ctIterator.getA();
adMapping[iCurrentA] = pt;
if (iCurrentA > m_iA)
{
pRight += pt;
}
if (iCurrentA < m_iA)
{
pLeft += pt;
}
if (p0 + p0Epsilon >= pt && iCurrentA != m_iA)
{
p = p + pt;
}
pt = ctIterator.incrementalHypergeometricProbability(pt);
ctIterator = ctIterator.next();
if ((ctIterator != null) && (pt == double.Epsilon))
{
pt *= (iMaxA - ctIterator.getA() + 1);
p += pt;
pRight += pt;
bApproximated = true;
for (iCurrentA = ctIterator.getA(); iCurrentA <= iMaxA; iCurrentA++)
{
adMapping[iCurrentA] = double.Epsilon;
}
ctIterator = null;
}
}
//Iterate to the left side - decreasing values of a
ctIterator = ctMaxTable.previous();
pt = ctMaxTable.decrementalHypergeometricProbability(ptMax);
while (ctIterator != null)
{
cPermutations++;
iCurrentA = ctIterator.getA();
adMapping[iCurrentA] = pt;
if (iCurrentA > m_iA)
{
pRight += pt;
}
if (iCurrentA < m_iA)
{
pLeft += pt;
}
if (p0 + p0Epsilon >= pt && iCurrentA != m_iA)
{
p = p + pt;
}
pt = ctIterator.decrementalHypergeometricProbability(pt);
ctIterator = ctIterator.previous();
if ((ctIterator != null) && (pt == double.Epsilon))
{
pt *= (ctIterator.getA() - getMinPossibleA());
p += pt;
pLeft += pt;
bApproximated = true;
for (iCurrentA = ctIterator.getA(); iCurrentA >= iMinA; iCurrentA--)
{
adMapping[iCurrentA] = double.Epsilon;
}
ctIterator = null;
}
}
for (iCurrentA = iMinA - 1; iCurrentA >= 0; iCurrentA--)
{
adMapping[iCurrentA] = 0.0;
}
adResults[1] = pLeft;
adResults[2] = pRight;
adResults[3] = p;
return(adMapping);
}