Example #1
0
        /**
         * Computes the Fisher scores for all the permutations in a single pass.
         * The algorithm works by starting on one side (min a) and moving to the other side (max a).
         * We compute all the probabilities that we encounter on the way incrementally.
         * Then we sort the probabilities in increasing order and sum them up in that direction.
         * The result is a mapping between the permutation probabilities and p-values.
         * TODO - we may want to cache the resulting list in case of multiple calls
         * */
        public double[,] computeAllPermutationsScores()
        {
            int           cPermutations   = getMaxPossibleA() - getMinPossibleA() + 1;
            List <double> alProbabilities = new List <double>();

            double[,] adScores = new double[cPermutations, 2];
            //We start from the table with the maximal value to avoid numeric computation problems
            ContingencyTable ctMaxValue = getMaxValueTable();
            ContingencyTable ctIterator = ctMaxValue;
            double           pStart     = ctIterator.getHypergeometricProbability();

            double pCurrent = pStart, dSum = 0.0;
            int    iCurrent = 0;

            //iterate to the right side
            while (ctIterator != null)
            {
                //Add the probability of the current permutation to the list
                alProbabilities.Add(pCurrent);
                //Increment the probability
                pCurrent = ctIterator.incrementalHypergeometricProbability(pCurrent);
                //Increment the table - will return null once a exceeds the max value
                ctIterator = ctIterator.next();
            }

            //iterate to the left side
            ctIterator = ctMaxValue;
            pCurrent   = ctIterator.decrementalHypergeometricProbability(pStart);
            ctIterator = ctIterator.previous();
            while (ctIterator != null)
            {
                //Add the probability of the current permutation to the list
                alProbabilities.Add(pCurrent);
                //Decrement the probability
                pCurrent = ctIterator.decrementalHypergeometricProbability(pCurrent);
                //Decrement the table - will return null once a drops below the min value
                ctIterator = ctIterator.previous();
            }

            //Sort the observed probabilities in increasing order
            alProbabilities.Sort();
            //BUGBUG - suspecting that we do not handle well identical entries. Not sure if this bug is occuring.
            dSum = 0.0;
            //Sum the probabilities in increasing order, computing two sided p-values
            //BUGBUG - Not sure how to make this work for one sided tests.
            for (iCurrent = 0; iCurrent < alProbabilities.Count; iCurrent++)
            {
                pCurrent = (double)alProbabilities[iCurrent];
                dSum    += pCurrent;
                if (dSum > 1.0)
                {
                    dSum = 1.0;
                }
                adScores[iCurrent, 0] = pCurrent;
                adScores[iCurrent, 1] = dSum;
            }
            return(adScores);
        }
Example #2
0
        /**
         * Computes the Fisher 2 sided p-value.
         * We iterate over all the premutations and sum the ones that have a lower probability (more extreme).
         * We compute from scratch only a single hypergeometric probability - the probability of the "real" table.
         * Then we iterate by incrementing the table and the probability (right side) and by decrementing the table and the probability (left side).
         * The algorithm has the complexity of O(n), but usually runs much faster.
         * Adding another possible optimization - when the p-value exceeds a cutoff - return 1. This is useful when we only need to know whether one value is larger than the other.
         * When the values are too small to be represented by a double (less than 1-E302) the computation returns an upper bound on the real value.
         * */
        private double computeFisher2TailPermutationTest(double dObservedTableProbability, double dCutoff)
        {
            double           p0 = dObservedTableProbability;
            double           p0Epsilon = p0 * 0.00001;
            double           p = p0, pt = 0, pAbove = 0.0, pLeft = p0, pRight = 0.0;
            ContingencyTable ctIterator = null;
            int cPermutations = 0, cRemiaingPermutations = getMaxPossibleA() - getMinPossibleA();

            m_dMinimalPValue = p0;

            if (p0 == double.Epsilon)
            {
                return(p0 * cRemiaingPermutations); //an upper bound estimation
            }
            //Iterate to the right side - increasing values of a
            if (g_ttTestType == TestType.Right || g_ttTestType == TestType.TwoSided)
            {
                ctIterator = next();
                pt         = incrementalHypergeometricProbability(p0);
                while (ctIterator != null)
                {
                    if (pt < m_dMinimalPValue)
                    {
                        m_dMinimalPValue = pt;
                    }
                    cPermutations++;

                    if (p0 + p0Epsilon >= pt)
                    {
                        p      = p + pt;
                        pLeft += pt;
                        if (p > dCutoff)
                        {
                            return(1.0);
                        }
                    }
                    else
                    {
                        pAbove += pt;
                    }
                    pt         = ctIterator.incrementalHypergeometricProbability(pt);
                    ctIterator = ctIterator.next();

                    if ((ctIterator != null) && (pt <= p0Epsilon))
                    {
                        pt        *= (getMaxPossibleA() - ctIterator.getA() + 1);
                        p         += pt;
                        pLeft     += pt;
                        ctIterator = null;
                    }
                }
            }
            //Iterate to the left side - decreasing values of a
            if (g_ttTestType == TestType.Left || g_ttTestType == TestType.TwoSided)
            {
                ctIterator = previous();
                pt         = decrementalHypergeometricProbability(p0);
                double dBackward = pt;
                while (ctIterator != null)
                {
                    if (pt < m_dMinimalPValue)
                    {
                        m_dMinimalPValue = pt;
                    }
                    cPermutations++;
                    if (p0 + p0Epsilon >= pt)
                    {
                        p       = p + pt;
                        pRight += pt;
                        if (p > dCutoff)
                        {
                            return(1.0);
                        }
                    }
                    else
                    {
                        pAbove += pt;
                    }
                    double dBefore = pt;
                    pt         = ctIterator.decrementalHypergeometricProbability(pt);
                    ctIterator = ctIterator.previous();

                    if ((ctIterator != null) && (pt <= p0Epsilon))
                    {
                        pt        *= (ctIterator.getA() - getMinPossibleA());
                        p         += pt;
                        pRight    += pt;
                        ctIterator = null;
                    }
                }
            }

            m_cComputedFisherScores++;

            return(p);
        }
Example #3
0
        public double[] computeAllFisherStatistics(double[] adResults, ref bool bApproximated)
        {
            double           p0 = getHypergeometricProbability();// probability of seeing the actual data
            double           p0Epsilon = p0 * 0.00001;
            double           p = p0, pt = 0, ptMax = 0.0, pLeft = p0, pRight = p0;
            ContingencyTable ctMaxTable = getMaxValueTable(), ctIterator = ctMaxTable;
            int iMaxA = getMaxPossibleA(), iMinA = getMinPossibleA();
            int cPermutations = 0, cRemiaingPermutations = iMaxA - iMinA;
            int iCurrentA = 0;

            double[] adMapping = new double[iMaxA + 1];

            adResults[0] = p0;

            ptMax = ctIterator.getHypergeometricProbability();
            pt    = ptMax;
            while (ctIterator != null)
            {
                cPermutations++;
                iCurrentA            = ctIterator.getA();
                adMapping[iCurrentA] = pt;

                if (iCurrentA > m_iA)
                {
                    pRight += pt;
                }
                if (iCurrentA < m_iA)
                {
                    pLeft += pt;
                }

                if (p0 + p0Epsilon >= pt && iCurrentA != m_iA)
                {
                    p = p + pt;
                }
                pt         = ctIterator.incrementalHypergeometricProbability(pt);
                ctIterator = ctIterator.next();

                if ((ctIterator != null) && (pt == double.Epsilon))
                {
                    pt           *= (iMaxA - ctIterator.getA() + 1);
                    p            += pt;
                    pRight       += pt;
                    bApproximated = true;
                    for (iCurrentA = ctIterator.getA(); iCurrentA <= iMaxA; iCurrentA++)
                    {
                        adMapping[iCurrentA] = double.Epsilon;
                    }
                    ctIterator = null;
                }
            }
            //Iterate to the left side - decreasing values of a
            ctIterator = ctMaxTable.previous();
            pt         = ctMaxTable.decrementalHypergeometricProbability(ptMax);
            while (ctIterator != null)
            {
                cPermutations++;
                iCurrentA            = ctIterator.getA();
                adMapping[iCurrentA] = pt;

                if (iCurrentA > m_iA)
                {
                    pRight += pt;
                }
                if (iCurrentA < m_iA)
                {
                    pLeft += pt;
                }

                if (p0 + p0Epsilon >= pt && iCurrentA != m_iA)
                {
                    p = p + pt;
                }
                pt         = ctIterator.decrementalHypergeometricProbability(pt);
                ctIterator = ctIterator.previous();

                if ((ctIterator != null) && (pt == double.Epsilon))
                {
                    pt           *= (ctIterator.getA() - getMinPossibleA());
                    p            += pt;
                    pLeft        += pt;
                    bApproximated = true;
                    for (iCurrentA = ctIterator.getA(); iCurrentA >= iMinA; iCurrentA--)
                    {
                        adMapping[iCurrentA] = double.Epsilon;
                    }
                    ctIterator = null;
                }
            }
            for (iCurrentA = iMinA - 1; iCurrentA >= 0; iCurrentA--)
            {
                adMapping[iCurrentA] = 0.0;
            }

            adResults[1] = pLeft;
            adResults[2] = pRight;
            adResults[3] = p;

            return(adMapping);
        }