Esempio n. 1
0
        public override EvaluationResults EvaluateModelOnDataGivenParams(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2, EvaluationResults previousResults)
        {
            EvaluationResults internalResults = _internalEvaluator.EvaluateModelOnDataGivenParams(v2, v1, previousResults);

            internalResults.ModelEvaluator = this;
            return(internalResults);
        }
 protected EvaluationResultsCrossValidate(ModelEvaluator modelEval, List <Score> nullScores, Score altScore, EvaluationResults representativeResults, int numberOfResults)
     :
     base(modelEval, nullScores, altScore, representativeResults.ChiSquareDegreesOfFreedom, representativeResults.GlobalNonMissingCount)
 {
     _representativeResults = representativeResults;
     _numberOfResults       = numberOfResults;
 }
Esempio n. 3
0
        public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2)
        {
            EvaluationResults internalResults = _internalEvaluator.EvaluateModelOnData(v2, v1);

            internalResults.ModelEvaluator = this;
            return(internalResults);
        }
Esempio n. 4
0
        protected override EvaluationResults CreateDummyResults(int[] fisherCounts)
        {
            EvaluationResults result = base.CreateDummyResults(fisherCounts);

            if (_includePredictorInScore)
            {
                result.NullScores.Add(Score.GetInstance(result.NullScores[0].Loglikelihood, result.NullScores[0].OptimizationParameters, result.NullScores[0].Distribution));
            }
            return(result);
        }
Esempio n. 5
0
        public override EvaluationResults EvaluateModelOnDataGivenParams(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2, EvaluationResults previousResults)
        {
            TwoByTwo fishers2by2 = TwoByTwo.GetInstance(
                SufficientStatisticsMapToIntDictionaryMap(v1, _fullLeafCollection),
                SufficientStatisticsMapToIntDictionaryMap(v2, _fullLeafCollection));

            int[] fisherCounts = fishers2by2.ToOneDArray();

            List <Score> nullScores;
            Score        altScore;

            ComputeIidScoresGivenParams(fisherCounts,
                                        previousResults.NullScores[0].OptimizationParameters, previousResults.NullScores[1].OptimizationParameters,
                                        previousResults.AltScore.OptimizationParameters,
                                        out nullScores, out altScore);

            //int tt = fisherCounts[(int)TwoByTwo.ParameterIndex.TT];
            //int tf = fisherCounts[(int)TwoByTwo.ParameterIndex.TF];
            //int ft = fisherCounts[(int)TwoByTwo.ParameterIndex.FT];
            //int ff = fisherCounts[(int)TwoByTwo.ParameterIndex.FF];
            //int sum = SpecialFunctions.Sum(fisherCounts);

            //double pi0 = (double)(tt + tf) / sum;
            //double pi1 = (double)(tt + ft) / sum;
            //double ptt = (double)tt / sum;
            //double ptf = (double)tf / sum;
            //double pft = (double)ft / sum;
            //double pff = 1 - ptt - ptf - pft;

            //double predicted_pi0 = previousResults.NullScores[0].OptimizationParameters[0].Value;
            //double predicted_pi1 = previousResults.NullScores[1].OptimizationParameters[0].Value;
            //double predicted_ptt = previousResults.AltScore.OptimizationParameters[0].Value;
            //double predicted_ptf = previousResults.AltScore.OptimizationParameters[1].Value;
            //double predicted_pft = previousResults.AltScore.OptimizationParameters[2].Value;
            //double predicted_pff = 1 - predicted_ptt - predicted_ptf - predicted_pft;


            //double nullLLLeft = pi0 * Math.Log(predicted_pi0);
            //double nullLLRight = pi1 * Math.Log(predicted_pi1);
            //double altLL = predicted_ptt * Math.Log(ptt) + predicted_ptf * Math.Log(ptf) + predicted_pft * Math.Log(pft) + predicted_pff * Math.Log(pff);

            //Score nullScoreLeft = Score.GetInstance(nullLLLeft, previousResults.NullScores[0].OptimizationParameters, null);
            //Score nullScoreRight = Score.GetInstance(nullLLRight, previousResults.NullScores[1].OptimizationParameters, null);
            //Score altScore = Score.GetInstance(altLL, previousResults.AltScore.OptimizationParameters, null);

            //List<Score> nullScores = new List<Score>(2);
            //nullScores.Add(nullScoreLeft);
            //nullScores.Add(nullScoreRight);

            EvaluationResults results = EvaluationResultsFisher.GetInstance(this, nullScores, altScore, fishers2by2);

            return(results);
        }
Esempio n. 6
0
        public override string ToHeaderString()
        {
            //EvaluationResults dummyResults = EvaluationResultsFisher.GetInstance(this, TwoByTwo.GetInstance(new int[] { 1, 1, 1, 1 }));
            List <Score> nullScores;
            Score        altScore;
            TwoByTwo     fishers2by2 = TwoByTwo.GetInstance(new int[] { 1, 1, 1, 1 });

            ComputeIidScores(fishers2by2.ToOneDArray(), out nullScores, out altScore);

            EvaluationResults dummyResults = EvaluationResultsFisher.GetInstance(this, nullScores, altScore, fishers2by2);

            return(dummyResults.ToHeaderString());
        }
        public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2)
        {
            EvaluationResults bestResults = null;

            foreach (ModelEvaluatorCrossValidate model in _modelsToEvaluate)
            {
                EvaluationResults results = model.EvaluateModelOnData(v1, v2);
                if (bestResults == null || results.AltScore.Loglikelihood > bestResults.AltScore.Loglikelihood)
                {
                    bestResults = results;
                }
            }
            //EvaluationResults resultsFromFullDataset = bestResults.ModelEvaluator.EvaluateModelOnData(v1, v2);
            EvaluationResults resultsFromFullDataset = ((ModelEvaluatorCrossValidate)(bestResults.ModelEvaluator)).InternalEvaluator.EvaluateModelOnData(v1, v2);

            return(resultsFromFullDataset);
        }
Esempio n. 8
0
        public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> predMap, Converter <Leaf, SufficientStatistics> targMap)
        {
            int predCount             = ModelScorer.PhyloTree.CountOfNonMissingLeaves(predMap);
            int targCount             = ModelScorer.PhyloTree.CountOfNonMissingLeaves(targMap);
            int globalNonMissingCount = ModelScorer.PhyloTree.CountOfNonMissingLeaves(predMap, targMap);

            MessageInitializerGaussian nullMessageInitializer = MessageInitializerGaussian.GetInstance(
                predMap, targMap, (DistributionGaussianConditional)NullDistns[0], ModelScorer.PhyloTree.LeafCollection);
            Score nullScore = ModelScorer.MaximizeLikelihood(nullMessageInitializer);

            MessageInitializerGaussian altMessageInitializer = MessageInitializerGaussian.GetInstance(
                predMap, targMap, (DistributionGaussianConditional)AltDistn, ModelScorer.PhyloTree.LeafCollection);
            Score altScore = ModelScorer.MaximizeLikelihood(altMessageInitializer);

            EvaluationResults evalResults = EvaluationResultsGaussian.GetInstance(this, nullScore, altScore, predCount, targCount, globalNonMissingCount, ChiSquareDegreesOfFreedom);

            return(evalResults);
        }
        public EvaluationResultsCrossValidate AddNewResults(EvaluationResults newResults)
        {
            SpecialFunctions.CheckCondition(newResults.GetType() == _representativeResults.GetType(), "Evaluation results can only be combined if their types are identical.");

            // first average the parameters.
            double       newParamWeight = 1.0 / (_numberOfResults + 1);
            List <Score> newNullScores  = new List <Score>(NullScores.Count);

            for (int i = 0; i < NullScores.Count; i++)
            {
                Score newScore = CombineScores(NullScores[i], newResults.NullScores[i], newParamWeight);
                newNullScores.Add(newScore);
            }
            Score newAltScore = CombineScores(AltScore, newResults.AltScore, newParamWeight);

            EvaluationResultsCrossValidate result = new EvaluationResultsCrossValidate(ModelEvaluator, newNullScores, newAltScore, newResults, _numberOfResults + 1);

            return(result);
        }
Esempio n. 10
0
        public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> predictorMap, Converter <Leaf, SufficientStatistics> targetMap)
        {
            int[] realFisherCounts = ModelScorer.PhyloTree.FisherCounts(predictorMap, targetMap);

            int tt = realFisherCounts[(int)TwoByTwo.ParameterIndex.TT];
            int tf = realFisherCounts[(int)TwoByTwo.ParameterIndex.TF];
            int ft = realFisherCounts[(int)TwoByTwo.ParameterIndex.FT];
            int ff = realFisherCounts[(int)TwoByTwo.ParameterIndex.FF];

            int[] fisherCountsPred = new int[] { tt, ft, tf, ff };  //ModelScorer.PhyloTree.FisherCounts(targetMap, predictorMap);
            int[] fisherCountsTarg = realFisherCounts;

#if NAIVE_EQUILIBRIUM
            //USE THIS FOR BACKWARDS COMPATABILITY
            int[] tempCountsPred = ModelScorer.PhyloTree.CountsOfLeaves(predictorMap);
            int[] tempCountsTarg = ModelScorer.PhyloTree.CountsOfLeaves(targetMap);
            fisherCountsPred = tempCountsPred;
            fisherCountsTarg = tempCountsTarg;
#endif
            bool predIsInvariant, targIsInvariant;

            Score nullScorePred = ComputeSingleVariableScore(targetMap, predictorMap, (DistributionDiscreteSingleVariable)NullDistns[0], fisherCountsPred, out predIsInvariant);
            Score nullScoreTarg = ComputeSingleVariableScore(predictorMap, targetMap, (DistributionDiscreteSingleVariable)NullDistns[1], fisherCountsTarg, out targIsInvariant);

            List <Score> nullScores = new List <Score>(new Score[] { nullScorePred, nullScoreTarg });
            OptimizationParameterList initParams = ((DistributionDiscreteJoint)AltDistn).GenerateInitialParams(nullScorePred.OptimizationParameters, nullScoreTarg.OptimizationParameters);
            Score jointScore;

            if (predIsInvariant || targIsInvariant)  // cannot compute parameters in this case. They come directly from the single variable params
            {
                double jointLL = nullScorePred.Loglikelihood + nullScoreTarg.Loglikelihood;
                jointScore = Score.GetInstance(jointLL, initParams, AltDistn);
            }
            else
            {
                MessageInitializerDiscrete altMessageInitializer = MessageInitializerDiscrete.GetInstance(CreateJointMap(predictorMap, targetMap), (DistributionDiscreteJoint)AltDistn, initParams, ModelScorer.PhyloTree.LeafCollection);
                jointScore = ModelScorer.MaximizeLikelihood(altMessageInitializer);
            }

            EvaluationResults evalResults = EvaluationResultsDiscrete.GetInstance(this, nullScores, jointScore, realFisherCounts, ChiSquareDegreesOfFreedom);

            return(evalResults);
        }
Esempio n. 11
0
        public override EvaluationResults EvaluateModelOnDataGivenParams(Converter <Leaf, SufficientStatistics> predMap, Converter <Leaf, SufficientStatistics> targMap, EvaluationResults previousResults)
        {
            int predCount             = ModelScorer.PhyloTree.CountOfNonMissingLeaves(predMap);
            int targCount             = ModelScorer.PhyloTree.CountOfNonMissingLeaves(targMap);
            int globalNonMissingCount = ModelScorer.PhyloTree.CountOfNonMissingLeaves(predMap, targMap);

            MessageInitializerGaussian nullMessageInitializer = MessageInitializerGaussian.GetInstance(
                predMap, targMap, (DistributionGaussianConditional)NullDistns[0], ModelScorer.PhyloTree.LeafCollection);
            double nullLL    = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializer, previousResults.NullScores[0].OptimizationParameters);
            Score  nullScore = Score.GetInstance(nullLL, previousResults.NullScores[0].OptimizationParameters, previousResults.NullScores[0].Distribution);

            MessageInitializerGaussian altMessageInitializer = MessageInitializerGaussian.GetInstance(
                predMap, targMap, (DistributionGaussianConditional)AltDistn, ModelScorer.PhyloTree.LeafCollection);
            double altLL    = ModelScorer.ComputeLogLikelihoodModelGivenData(altMessageInitializer, previousResults.AltScore.OptimizationParameters);
            Score  altScore = Score.GetInstance(altLL, previousResults.AltScore.OptimizationParameters, previousResults.AltScore.Distribution);

            EvaluationResults evalResults = EvaluationResultsGaussian.GetInstance(this, nullScore, altScore, predCount, targCount, globalNonMissingCount, ChiSquareDegreesOfFreedom);

            return(evalResults);
        }
Esempio n. 12
0
        //public static Converter<Leaf, SufficientStatistics> TESTPRED, TESTTARG;
        //public static EvaluationResults TESTEVALRESULTS;

        public override EvaluationResults EvaluateModelOnDataGivenParams(
            Converter <Leaf, SufficientStatistics> predictorMap, Converter <Leaf, SufficientStatistics> targetMap, EvaluationResults previousResults)
        {
            int[] fisherCounts = ModelScorer.PhyloTree.FisherCounts(predictorMap, targetMap);

            int targNullIdx = _includePredictorInScore ? 1 : 0;

            OptimizationParameterList  nullParamsTarg             = previousResults.NullScores[targNullIdx].OptimizationParameters;
            MessageInitializerDiscrete nullMessageInitializerTarg = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap,
                                                                                                           NullDistn, new int[0], ModelScorer.PhyloTree.LeafCollection);
            double nullLLTarg    = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializerTarg, nullParamsTarg);
            Score  nullScoreTarg = Score.GetInstance(nullLLTarg, nullParamsTarg, NullDistn);


            OptimizationParameterList  altParams             = previousResults.AltScore.OptimizationParameters;
            MessageInitializerDiscrete altMessageInitializer = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap,
                                                                                                      (DistributionDiscreteConditional)AltDistn, new int[0], ModelScorer.PhyloTree.LeafCollection);
            double condLL   = ModelScorer.ComputeLogLikelihoodModelGivenData(altMessageInitializer, altParams);
            Score  altScore = Score.GetInstance(condLL, altParams, AltDistn);

            List <Score> nullScores = new List <Score>();

            if (_includePredictorInScore)
            {
                OptimizationParameterList  nullParamsPred             = previousResults.NullScores[0].OptimizationParameters;
                MessageInitializerDiscrete nullMessageInitializerPred = MessageInitializerDiscrete.GetInstance(targetMap, predictorMap,
                                                                                                               NullDistn, new int[0], ModelScorer.PhyloTree.LeafCollection);
                double nullLLPred    = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializerPred, nullParamsPred);
                Score  nullScorePred = Score.GetInstance(nullLLPred, nullParamsPred, NullDistn);
                nullScores.Add(nullScorePred);
                // conditional model altScore doesn't include predLL. If we're here, we want to add it to make it comparable to joint or reverseConditional
                altScore = Score.GetInstance(altScore.Loglikelihood + nullScorePred.Loglikelihood, altScore.OptimizationParameters, altScore.Distribution);
            }
            nullScores.Add(nullScoreTarg);


            EvaluationResults evalResults = EvaluationResultsDiscrete.GetInstance(this, nullScores, altScore, fisherCounts, ChiSquareDegreesOfFreedom);

            return(evalResults);
        }
Esempio n. 13
0
        public override EvaluationResults EvaluateModelOnDataGivenParams(Converter <Leaf, SufficientStatistics> predictorMap, Converter <Leaf, SufficientStatistics> targetMap, EvaluationResults previousResults)
        {
            int[] fisherCounts = ModelScorer.PhyloTree.FisherCounts(predictorMap, targetMap);

            OptimizationParameterList  nullParamsTarg             = previousResults.NullScores[1].OptimizationParameters;
            MessageInitializerDiscrete nullMessageInitializerTarg = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap, (DistributionDiscreteSingleVariable)NullDistns[0], fisherCounts, ModelScorer.PhyloTree.LeafCollection);
            double nullLLTarg    = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializerTarg, nullParamsTarg);
            Score  nullScoreTarg = Score.GetInstance(nullLLTarg, nullParamsTarg, previousResults.NullScores[1].Distribution);

            OptimizationParameterList  nullParamsPred             = previousResults.NullScores[0].OptimizationParameters;
            MessageInitializerDiscrete nullMessageInitializerPred = MessageInitializerDiscrete.GetInstance(targetMap, predictorMap, (DistributionDiscreteSingleVariable)NullDistns[1], fisherCounts, ModelScorer.PhyloTree.LeafCollection);
            double nullLLPred    = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializerPred, nullParamsPred);
            Score  nullScorePred = Score.GetInstance(nullLLPred, nullParamsPred, previousResults.NullScores[0].Distribution);

            List <Score> nullScores = new List <Score>(new Score[] { nullScorePred, nullScoreTarg });

            OptimizationParameterList altParams = previousResults.AltScore.OptimizationParameters;

            double altLL;

            if (((DistributionDiscreteJoint)AltDistn).ParametersCannotBeEvaluated(altParams))
            {
                // we'll get here only if one of the variables is always (or never) true. In this case, the variables must be independent.
                altLL = nullLLTarg + nullLLPred;
            }
            else
            {
                MessageInitializerDiscrete altMessageInitializer = MessageInitializerDiscrete.GetInstance(CreateJointMap(predictorMap, targetMap), (DistributionDiscreteJoint)AltDistn, fisherCounts, ModelScorer.PhyloTree.LeafCollection);
                altLL = ModelScorer.ComputeLogLikelihoodModelGivenData(altMessageInitializer, altParams);
            }

            Score altScore = Score.GetInstance(altLL, altParams, previousResults.AltScore.Distribution);

            EvaluationResults evalResults = EvaluationResultsDiscrete.GetInstance(this, nullScores, altScore, fisherCounts, ChiSquareDegreesOfFreedom);

            return(evalResults);
        }
        private string CreateReportLine(
            ModelEvaluator modelEvaluator,
            RowData rowAndTargetData,
            UniversalWorkList workList,
            int rowIndex, int workListCount, int workIndex)
        {
            Dictionary <string, string> row = rowAndTargetData.Row;
            string predictorVariable        = row[Tabulate.PredictorVariableColumnName];
            string targetVariable           = row[Tabulate.TargetVariableColumnName];
            int    nullIndex = int.Parse(row[Tabulate.NullIndexColumnName]);

            Dictionary <string, SufficientStatistics> caseIdToNonMissingPredictorValue = rowAndTargetData.PredictorData;
            Dictionary <string, SufficientStatistics> caseIdToNonMissingTargetValue    = rowAndTargetData.TargetData;

            Converter <Leaf, SufficientStatistics> predictorDistributionClassFunction = CreateSufficientStatisticsMap(caseIdToNonMissingPredictorValue);
            Converter <Leaf, SufficientStatistics> targetDistributionClassFunction    = CreateSufficientStatisticsMap(caseIdToNonMissingTargetValue);

            EvaluationResults results = modelEvaluator.EvaluateModelOnData(predictorDistributionClassFunction, targetDistributionClassFunction);

            string reportLine = SpecialFunctions.CreateTabString(
                results.ModelEvaluator.Name, rowIndex, workListCount, workIndex, nullIndex, predictorVariable, targetVariable, results.ToString());

            return(reportLine);
        }
Esempio n. 15
0
        public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> predMap, Converter <Leaf, SufficientStatistics> targMap)
        {
            TwoByTwo fishers2by2 = TwoByTwo.GetInstance(
                SufficientStatisticsMapToIntDictionaryMap(predMap, _fullLeafCollection),
                SufficientStatisticsMapToIntDictionaryMap(targMap, _fullLeafCollection));

            int[]        fisherCounts = fishers2by2.ToOneDArray();
            List <Score> nullScores;
            Score        altScore;

            ComputeIidScores(fisherCounts, out nullScores, out altScore);

            EvaluationResultsFisher results = EvaluationResultsFisher.GetInstance(this, nullScores, altScore, fishers2by2);

#if DEBUG
            EvaluationResults results2 = EvaluateModelOnDataGivenParams(predMap, targMap, results);
            double            eps      = 1E-14;
            Debug.Assert(ComplexNumber.ApproxEqual(results.AltLL, results2.AltLL, eps));
            Debug.Assert(ComplexNumber.ApproxEqual(results.NullLL, results2.NullLL, eps));
            Debug.Assert(ComplexNumber.ApproxEqual(results.ComputePValue(), results2.ComputePValue(), eps));
#endif

            return(results);
        }
        public override EvaluationResults EvaluateModelOnDataGivenParams(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2, EvaluationResults previousResults)
        {
            EvaluationResults newResults = previousResults.ModelEvaluator.EvaluateModelOnDataGivenParams(v1, v2, previousResults);

            return(newResults);
        }
Esempio n. 17
0
        //public override NullDataGenerator CreateNullDataGenerator(string nullDataGeneratorName)
        //{
        //    return NullDataGenerator.GetInstance(nullDataGeneratorName, ModelScorer, NullDistns[0]);
        //}

        public override string ToHeaderString()
        {
            EvaluationResults dummyResults = CreateDummyResults(new int[4]);

            return(dummyResults.ToHeaderString());
        }
 public static EvaluationResultsCrossValidate GetInstance(ModelEvaluator modelEval, EvaluationResults representativeResults)
 {
     return(new EvaluationResultsCrossValidate(modelEval, representativeResults.NullScores, representativeResults.AltScore, representativeResults, 1));
 }
        public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2)
        {
            List <Leaf> nonMissingLeaves = new List <Leaf>(100);
            int         seed             = 0;

            foreach (Leaf leaf in ModelScorer.PhyloTree.LeafCollection)
            {
                SufficientStatistics class1 = v1(leaf);
                SufficientStatistics class2 = v2(leaf);
                if (!class1.IsMissing() && !class2.IsMissing())
                {
                    nonMissingLeaves.Add(leaf);
                    seed ^= (leaf.CaseName + class1.ToString() + class2.ToString()).GetHashCode();
                }
            }

            Random rand = new Random(seed);

            nonMissingLeaves = SpecialFunctions.Shuffle(nonMissingLeaves, ref rand);

            int groupSize = nonMissingLeaves.Count / _crossValidateCount;

            EvaluationResultsCrossValidate combinedResults = null;
            double testAltLLSum  = 0;   // for debugging
            double testNullLLSum = 0;   // for debugging

            for (int i = 0; i < _crossValidateCount; i++)
            {
                int        testStart  = i * groupSize;
                int        trainStart = testStart + groupSize;
                Set <Leaf> trainSet   = new Set <Leaf>(SpecialFunctions.SubList(nonMissingLeaves, trainStart, nonMissingLeaves.Count - trainStart));
                trainSet.AddNewRange(SpecialFunctions.SubList(nonMissingLeaves, 0, testStart));

                Converter <Leaf, SufficientStatistics> v1Train = CreateFilteredMap(v1, trainSet);
                Converter <Leaf, SufficientStatistics> v2Train = CreateFilteredMap(v2, trainSet);

                EvaluationResults trainingResults    = InternalEvaluator.EvaluateModelOnData(v1Train, v2Train);
                EvaluationResults testAndTrainResult = InternalEvaluator.EvaluateModelOnDataGivenParams(v1, v2, trainingResults);
                EvaluationResultsTestGivenTrain testGivenTrainResult = EvaluationResultsTestGivenTrain.GetInstance(this, trainingResults, testAndTrainResult);

                if (combinedResults == null)
                {
                    combinedResults = EvaluationResultsCrossValidate.GetInstance(this, testGivenTrainResult);
                }
                else
                {
                    combinedResults = combinedResults.AddNewResults(testGivenTrainResult);
                }

                if (double.IsInfinity(combinedResults.AltLL))   // no point in continuing...infinity will kill everything.
                {
                    break;
                }
#if DEBUG
                double            eps = 1E-10;
                EvaluationResults testTrainingResults = InternalEvaluator.EvaluateModelOnDataGivenParams(v1Train, v2Train, trainingResults);
                Debug.Assert(ComplexNumber.ApproxEqual(testTrainingResults.AltLL, trainingResults.AltLL, eps) &&
                             ComplexNumber.ApproxEqual(testTrainingResults.NullLL, trainingResults.NullLL, eps));
                //Debug.Assert(testTrainingResults.Equals(trainingResults));

                double newNullLL = testAndTrainResult.NullLL - trainingResults.NullLL;
                double newAltLL  = testAndTrainResult.AltLL - trainingResults.AltLL;

                Debug.Assert(ComplexNumber.ApproxEqual(newNullLL, testGivenTrainResult.NullLL, eps));
                Debug.Assert(ComplexNumber.ApproxEqual(newAltLL, testGivenTrainResult.AltLL, eps));

                testNullLLSum += newNullLL;
                testAltLLSum  += newAltLL;

                Debug.Assert(ComplexNumber.ApproxEqual(testNullLLSum, combinedResults.NullLL, eps), "Combined result has wrong NullLL");
                Debug.Assert(ComplexNumber.ApproxEqual(testAltLLSum, combinedResults.AltLL, eps), "Combined result has wrong AltLL");
#endif
            }
            return(combinedResults);
        }
Esempio n. 20
0
 // public abstract EvaluationResults EvaluateModelOnData(Converter<Leaf, SufficientStatistics> v1, Converter<Leaf, SufficientStatistics> v2, bool ignoreSimpleCases);
 public abstract EvaluationResults EvaluateModelOnDataGivenParams(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2, EvaluationResults previousResults);
        public static EvaluationResultsTestGivenTrain GetInstance(ModelEvaluator modelEval, EvaluationResults train, EvaluationResults testAndTrain)
        {
            List <Score> nullScores = new List <Score>(train.NullScores.Count);

            for (int i = 0; i < train.NullScores.Count; i++)
            {
                Score score = Score.GetInstance(
                    testAndTrain.NullScores[i].Loglikelihood - train.NullScores[i].Loglikelihood,
                    testAndTrain.NullScores[i].OptimizationParameters,
                    testAndTrain.NullScores[i].Distribution);
                nullScores.Add(score);
            }
            Score altScore = Score.GetInstance(
                testAndTrain.AltScore.Loglikelihood - train.AltScore.Loglikelihood,
                testAndTrain.AltScore.OptimizationParameters,
                testAndTrain.AltScore.Distribution);

            return(new EvaluationResultsTestGivenTrain(modelEval, testAndTrain, nullScores, altScore));
        }
Esempio n. 22
0
        public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> predictorMap, Converter <Leaf, SufficientStatistics> targetMap)
        {
            EvaluationResults evalResults;

            int[] fisherCounts     = ModelScorer.PhyloTree.FisherCounts(predictorMap, targetMap);
            int[] realFisherCounts = fisherCounts;  // for compatability when NAIVE_EQUILIBRIUM is set

#if NAIVE_EQUILIBRIUM
            //USE THIS FOR BACKWARDS COMPATABILITY
            int[] tempCounts = ModelScorer.PhyloTree.CountsOfLeaves(targetMap);
            fisherCounts = tempCounts;
#endif

            //MessageInitializerDiscrete nullMessageInitializer = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap, NullDistn, fisherCounts, ModelScorer.PhyloTree.LeafCollection);
            //if (TryShortCutFromCounts(realFisherCounts, nullMessageInitializer, out evalResults))
            //{
            //    return evalResults;
            //}

            //Score nullScore = ModelScorer.MaximizeLikelihood(nullMessageInitializer);
            bool isInvariant;

            Score nullScoreTarg = ComputeSingleVariableScore(predictorMap, targetMap, NullDistn, fisherCounts, out isInvariant);
            Score altScore      = ComputeConditionalVariableScore(predictorMap, targetMap, nullScoreTarg, fisherCounts);

            //(realFisherCounts, nullScoreTarg, out evalResults))
            //{
            //    return evalResults;
            //}

            //MessageInitializerDiscrete altMessageInitializer = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap, (DistributionDiscreteConditional)AltDistn, nullScore.OptimizationParameters, ModelScorer.PhyloTree.LeafCollection);
            //Score condScore = ModelScorer.MaximizeLikelihood(altMessageInitializer);

            List <Score> nullScores = new List <Score>();
            if (_includePredictorInScore)
            {
                int[] predFisherCounts = new int[] { realFisherCounts[0], realFisherCounts[2], realFisherCounts[1], realFisherCounts[3] };
                Score predNullScore    = ComputeSingleVariableScore(targetMap, predictorMap, NullDistn, predFisherCounts, out isInvariant);
                nullScores.Add(predNullScore);
                // conditional model altScore doesn't include predLL. If we're here, we want to add it to make it comparable to joint or reverseConditional
                altScore = Score.GetInstance(altScore.Loglikelihood + predNullScore.Loglikelihood, altScore.OptimizationParameters, altScore.Distribution);
            }
            nullScores.Add(nullScoreTarg);

            evalResults = EvaluationResultsDiscrete.GetInstance(this, nullScores, altScore, realFisherCounts, ChiSquareDegreesOfFreedom);



#if DEBUG
            MessageInitializerDiscrete nullMessageInitializer = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap, NullDistn, fisherCounts, ModelScorer.PhyloTree.LeafCollection);
            MessageInitializerDiscrete altMessageInitializer  = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap, (DistributionDiscreteConditional)AltDistn, nullScoreTarg.OptimizationParameters, ModelScorer.PhyloTree.LeafCollection);
            double nullLL = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializer, nullScoreTarg.OptimizationParameters);
            double altLL  = ModelScorer.ComputeLogLikelihoodModelGivenData(altMessageInitializer, altScore.OptimizationParameters);

            if (_includePredictorInScore)
            {
                int[] predFisherCounts = new int[] { realFisherCounts[0], realFisherCounts[2], realFisherCounts[1], realFisherCounts[3] };
                MessageInitializerDiscrete nullMessageInitializerPred = MessageInitializerDiscrete.GetInstance(targetMap, predictorMap, NullDistn, predFisherCounts, ModelScorer.PhyloTree.LeafCollection);
                double nullLLPred = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializerPred, nullScores[0].OptimizationParameters);
                altLL += nullLLPred;
            }

            EvaluationResults evalResults2 = EvaluateModelOnDataGivenParams(predictorMap, targetMap, evalResults);

            double eps = 1E-10;
            Debug.Assert(ComplexNumber.ApproxEqual(nullLL, nullScoreTarg.Loglikelihood, eps));
            Debug.Assert(ComplexNumber.ApproxEqual(altLL, altScore.Loglikelihood, eps));
            Debug.Assert(ComplexNumber.ApproxEqual(evalResults.NullLL, evalResults2.NullLL, eps) && ComplexNumber.ApproxEqual(evalResults.AltLL, evalResults2.AltLL, eps), "In ModelEvaluatorCond, results of maximizing LL and computing LL from same params are not the same.");
#endif

            return(evalResults);
        }
 protected EvaluationResultsTestGivenTrain(ModelEvaluator modelEval, EvaluationResults testAndTrain, List <Score> testGiveTrainNullScores, Score testGivenTrainAltScore)
     :
     base(modelEval, testGiveTrainNullScores, testGivenTrainAltScore, testAndTrain.ChiSquareDegreesOfFreedom, testAndTrain.GlobalNonMissingCount)
 {
     _testAndTrain = testAndTrain;
 }