public override EvaluationResults EvaluateModelOnDataGivenParams(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2, EvaluationResults previousResults) { EvaluationResults internalResults = _internalEvaluator.EvaluateModelOnDataGivenParams(v2, v1, previousResults); internalResults.ModelEvaluator = this; return(internalResults); }
protected EvaluationResultsCrossValidate(ModelEvaluator modelEval, List <Score> nullScores, Score altScore, EvaluationResults representativeResults, int numberOfResults) : base(modelEval, nullScores, altScore, representativeResults.ChiSquareDegreesOfFreedom, representativeResults.GlobalNonMissingCount) { _representativeResults = representativeResults; _numberOfResults = numberOfResults; }
public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2) { EvaluationResults internalResults = _internalEvaluator.EvaluateModelOnData(v2, v1); internalResults.ModelEvaluator = this; return(internalResults); }
protected override EvaluationResults CreateDummyResults(int[] fisherCounts) { EvaluationResults result = base.CreateDummyResults(fisherCounts); if (_includePredictorInScore) { result.NullScores.Add(Score.GetInstance(result.NullScores[0].Loglikelihood, result.NullScores[0].OptimizationParameters, result.NullScores[0].Distribution)); } return(result); }
public override EvaluationResults EvaluateModelOnDataGivenParams(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2, EvaluationResults previousResults) { TwoByTwo fishers2by2 = TwoByTwo.GetInstance( SufficientStatisticsMapToIntDictionaryMap(v1, _fullLeafCollection), SufficientStatisticsMapToIntDictionaryMap(v2, _fullLeafCollection)); int[] fisherCounts = fishers2by2.ToOneDArray(); List <Score> nullScores; Score altScore; ComputeIidScoresGivenParams(fisherCounts, previousResults.NullScores[0].OptimizationParameters, previousResults.NullScores[1].OptimizationParameters, previousResults.AltScore.OptimizationParameters, out nullScores, out altScore); //int tt = fisherCounts[(int)TwoByTwo.ParameterIndex.TT]; //int tf = fisherCounts[(int)TwoByTwo.ParameterIndex.TF]; //int ft = fisherCounts[(int)TwoByTwo.ParameterIndex.FT]; //int ff = fisherCounts[(int)TwoByTwo.ParameterIndex.FF]; //int sum = SpecialFunctions.Sum(fisherCounts); //double pi0 = (double)(tt + tf) / sum; //double pi1 = (double)(tt + ft) / sum; //double ptt = (double)tt / sum; //double ptf = (double)tf / sum; //double pft = (double)ft / sum; //double pff = 1 - ptt - ptf - pft; //double predicted_pi0 = previousResults.NullScores[0].OptimizationParameters[0].Value; //double predicted_pi1 = previousResults.NullScores[1].OptimizationParameters[0].Value; //double predicted_ptt = previousResults.AltScore.OptimizationParameters[0].Value; //double predicted_ptf = previousResults.AltScore.OptimizationParameters[1].Value; //double predicted_pft = previousResults.AltScore.OptimizationParameters[2].Value; //double predicted_pff = 1 - predicted_ptt - predicted_ptf - predicted_pft; //double nullLLLeft = pi0 * Math.Log(predicted_pi0); //double nullLLRight = pi1 * Math.Log(predicted_pi1); //double altLL = predicted_ptt * Math.Log(ptt) + predicted_ptf * Math.Log(ptf) + predicted_pft * Math.Log(pft) + predicted_pff * Math.Log(pff); //Score nullScoreLeft = Score.GetInstance(nullLLLeft, previousResults.NullScores[0].OptimizationParameters, null); //Score nullScoreRight = Score.GetInstance(nullLLRight, previousResults.NullScores[1].OptimizationParameters, null); //Score altScore = Score.GetInstance(altLL, previousResults.AltScore.OptimizationParameters, null); //List<Score> nullScores = new List<Score>(2); //nullScores.Add(nullScoreLeft); //nullScores.Add(nullScoreRight); EvaluationResults results = EvaluationResultsFisher.GetInstance(this, nullScores, altScore, fishers2by2); return(results); }
public override string ToHeaderString() { //EvaluationResults dummyResults = EvaluationResultsFisher.GetInstance(this, TwoByTwo.GetInstance(new int[] { 1, 1, 1, 1 })); List <Score> nullScores; Score altScore; TwoByTwo fishers2by2 = TwoByTwo.GetInstance(new int[] { 1, 1, 1, 1 }); ComputeIidScores(fishers2by2.ToOneDArray(), out nullScores, out altScore); EvaluationResults dummyResults = EvaluationResultsFisher.GetInstance(this, nullScores, altScore, fishers2by2); return(dummyResults.ToHeaderString()); }
public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2) { EvaluationResults bestResults = null; foreach (ModelEvaluatorCrossValidate model in _modelsToEvaluate) { EvaluationResults results = model.EvaluateModelOnData(v1, v2); if (bestResults == null || results.AltScore.Loglikelihood > bestResults.AltScore.Loglikelihood) { bestResults = results; } } //EvaluationResults resultsFromFullDataset = bestResults.ModelEvaluator.EvaluateModelOnData(v1, v2); EvaluationResults resultsFromFullDataset = ((ModelEvaluatorCrossValidate)(bestResults.ModelEvaluator)).InternalEvaluator.EvaluateModelOnData(v1, v2); return(resultsFromFullDataset); }
public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> predMap, Converter <Leaf, SufficientStatistics> targMap) { int predCount = ModelScorer.PhyloTree.CountOfNonMissingLeaves(predMap); int targCount = ModelScorer.PhyloTree.CountOfNonMissingLeaves(targMap); int globalNonMissingCount = ModelScorer.PhyloTree.CountOfNonMissingLeaves(predMap, targMap); MessageInitializerGaussian nullMessageInitializer = MessageInitializerGaussian.GetInstance( predMap, targMap, (DistributionGaussianConditional)NullDistns[0], ModelScorer.PhyloTree.LeafCollection); Score nullScore = ModelScorer.MaximizeLikelihood(nullMessageInitializer); MessageInitializerGaussian altMessageInitializer = MessageInitializerGaussian.GetInstance( predMap, targMap, (DistributionGaussianConditional)AltDistn, ModelScorer.PhyloTree.LeafCollection); Score altScore = ModelScorer.MaximizeLikelihood(altMessageInitializer); EvaluationResults evalResults = EvaluationResultsGaussian.GetInstance(this, nullScore, altScore, predCount, targCount, globalNonMissingCount, ChiSquareDegreesOfFreedom); return(evalResults); }
public EvaluationResultsCrossValidate AddNewResults(EvaluationResults newResults) { SpecialFunctions.CheckCondition(newResults.GetType() == _representativeResults.GetType(), "Evaluation results can only be combined if their types are identical."); // first average the parameters. double newParamWeight = 1.0 / (_numberOfResults + 1); List <Score> newNullScores = new List <Score>(NullScores.Count); for (int i = 0; i < NullScores.Count; i++) { Score newScore = CombineScores(NullScores[i], newResults.NullScores[i], newParamWeight); newNullScores.Add(newScore); } Score newAltScore = CombineScores(AltScore, newResults.AltScore, newParamWeight); EvaluationResultsCrossValidate result = new EvaluationResultsCrossValidate(ModelEvaluator, newNullScores, newAltScore, newResults, _numberOfResults + 1); return(result); }
public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> predictorMap, Converter <Leaf, SufficientStatistics> targetMap) { int[] realFisherCounts = ModelScorer.PhyloTree.FisherCounts(predictorMap, targetMap); int tt = realFisherCounts[(int)TwoByTwo.ParameterIndex.TT]; int tf = realFisherCounts[(int)TwoByTwo.ParameterIndex.TF]; int ft = realFisherCounts[(int)TwoByTwo.ParameterIndex.FT]; int ff = realFisherCounts[(int)TwoByTwo.ParameterIndex.FF]; int[] fisherCountsPred = new int[] { tt, ft, tf, ff }; //ModelScorer.PhyloTree.FisherCounts(targetMap, predictorMap); int[] fisherCountsTarg = realFisherCounts; #if NAIVE_EQUILIBRIUM //USE THIS FOR BACKWARDS COMPATABILITY int[] tempCountsPred = ModelScorer.PhyloTree.CountsOfLeaves(predictorMap); int[] tempCountsTarg = ModelScorer.PhyloTree.CountsOfLeaves(targetMap); fisherCountsPred = tempCountsPred; fisherCountsTarg = tempCountsTarg; #endif bool predIsInvariant, targIsInvariant; Score nullScorePred = ComputeSingleVariableScore(targetMap, predictorMap, (DistributionDiscreteSingleVariable)NullDistns[0], fisherCountsPred, out predIsInvariant); Score nullScoreTarg = ComputeSingleVariableScore(predictorMap, targetMap, (DistributionDiscreteSingleVariable)NullDistns[1], fisherCountsTarg, out targIsInvariant); List <Score> nullScores = new List <Score>(new Score[] { nullScorePred, nullScoreTarg }); OptimizationParameterList initParams = ((DistributionDiscreteJoint)AltDistn).GenerateInitialParams(nullScorePred.OptimizationParameters, nullScoreTarg.OptimizationParameters); Score jointScore; if (predIsInvariant || targIsInvariant) // cannot compute parameters in this case. They come directly from the single variable params { double jointLL = nullScorePred.Loglikelihood + nullScoreTarg.Loglikelihood; jointScore = Score.GetInstance(jointLL, initParams, AltDistn); } else { MessageInitializerDiscrete altMessageInitializer = MessageInitializerDiscrete.GetInstance(CreateJointMap(predictorMap, targetMap), (DistributionDiscreteJoint)AltDistn, initParams, ModelScorer.PhyloTree.LeafCollection); jointScore = ModelScorer.MaximizeLikelihood(altMessageInitializer); } EvaluationResults evalResults = EvaluationResultsDiscrete.GetInstance(this, nullScores, jointScore, realFisherCounts, ChiSquareDegreesOfFreedom); return(evalResults); }
public override EvaluationResults EvaluateModelOnDataGivenParams(Converter <Leaf, SufficientStatistics> predMap, Converter <Leaf, SufficientStatistics> targMap, EvaluationResults previousResults) { int predCount = ModelScorer.PhyloTree.CountOfNonMissingLeaves(predMap); int targCount = ModelScorer.PhyloTree.CountOfNonMissingLeaves(targMap); int globalNonMissingCount = ModelScorer.PhyloTree.CountOfNonMissingLeaves(predMap, targMap); MessageInitializerGaussian nullMessageInitializer = MessageInitializerGaussian.GetInstance( predMap, targMap, (DistributionGaussianConditional)NullDistns[0], ModelScorer.PhyloTree.LeafCollection); double nullLL = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializer, previousResults.NullScores[0].OptimizationParameters); Score nullScore = Score.GetInstance(nullLL, previousResults.NullScores[0].OptimizationParameters, previousResults.NullScores[0].Distribution); MessageInitializerGaussian altMessageInitializer = MessageInitializerGaussian.GetInstance( predMap, targMap, (DistributionGaussianConditional)AltDistn, ModelScorer.PhyloTree.LeafCollection); double altLL = ModelScorer.ComputeLogLikelihoodModelGivenData(altMessageInitializer, previousResults.AltScore.OptimizationParameters); Score altScore = Score.GetInstance(altLL, previousResults.AltScore.OptimizationParameters, previousResults.AltScore.Distribution); EvaluationResults evalResults = EvaluationResultsGaussian.GetInstance(this, nullScore, altScore, predCount, targCount, globalNonMissingCount, ChiSquareDegreesOfFreedom); return(evalResults); }
//public static Converter<Leaf, SufficientStatistics> TESTPRED, TESTTARG; //public static EvaluationResults TESTEVALRESULTS; public override EvaluationResults EvaluateModelOnDataGivenParams( Converter <Leaf, SufficientStatistics> predictorMap, Converter <Leaf, SufficientStatistics> targetMap, EvaluationResults previousResults) { int[] fisherCounts = ModelScorer.PhyloTree.FisherCounts(predictorMap, targetMap); int targNullIdx = _includePredictorInScore ? 1 : 0; OptimizationParameterList nullParamsTarg = previousResults.NullScores[targNullIdx].OptimizationParameters; MessageInitializerDiscrete nullMessageInitializerTarg = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap, NullDistn, new int[0], ModelScorer.PhyloTree.LeafCollection); double nullLLTarg = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializerTarg, nullParamsTarg); Score nullScoreTarg = Score.GetInstance(nullLLTarg, nullParamsTarg, NullDistn); OptimizationParameterList altParams = previousResults.AltScore.OptimizationParameters; MessageInitializerDiscrete altMessageInitializer = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap, (DistributionDiscreteConditional)AltDistn, new int[0], ModelScorer.PhyloTree.LeafCollection); double condLL = ModelScorer.ComputeLogLikelihoodModelGivenData(altMessageInitializer, altParams); Score altScore = Score.GetInstance(condLL, altParams, AltDistn); List <Score> nullScores = new List <Score>(); if (_includePredictorInScore) { OptimizationParameterList nullParamsPred = previousResults.NullScores[0].OptimizationParameters; MessageInitializerDiscrete nullMessageInitializerPred = MessageInitializerDiscrete.GetInstance(targetMap, predictorMap, NullDistn, new int[0], ModelScorer.PhyloTree.LeafCollection); double nullLLPred = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializerPred, nullParamsPred); Score nullScorePred = Score.GetInstance(nullLLPred, nullParamsPred, NullDistn); nullScores.Add(nullScorePred); // conditional model altScore doesn't include predLL. If we're here, we want to add it to make it comparable to joint or reverseConditional altScore = Score.GetInstance(altScore.Loglikelihood + nullScorePred.Loglikelihood, altScore.OptimizationParameters, altScore.Distribution); } nullScores.Add(nullScoreTarg); EvaluationResults evalResults = EvaluationResultsDiscrete.GetInstance(this, nullScores, altScore, fisherCounts, ChiSquareDegreesOfFreedom); return(evalResults); }
public override EvaluationResults EvaluateModelOnDataGivenParams(Converter <Leaf, SufficientStatistics> predictorMap, Converter <Leaf, SufficientStatistics> targetMap, EvaluationResults previousResults) { int[] fisherCounts = ModelScorer.PhyloTree.FisherCounts(predictorMap, targetMap); OptimizationParameterList nullParamsTarg = previousResults.NullScores[1].OptimizationParameters; MessageInitializerDiscrete nullMessageInitializerTarg = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap, (DistributionDiscreteSingleVariable)NullDistns[0], fisherCounts, ModelScorer.PhyloTree.LeafCollection); double nullLLTarg = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializerTarg, nullParamsTarg); Score nullScoreTarg = Score.GetInstance(nullLLTarg, nullParamsTarg, previousResults.NullScores[1].Distribution); OptimizationParameterList nullParamsPred = previousResults.NullScores[0].OptimizationParameters; MessageInitializerDiscrete nullMessageInitializerPred = MessageInitializerDiscrete.GetInstance(targetMap, predictorMap, (DistributionDiscreteSingleVariable)NullDistns[1], fisherCounts, ModelScorer.PhyloTree.LeafCollection); double nullLLPred = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializerPred, nullParamsPred); Score nullScorePred = Score.GetInstance(nullLLPred, nullParamsPred, previousResults.NullScores[0].Distribution); List <Score> nullScores = new List <Score>(new Score[] { nullScorePred, nullScoreTarg }); OptimizationParameterList altParams = previousResults.AltScore.OptimizationParameters; double altLL; if (((DistributionDiscreteJoint)AltDistn).ParametersCannotBeEvaluated(altParams)) { // we'll get here only if one of the variables is always (or never) true. In this case, the variables must be independent. altLL = nullLLTarg + nullLLPred; } else { MessageInitializerDiscrete altMessageInitializer = MessageInitializerDiscrete.GetInstance(CreateJointMap(predictorMap, targetMap), (DistributionDiscreteJoint)AltDistn, fisherCounts, ModelScorer.PhyloTree.LeafCollection); altLL = ModelScorer.ComputeLogLikelihoodModelGivenData(altMessageInitializer, altParams); } Score altScore = Score.GetInstance(altLL, altParams, previousResults.AltScore.Distribution); EvaluationResults evalResults = EvaluationResultsDiscrete.GetInstance(this, nullScores, altScore, fisherCounts, ChiSquareDegreesOfFreedom); return(evalResults); }
private string CreateReportLine( ModelEvaluator modelEvaluator, RowData rowAndTargetData, UniversalWorkList workList, int rowIndex, int workListCount, int workIndex) { Dictionary <string, string> row = rowAndTargetData.Row; string predictorVariable = row[Tabulate.PredictorVariableColumnName]; string targetVariable = row[Tabulate.TargetVariableColumnName]; int nullIndex = int.Parse(row[Tabulate.NullIndexColumnName]); Dictionary <string, SufficientStatistics> caseIdToNonMissingPredictorValue = rowAndTargetData.PredictorData; Dictionary <string, SufficientStatistics> caseIdToNonMissingTargetValue = rowAndTargetData.TargetData; Converter <Leaf, SufficientStatistics> predictorDistributionClassFunction = CreateSufficientStatisticsMap(caseIdToNonMissingPredictorValue); Converter <Leaf, SufficientStatistics> targetDistributionClassFunction = CreateSufficientStatisticsMap(caseIdToNonMissingTargetValue); EvaluationResults results = modelEvaluator.EvaluateModelOnData(predictorDistributionClassFunction, targetDistributionClassFunction); string reportLine = SpecialFunctions.CreateTabString( results.ModelEvaluator.Name, rowIndex, workListCount, workIndex, nullIndex, predictorVariable, targetVariable, results.ToString()); return(reportLine); }
public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> predMap, Converter <Leaf, SufficientStatistics> targMap) { TwoByTwo fishers2by2 = TwoByTwo.GetInstance( SufficientStatisticsMapToIntDictionaryMap(predMap, _fullLeafCollection), SufficientStatisticsMapToIntDictionaryMap(targMap, _fullLeafCollection)); int[] fisherCounts = fishers2by2.ToOneDArray(); List <Score> nullScores; Score altScore; ComputeIidScores(fisherCounts, out nullScores, out altScore); EvaluationResultsFisher results = EvaluationResultsFisher.GetInstance(this, nullScores, altScore, fishers2by2); #if DEBUG EvaluationResults results2 = EvaluateModelOnDataGivenParams(predMap, targMap, results); double eps = 1E-14; Debug.Assert(ComplexNumber.ApproxEqual(results.AltLL, results2.AltLL, eps)); Debug.Assert(ComplexNumber.ApproxEqual(results.NullLL, results2.NullLL, eps)); Debug.Assert(ComplexNumber.ApproxEqual(results.ComputePValue(), results2.ComputePValue(), eps)); #endif return(results); }
public override EvaluationResults EvaluateModelOnDataGivenParams(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2, EvaluationResults previousResults) { EvaluationResults newResults = previousResults.ModelEvaluator.EvaluateModelOnDataGivenParams(v1, v2, previousResults); return(newResults); }
//public override NullDataGenerator CreateNullDataGenerator(string nullDataGeneratorName) //{ // return NullDataGenerator.GetInstance(nullDataGeneratorName, ModelScorer, NullDistns[0]); //} public override string ToHeaderString() { EvaluationResults dummyResults = CreateDummyResults(new int[4]); return(dummyResults.ToHeaderString()); }
public static EvaluationResultsCrossValidate GetInstance(ModelEvaluator modelEval, EvaluationResults representativeResults) { return(new EvaluationResultsCrossValidate(modelEval, representativeResults.NullScores, representativeResults.AltScore, representativeResults, 1)); }
public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2) { List <Leaf> nonMissingLeaves = new List <Leaf>(100); int seed = 0; foreach (Leaf leaf in ModelScorer.PhyloTree.LeafCollection) { SufficientStatistics class1 = v1(leaf); SufficientStatistics class2 = v2(leaf); if (!class1.IsMissing() && !class2.IsMissing()) { nonMissingLeaves.Add(leaf); seed ^= (leaf.CaseName + class1.ToString() + class2.ToString()).GetHashCode(); } } Random rand = new Random(seed); nonMissingLeaves = SpecialFunctions.Shuffle(nonMissingLeaves, ref rand); int groupSize = nonMissingLeaves.Count / _crossValidateCount; EvaluationResultsCrossValidate combinedResults = null; double testAltLLSum = 0; // for debugging double testNullLLSum = 0; // for debugging for (int i = 0; i < _crossValidateCount; i++) { int testStart = i * groupSize; int trainStart = testStart + groupSize; Set <Leaf> trainSet = new Set <Leaf>(SpecialFunctions.SubList(nonMissingLeaves, trainStart, nonMissingLeaves.Count - trainStart)); trainSet.AddNewRange(SpecialFunctions.SubList(nonMissingLeaves, 0, testStart)); Converter <Leaf, SufficientStatistics> v1Train = CreateFilteredMap(v1, trainSet); Converter <Leaf, SufficientStatistics> v2Train = CreateFilteredMap(v2, trainSet); EvaluationResults trainingResults = InternalEvaluator.EvaluateModelOnData(v1Train, v2Train); EvaluationResults testAndTrainResult = InternalEvaluator.EvaluateModelOnDataGivenParams(v1, v2, trainingResults); EvaluationResultsTestGivenTrain testGivenTrainResult = EvaluationResultsTestGivenTrain.GetInstance(this, trainingResults, testAndTrainResult); if (combinedResults == null) { combinedResults = EvaluationResultsCrossValidate.GetInstance(this, testGivenTrainResult); } else { combinedResults = combinedResults.AddNewResults(testGivenTrainResult); } if (double.IsInfinity(combinedResults.AltLL)) // no point in continuing...infinity will kill everything. { break; } #if DEBUG double eps = 1E-10; EvaluationResults testTrainingResults = InternalEvaluator.EvaluateModelOnDataGivenParams(v1Train, v2Train, trainingResults); Debug.Assert(ComplexNumber.ApproxEqual(testTrainingResults.AltLL, trainingResults.AltLL, eps) && ComplexNumber.ApproxEqual(testTrainingResults.NullLL, trainingResults.NullLL, eps)); //Debug.Assert(testTrainingResults.Equals(trainingResults)); double newNullLL = testAndTrainResult.NullLL - trainingResults.NullLL; double newAltLL = testAndTrainResult.AltLL - trainingResults.AltLL; Debug.Assert(ComplexNumber.ApproxEqual(newNullLL, testGivenTrainResult.NullLL, eps)); Debug.Assert(ComplexNumber.ApproxEqual(newAltLL, testGivenTrainResult.AltLL, eps)); testNullLLSum += newNullLL; testAltLLSum += newAltLL; Debug.Assert(ComplexNumber.ApproxEqual(testNullLLSum, combinedResults.NullLL, eps), "Combined result has wrong NullLL"); Debug.Assert(ComplexNumber.ApproxEqual(testAltLLSum, combinedResults.AltLL, eps), "Combined result has wrong AltLL"); #endif } return(combinedResults); }
// public abstract EvaluationResults EvaluateModelOnData(Converter<Leaf, SufficientStatistics> v1, Converter<Leaf, SufficientStatistics> v2, bool ignoreSimpleCases); public abstract EvaluationResults EvaluateModelOnDataGivenParams(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2, EvaluationResults previousResults);
public static EvaluationResultsTestGivenTrain GetInstance(ModelEvaluator modelEval, EvaluationResults train, EvaluationResults testAndTrain) { List <Score> nullScores = new List <Score>(train.NullScores.Count); for (int i = 0; i < train.NullScores.Count; i++) { Score score = Score.GetInstance( testAndTrain.NullScores[i].Loglikelihood - train.NullScores[i].Loglikelihood, testAndTrain.NullScores[i].OptimizationParameters, testAndTrain.NullScores[i].Distribution); nullScores.Add(score); } Score altScore = Score.GetInstance( testAndTrain.AltScore.Loglikelihood - train.AltScore.Loglikelihood, testAndTrain.AltScore.OptimizationParameters, testAndTrain.AltScore.Distribution); return(new EvaluationResultsTestGivenTrain(modelEval, testAndTrain, nullScores, altScore)); }
public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> predictorMap, Converter <Leaf, SufficientStatistics> targetMap) { EvaluationResults evalResults; int[] fisherCounts = ModelScorer.PhyloTree.FisherCounts(predictorMap, targetMap); int[] realFisherCounts = fisherCounts; // for compatability when NAIVE_EQUILIBRIUM is set #if NAIVE_EQUILIBRIUM //USE THIS FOR BACKWARDS COMPATABILITY int[] tempCounts = ModelScorer.PhyloTree.CountsOfLeaves(targetMap); fisherCounts = tempCounts; #endif //MessageInitializerDiscrete nullMessageInitializer = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap, NullDistn, fisherCounts, ModelScorer.PhyloTree.LeafCollection); //if (TryShortCutFromCounts(realFisherCounts, nullMessageInitializer, out evalResults)) //{ // return evalResults; //} //Score nullScore = ModelScorer.MaximizeLikelihood(nullMessageInitializer); bool isInvariant; Score nullScoreTarg = ComputeSingleVariableScore(predictorMap, targetMap, NullDistn, fisherCounts, out isInvariant); Score altScore = ComputeConditionalVariableScore(predictorMap, targetMap, nullScoreTarg, fisherCounts); //(realFisherCounts, nullScoreTarg, out evalResults)) //{ // return evalResults; //} //MessageInitializerDiscrete altMessageInitializer = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap, (DistributionDiscreteConditional)AltDistn, nullScore.OptimizationParameters, ModelScorer.PhyloTree.LeafCollection); //Score condScore = ModelScorer.MaximizeLikelihood(altMessageInitializer); List <Score> nullScores = new List <Score>(); if (_includePredictorInScore) { int[] predFisherCounts = new int[] { realFisherCounts[0], realFisherCounts[2], realFisherCounts[1], realFisherCounts[3] }; Score predNullScore = ComputeSingleVariableScore(targetMap, predictorMap, NullDistn, predFisherCounts, out isInvariant); nullScores.Add(predNullScore); // conditional model altScore doesn't include predLL. If we're here, we want to add it to make it comparable to joint or reverseConditional altScore = Score.GetInstance(altScore.Loglikelihood + predNullScore.Loglikelihood, altScore.OptimizationParameters, altScore.Distribution); } nullScores.Add(nullScoreTarg); evalResults = EvaluationResultsDiscrete.GetInstance(this, nullScores, altScore, realFisherCounts, ChiSquareDegreesOfFreedom); #if DEBUG MessageInitializerDiscrete nullMessageInitializer = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap, NullDistn, fisherCounts, ModelScorer.PhyloTree.LeafCollection); MessageInitializerDiscrete altMessageInitializer = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap, (DistributionDiscreteConditional)AltDistn, nullScoreTarg.OptimizationParameters, ModelScorer.PhyloTree.LeafCollection); double nullLL = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializer, nullScoreTarg.OptimizationParameters); double altLL = ModelScorer.ComputeLogLikelihoodModelGivenData(altMessageInitializer, altScore.OptimizationParameters); if (_includePredictorInScore) { int[] predFisherCounts = new int[] { realFisherCounts[0], realFisherCounts[2], realFisherCounts[1], realFisherCounts[3] }; MessageInitializerDiscrete nullMessageInitializerPred = MessageInitializerDiscrete.GetInstance(targetMap, predictorMap, NullDistn, predFisherCounts, ModelScorer.PhyloTree.LeafCollection); double nullLLPred = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializerPred, nullScores[0].OptimizationParameters); altLL += nullLLPred; } EvaluationResults evalResults2 = EvaluateModelOnDataGivenParams(predictorMap, targetMap, evalResults); double eps = 1E-10; Debug.Assert(ComplexNumber.ApproxEqual(nullLL, nullScoreTarg.Loglikelihood, eps)); Debug.Assert(ComplexNumber.ApproxEqual(altLL, altScore.Loglikelihood, eps)); Debug.Assert(ComplexNumber.ApproxEqual(evalResults.NullLL, evalResults2.NullLL, eps) && ComplexNumber.ApproxEqual(evalResults.AltLL, evalResults2.AltLL, eps), "In ModelEvaluatorCond, results of maximizing LL and computing LL from same params are not the same."); #endif return(evalResults); }
protected EvaluationResultsTestGivenTrain(ModelEvaluator modelEval, EvaluationResults testAndTrain, List <Score> testGiveTrainNullScores, Score testGivenTrainAltScore) : base(modelEval, testGiveTrainNullScores, testGivenTrainAltScore, testAndTrain.ChiSquareDegreesOfFreedom, testAndTrain.GlobalNonMissingCount) { _testAndTrain = testAndTrain; }