public override EvaluationResults EvaluateModelOnDataGivenParams(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2, EvaluationResults previousResults)
{
EvaluationResults internalResults = _internalEvaluator.EvaluateModelOnDataGivenParams(v2, v1, previousResults);
internalResults.ModelEvaluator = this;
return(internalResults);
}
protected EvaluationResultsCrossValidate(ModelEvaluator modelEval, List <Score> nullScores, Score altScore, EvaluationResults representativeResults, int numberOfResults)
:
base(modelEval, nullScores, altScore, representativeResults.ChiSquareDegreesOfFreedom, representativeResults.GlobalNonMissingCount)
{
_representativeResults = representativeResults;
_numberOfResults = numberOfResults;
}
public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2)
{
EvaluationResults internalResults = _internalEvaluator.EvaluateModelOnData(v2, v1);
internalResults.ModelEvaluator = this;
return(internalResults);
}
protected override EvaluationResults CreateDummyResults(int[] fisherCounts)
{
EvaluationResults result = base.CreateDummyResults(fisherCounts);
if (_includePredictorInScore)
{
result.NullScores.Add(Score.GetInstance(result.NullScores[0].Loglikelihood, result.NullScores[0].OptimizationParameters, result.NullScores[0].Distribution));
}
return(result);
}
public override EvaluationResults EvaluateModelOnDataGivenParams(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2, EvaluationResults previousResults)
{
TwoByTwo fishers2by2 = TwoByTwo.GetInstance(
SufficientStatisticsMapToIntDictionaryMap(v1, _fullLeafCollection),
SufficientStatisticsMapToIntDictionaryMap(v2, _fullLeafCollection));
int[] fisherCounts = fishers2by2.ToOneDArray();
List <Score> nullScores;
Score altScore;
ComputeIidScoresGivenParams(fisherCounts,
previousResults.NullScores[0].OptimizationParameters, previousResults.NullScores[1].OptimizationParameters,
previousResults.AltScore.OptimizationParameters,
out nullScores, out altScore);
//int tt = fisherCounts[(int)TwoByTwo.ParameterIndex.TT];
//int tf = fisherCounts[(int)TwoByTwo.ParameterIndex.TF];
//int ft = fisherCounts[(int)TwoByTwo.ParameterIndex.FT];
//int ff = fisherCounts[(int)TwoByTwo.ParameterIndex.FF];
//int sum = SpecialFunctions.Sum(fisherCounts);
//double pi0 = (double)(tt + tf) / sum;
//double pi1 = (double)(tt + ft) / sum;
//double ptt = (double)tt / sum;
//double ptf = (double)tf / sum;
//double pft = (double)ft / sum;
//double pff = 1 - ptt - ptf - pft;
//double predicted_pi0 = previousResults.NullScores[0].OptimizationParameters[0].Value;
//double predicted_pi1 = previousResults.NullScores[1].OptimizationParameters[0].Value;
//double predicted_ptt = previousResults.AltScore.OptimizationParameters[0].Value;
//double predicted_ptf = previousResults.AltScore.OptimizationParameters[1].Value;
//double predicted_pft = previousResults.AltScore.OptimizationParameters[2].Value;
//double predicted_pff = 1 - predicted_ptt - predicted_ptf - predicted_pft;
//double nullLLLeft = pi0 * Math.Log(predicted_pi0);
//double nullLLRight = pi1 * Math.Log(predicted_pi1);
//double altLL = predicted_ptt * Math.Log(ptt) + predicted_ptf * Math.Log(ptf) + predicted_pft * Math.Log(pft) + predicted_pff * Math.Log(pff);
//Score nullScoreLeft = Score.GetInstance(nullLLLeft, previousResults.NullScores[0].OptimizationParameters, null);
//Score nullScoreRight = Score.GetInstance(nullLLRight, previousResults.NullScores[1].OptimizationParameters, null);
//Score altScore = Score.GetInstance(altLL, previousResults.AltScore.OptimizationParameters, null);
//List<Score> nullScores = new List<Score>(2);
//nullScores.Add(nullScoreLeft);
//nullScores.Add(nullScoreRight);
EvaluationResults results = EvaluationResultsFisher.GetInstance(this, nullScores, altScore, fishers2by2);
return(results);
}
public override string ToHeaderString()
{
//EvaluationResults dummyResults = EvaluationResultsFisher.GetInstance(this, TwoByTwo.GetInstance(new int[] { 1, 1, 1, 1 }));
List <Score> nullScores;
Score altScore;
TwoByTwo fishers2by2 = TwoByTwo.GetInstance(new int[] { 1, 1, 1, 1 });
ComputeIidScores(fishers2by2.ToOneDArray(), out nullScores, out altScore);
EvaluationResults dummyResults = EvaluationResultsFisher.GetInstance(this, nullScores, altScore, fishers2by2);
return(dummyResults.ToHeaderString());
}
public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2)
{
EvaluationResults bestResults = null;
foreach (ModelEvaluatorCrossValidate model in _modelsToEvaluate)
{
EvaluationResults results = model.EvaluateModelOnData(v1, v2);
if (bestResults == null || results.AltScore.Loglikelihood > bestResults.AltScore.Loglikelihood)
{
bestResults = results;
}
}
//EvaluationResults resultsFromFullDataset = bestResults.ModelEvaluator.EvaluateModelOnData(v1, v2);
EvaluationResults resultsFromFullDataset = ((ModelEvaluatorCrossValidate)(bestResults.ModelEvaluator)).InternalEvaluator.EvaluateModelOnData(v1, v2);
return(resultsFromFullDataset);
}
public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> predMap, Converter <Leaf, SufficientStatistics> targMap)
{
int predCount = ModelScorer.PhyloTree.CountOfNonMissingLeaves(predMap);
int targCount = ModelScorer.PhyloTree.CountOfNonMissingLeaves(targMap);
int globalNonMissingCount = ModelScorer.PhyloTree.CountOfNonMissingLeaves(predMap, targMap);
MessageInitializerGaussian nullMessageInitializer = MessageInitializerGaussian.GetInstance(
predMap, targMap, (DistributionGaussianConditional)NullDistns[0], ModelScorer.PhyloTree.LeafCollection);
Score nullScore = ModelScorer.MaximizeLikelihood(nullMessageInitializer);
MessageInitializerGaussian altMessageInitializer = MessageInitializerGaussian.GetInstance(
predMap, targMap, (DistributionGaussianConditional)AltDistn, ModelScorer.PhyloTree.LeafCollection);
Score altScore = ModelScorer.MaximizeLikelihood(altMessageInitializer);
EvaluationResults evalResults = EvaluationResultsGaussian.GetInstance(this, nullScore, altScore, predCount, targCount, globalNonMissingCount, ChiSquareDegreesOfFreedom);
return(evalResults);
}
public EvaluationResultsCrossValidate AddNewResults(EvaluationResults newResults)
{
SpecialFunctions.CheckCondition(newResults.GetType() == _representativeResults.GetType(), "Evaluation results can only be combined if their types are identical.");
// first average the parameters.
double newParamWeight = 1.0 / (_numberOfResults + 1);
List <Score> newNullScores = new List <Score>(NullScores.Count);
for (int i = 0; i < NullScores.Count; i++)
{
Score newScore = CombineScores(NullScores[i], newResults.NullScores[i], newParamWeight);
newNullScores.Add(newScore);
}
Score newAltScore = CombineScores(AltScore, newResults.AltScore, newParamWeight);
EvaluationResultsCrossValidate result = new EvaluationResultsCrossValidate(ModelEvaluator, newNullScores, newAltScore, newResults, _numberOfResults + 1);
return(result);
}
public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> predictorMap, Converter <Leaf, SufficientStatistics> targetMap)
{
int[] realFisherCounts = ModelScorer.PhyloTree.FisherCounts(predictorMap, targetMap);
int tt = realFisherCounts[(int)TwoByTwo.ParameterIndex.TT];
int tf = realFisherCounts[(int)TwoByTwo.ParameterIndex.TF];
int ft = realFisherCounts[(int)TwoByTwo.ParameterIndex.FT];
int ff = realFisherCounts[(int)TwoByTwo.ParameterIndex.FF];
int[] fisherCountsPred = new int[] { tt, ft, tf, ff }; //ModelScorer.PhyloTree.FisherCounts(targetMap, predictorMap);
int[] fisherCountsTarg = realFisherCounts;
#if NAIVE_EQUILIBRIUM
//USE THIS FOR BACKWARDS COMPATABILITY
int[] tempCountsPred = ModelScorer.PhyloTree.CountsOfLeaves(predictorMap);
int[] tempCountsTarg = ModelScorer.PhyloTree.CountsOfLeaves(targetMap);
fisherCountsPred = tempCountsPred;
fisherCountsTarg = tempCountsTarg;
#endif
bool predIsInvariant, targIsInvariant;
Score nullScorePred = ComputeSingleVariableScore(targetMap, predictorMap, (DistributionDiscreteSingleVariable)NullDistns[0], fisherCountsPred, out predIsInvariant);
Score nullScoreTarg = ComputeSingleVariableScore(predictorMap, targetMap, (DistributionDiscreteSingleVariable)NullDistns[1], fisherCountsTarg, out targIsInvariant);
List <Score> nullScores = new List <Score>(new Score[] { nullScorePred, nullScoreTarg });
OptimizationParameterList initParams = ((DistributionDiscreteJoint)AltDistn).GenerateInitialParams(nullScorePred.OptimizationParameters, nullScoreTarg.OptimizationParameters);
Score jointScore;
if (predIsInvariant || targIsInvariant) // cannot compute parameters in this case. They come directly from the single variable params
{
double jointLL = nullScorePred.Loglikelihood + nullScoreTarg.Loglikelihood;
jointScore = Score.GetInstance(jointLL, initParams, AltDistn);
}
else
{
MessageInitializerDiscrete altMessageInitializer = MessageInitializerDiscrete.GetInstance(CreateJointMap(predictorMap, targetMap), (DistributionDiscreteJoint)AltDistn, initParams, ModelScorer.PhyloTree.LeafCollection);
jointScore = ModelScorer.MaximizeLikelihood(altMessageInitializer);
}
EvaluationResults evalResults = EvaluationResultsDiscrete.GetInstance(this, nullScores, jointScore, realFisherCounts, ChiSquareDegreesOfFreedom);
return(evalResults);
}
public override EvaluationResults EvaluateModelOnDataGivenParams(Converter <Leaf, SufficientStatistics> predMap, Converter <Leaf, SufficientStatistics> targMap, EvaluationResults previousResults)
{
int predCount = ModelScorer.PhyloTree.CountOfNonMissingLeaves(predMap);
int targCount = ModelScorer.PhyloTree.CountOfNonMissingLeaves(targMap);
int globalNonMissingCount = ModelScorer.PhyloTree.CountOfNonMissingLeaves(predMap, targMap);
MessageInitializerGaussian nullMessageInitializer = MessageInitializerGaussian.GetInstance(
predMap, targMap, (DistributionGaussianConditional)NullDistns[0], ModelScorer.PhyloTree.LeafCollection);
double nullLL = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializer, previousResults.NullScores[0].OptimizationParameters);
Score nullScore = Score.GetInstance(nullLL, previousResults.NullScores[0].OptimizationParameters, previousResults.NullScores[0].Distribution);
MessageInitializerGaussian altMessageInitializer = MessageInitializerGaussian.GetInstance(
predMap, targMap, (DistributionGaussianConditional)AltDistn, ModelScorer.PhyloTree.LeafCollection);
double altLL = ModelScorer.ComputeLogLikelihoodModelGivenData(altMessageInitializer, previousResults.AltScore.OptimizationParameters);
Score altScore = Score.GetInstance(altLL, previousResults.AltScore.OptimizationParameters, previousResults.AltScore.Distribution);
EvaluationResults evalResults = EvaluationResultsGaussian.GetInstance(this, nullScore, altScore, predCount, targCount, globalNonMissingCount, ChiSquareDegreesOfFreedom);
return(evalResults);
}
//public static Converter<Leaf, SufficientStatistics> TESTPRED, TESTTARG;
//public static EvaluationResults TESTEVALRESULTS;
public override EvaluationResults EvaluateModelOnDataGivenParams(
Converter <Leaf, SufficientStatistics> predictorMap, Converter <Leaf, SufficientStatistics> targetMap, EvaluationResults previousResults)
{
int[] fisherCounts = ModelScorer.PhyloTree.FisherCounts(predictorMap, targetMap);
int targNullIdx = _includePredictorInScore ? 1 : 0;
OptimizationParameterList nullParamsTarg = previousResults.NullScores[targNullIdx].OptimizationParameters;
MessageInitializerDiscrete nullMessageInitializerTarg = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap,
NullDistn, new int[0], ModelScorer.PhyloTree.LeafCollection);
double nullLLTarg = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializerTarg, nullParamsTarg);
Score nullScoreTarg = Score.GetInstance(nullLLTarg, nullParamsTarg, NullDistn);
OptimizationParameterList altParams = previousResults.AltScore.OptimizationParameters;
MessageInitializerDiscrete altMessageInitializer = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap,
(DistributionDiscreteConditional)AltDistn, new int[0], ModelScorer.PhyloTree.LeafCollection);
double condLL = ModelScorer.ComputeLogLikelihoodModelGivenData(altMessageInitializer, altParams);
Score altScore = Score.GetInstance(condLL, altParams, AltDistn);
List <Score> nullScores = new List <Score>();
if (_includePredictorInScore)
{
OptimizationParameterList nullParamsPred = previousResults.NullScores[0].OptimizationParameters;
MessageInitializerDiscrete nullMessageInitializerPred = MessageInitializerDiscrete.GetInstance(targetMap, predictorMap,
NullDistn, new int[0], ModelScorer.PhyloTree.LeafCollection);
double nullLLPred = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializerPred, nullParamsPred);
Score nullScorePred = Score.GetInstance(nullLLPred, nullParamsPred, NullDistn);
nullScores.Add(nullScorePred);
// conditional model altScore doesn't include predLL. If we're here, we want to add it to make it comparable to joint or reverseConditional
altScore = Score.GetInstance(altScore.Loglikelihood + nullScorePred.Loglikelihood, altScore.OptimizationParameters, altScore.Distribution);
}
nullScores.Add(nullScoreTarg);
EvaluationResults evalResults = EvaluationResultsDiscrete.GetInstance(this, nullScores, altScore, fisherCounts, ChiSquareDegreesOfFreedom);
return(evalResults);
}
public override EvaluationResults EvaluateModelOnDataGivenParams(Converter <Leaf, SufficientStatistics> predictorMap, Converter <Leaf, SufficientStatistics> targetMap, EvaluationResults previousResults)
{
int[] fisherCounts = ModelScorer.PhyloTree.FisherCounts(predictorMap, targetMap);
OptimizationParameterList nullParamsTarg = previousResults.NullScores[1].OptimizationParameters;
MessageInitializerDiscrete nullMessageInitializerTarg = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap, (DistributionDiscreteSingleVariable)NullDistns[0], fisherCounts, ModelScorer.PhyloTree.LeafCollection);
double nullLLTarg = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializerTarg, nullParamsTarg);
Score nullScoreTarg = Score.GetInstance(nullLLTarg, nullParamsTarg, previousResults.NullScores[1].Distribution);
OptimizationParameterList nullParamsPred = previousResults.NullScores[0].OptimizationParameters;
MessageInitializerDiscrete nullMessageInitializerPred = MessageInitializerDiscrete.GetInstance(targetMap, predictorMap, (DistributionDiscreteSingleVariable)NullDistns[1], fisherCounts, ModelScorer.PhyloTree.LeafCollection);
double nullLLPred = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializerPred, nullParamsPred);
Score nullScorePred = Score.GetInstance(nullLLPred, nullParamsPred, previousResults.NullScores[0].Distribution);
List <Score> nullScores = new List <Score>(new Score[] { nullScorePred, nullScoreTarg });
OptimizationParameterList altParams = previousResults.AltScore.OptimizationParameters;
double altLL;
if (((DistributionDiscreteJoint)AltDistn).ParametersCannotBeEvaluated(altParams))
{
// we'll get here only if one of the variables is always (or never) true. In this case, the variables must be independent.
altLL = nullLLTarg + nullLLPred;
}
else
{
MessageInitializerDiscrete altMessageInitializer = MessageInitializerDiscrete.GetInstance(CreateJointMap(predictorMap, targetMap), (DistributionDiscreteJoint)AltDistn, fisherCounts, ModelScorer.PhyloTree.LeafCollection);
altLL = ModelScorer.ComputeLogLikelihoodModelGivenData(altMessageInitializer, altParams);
}
Score altScore = Score.GetInstance(altLL, altParams, previousResults.AltScore.Distribution);
EvaluationResults evalResults = EvaluationResultsDiscrete.GetInstance(this, nullScores, altScore, fisherCounts, ChiSquareDegreesOfFreedom);
return(evalResults);
}
private string CreateReportLine(
ModelEvaluator modelEvaluator,
RowData rowAndTargetData,
UniversalWorkList workList,
int rowIndex, int workListCount, int workIndex)
{
Dictionary <string, string> row = rowAndTargetData.Row;
string predictorVariable = row[Tabulate.PredictorVariableColumnName];
string targetVariable = row[Tabulate.TargetVariableColumnName];
int nullIndex = int.Parse(row[Tabulate.NullIndexColumnName]);
Dictionary <string, SufficientStatistics> caseIdToNonMissingPredictorValue = rowAndTargetData.PredictorData;
Dictionary <string, SufficientStatistics> caseIdToNonMissingTargetValue = rowAndTargetData.TargetData;
Converter <Leaf, SufficientStatistics> predictorDistributionClassFunction = CreateSufficientStatisticsMap(caseIdToNonMissingPredictorValue);
Converter <Leaf, SufficientStatistics> targetDistributionClassFunction = CreateSufficientStatisticsMap(caseIdToNonMissingTargetValue);
EvaluationResults results = modelEvaluator.EvaluateModelOnData(predictorDistributionClassFunction, targetDistributionClassFunction);
string reportLine = SpecialFunctions.CreateTabString(
results.ModelEvaluator.Name, rowIndex, workListCount, workIndex, nullIndex, predictorVariable, targetVariable, results.ToString());
return(reportLine);
}
public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> predMap, Converter <Leaf, SufficientStatistics> targMap)
{
TwoByTwo fishers2by2 = TwoByTwo.GetInstance(
SufficientStatisticsMapToIntDictionaryMap(predMap, _fullLeafCollection),
SufficientStatisticsMapToIntDictionaryMap(targMap, _fullLeafCollection));
int[] fisherCounts = fishers2by2.ToOneDArray();
List <Score> nullScores;
Score altScore;
ComputeIidScores(fisherCounts, out nullScores, out altScore);
EvaluationResultsFisher results = EvaluationResultsFisher.GetInstance(this, nullScores, altScore, fishers2by2);
#if DEBUG
EvaluationResults results2 = EvaluateModelOnDataGivenParams(predMap, targMap, results);
double eps = 1E-14;
Debug.Assert(ComplexNumber.ApproxEqual(results.AltLL, results2.AltLL, eps));
Debug.Assert(ComplexNumber.ApproxEqual(results.NullLL, results2.NullLL, eps));
Debug.Assert(ComplexNumber.ApproxEqual(results.ComputePValue(), results2.ComputePValue(), eps));
#endif
return(results);
}
public override EvaluationResults EvaluateModelOnDataGivenParams(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2, EvaluationResults previousResults)
{
EvaluationResults newResults = previousResults.ModelEvaluator.EvaluateModelOnDataGivenParams(v1, v2, previousResults);
return(newResults);
}
//public override NullDataGenerator CreateNullDataGenerator(string nullDataGeneratorName)
//{
// return NullDataGenerator.GetInstance(nullDataGeneratorName, ModelScorer, NullDistns[0]);
//}
public override string ToHeaderString()
{
EvaluationResults dummyResults = CreateDummyResults(new int[4]);
return(dummyResults.ToHeaderString());
}
public static EvaluationResultsCrossValidate GetInstance(ModelEvaluator modelEval, EvaluationResults representativeResults)
{
return(new EvaluationResultsCrossValidate(modelEval, representativeResults.NullScores, representativeResults.AltScore, representativeResults, 1));
}
public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2)
{
List <Leaf> nonMissingLeaves = new List <Leaf>(100);
int seed = 0;
foreach (Leaf leaf in ModelScorer.PhyloTree.LeafCollection)
{
SufficientStatistics class1 = v1(leaf);
SufficientStatistics class2 = v2(leaf);
if (!class1.IsMissing() && !class2.IsMissing())
{
nonMissingLeaves.Add(leaf);
seed ^= (leaf.CaseName + class1.ToString() + class2.ToString()).GetHashCode();
}
}
Random rand = new Random(seed);
nonMissingLeaves = SpecialFunctions.Shuffle(nonMissingLeaves, ref rand);
int groupSize = nonMissingLeaves.Count / _crossValidateCount;
EvaluationResultsCrossValidate combinedResults = null;
double testAltLLSum = 0; // for debugging
double testNullLLSum = 0; // for debugging
for (int i = 0; i < _crossValidateCount; i++)
{
int testStart = i * groupSize;
int trainStart = testStart + groupSize;
Set <Leaf> trainSet = new Set <Leaf>(SpecialFunctions.SubList(nonMissingLeaves, trainStart, nonMissingLeaves.Count - trainStart));
trainSet.AddNewRange(SpecialFunctions.SubList(nonMissingLeaves, 0, testStart));
Converter <Leaf, SufficientStatistics> v1Train = CreateFilteredMap(v1, trainSet);
Converter <Leaf, SufficientStatistics> v2Train = CreateFilteredMap(v2, trainSet);
EvaluationResults trainingResults = InternalEvaluator.EvaluateModelOnData(v1Train, v2Train);
EvaluationResults testAndTrainResult = InternalEvaluator.EvaluateModelOnDataGivenParams(v1, v2, trainingResults);
EvaluationResultsTestGivenTrain testGivenTrainResult = EvaluationResultsTestGivenTrain.GetInstance(this, trainingResults, testAndTrainResult);
if (combinedResults == null)
{
combinedResults = EvaluationResultsCrossValidate.GetInstance(this, testGivenTrainResult);
}
else
{
combinedResults = combinedResults.AddNewResults(testGivenTrainResult);
}
if (double.IsInfinity(combinedResults.AltLL)) // no point in continuing...infinity will kill everything.
{
break;
}
#if DEBUG
double eps = 1E-10;
EvaluationResults testTrainingResults = InternalEvaluator.EvaluateModelOnDataGivenParams(v1Train, v2Train, trainingResults);
Debug.Assert(ComplexNumber.ApproxEqual(testTrainingResults.AltLL, trainingResults.AltLL, eps) &&
ComplexNumber.ApproxEqual(testTrainingResults.NullLL, trainingResults.NullLL, eps));
//Debug.Assert(testTrainingResults.Equals(trainingResults));
double newNullLL = testAndTrainResult.NullLL - trainingResults.NullLL;
double newAltLL = testAndTrainResult.AltLL - trainingResults.AltLL;
Debug.Assert(ComplexNumber.ApproxEqual(newNullLL, testGivenTrainResult.NullLL, eps));
Debug.Assert(ComplexNumber.ApproxEqual(newAltLL, testGivenTrainResult.AltLL, eps));
testNullLLSum += newNullLL;
testAltLLSum += newAltLL;
Debug.Assert(ComplexNumber.ApproxEqual(testNullLLSum, combinedResults.NullLL, eps), "Combined result has wrong NullLL");
Debug.Assert(ComplexNumber.ApproxEqual(testAltLLSum, combinedResults.AltLL, eps), "Combined result has wrong AltLL");
#endif
}
return(combinedResults);
}
// public abstract EvaluationResults EvaluateModelOnData(Converter<Leaf, SufficientStatistics> v1, Converter<Leaf, SufficientStatistics> v2, bool ignoreSimpleCases); public abstract EvaluationResults EvaluateModelOnDataGivenParams(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2, EvaluationResults previousResults);
public static EvaluationResultsTestGivenTrain GetInstance(ModelEvaluator modelEval, EvaluationResults train, EvaluationResults testAndTrain)
{
List <Score> nullScores = new List <Score>(train.NullScores.Count);
for (int i = 0; i < train.NullScores.Count; i++)
{
Score score = Score.GetInstance(
testAndTrain.NullScores[i].Loglikelihood - train.NullScores[i].Loglikelihood,
testAndTrain.NullScores[i].OptimizationParameters,
testAndTrain.NullScores[i].Distribution);
nullScores.Add(score);
}
Score altScore = Score.GetInstance(
testAndTrain.AltScore.Loglikelihood - train.AltScore.Loglikelihood,
testAndTrain.AltScore.OptimizationParameters,
testAndTrain.AltScore.Distribution);
return(new EvaluationResultsTestGivenTrain(modelEval, testAndTrain, nullScores, altScore));
}
public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> predictorMap, Converter <Leaf, SufficientStatistics> targetMap)
{
EvaluationResults evalResults;
int[] fisherCounts = ModelScorer.PhyloTree.FisherCounts(predictorMap, targetMap);
int[] realFisherCounts = fisherCounts; // for compatability when NAIVE_EQUILIBRIUM is set
#if NAIVE_EQUILIBRIUM
//USE THIS FOR BACKWARDS COMPATABILITY
int[] tempCounts = ModelScorer.PhyloTree.CountsOfLeaves(targetMap);
fisherCounts = tempCounts;
#endif
//MessageInitializerDiscrete nullMessageInitializer = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap, NullDistn, fisherCounts, ModelScorer.PhyloTree.LeafCollection);
//if (TryShortCutFromCounts(realFisherCounts, nullMessageInitializer, out evalResults))
//{
// return evalResults;
//}
//Score nullScore = ModelScorer.MaximizeLikelihood(nullMessageInitializer);
bool isInvariant;
Score nullScoreTarg = ComputeSingleVariableScore(predictorMap, targetMap, NullDistn, fisherCounts, out isInvariant);
Score altScore = ComputeConditionalVariableScore(predictorMap, targetMap, nullScoreTarg, fisherCounts);
//(realFisherCounts, nullScoreTarg, out evalResults))
//{
// return evalResults;
//}
//MessageInitializerDiscrete altMessageInitializer = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap, (DistributionDiscreteConditional)AltDistn, nullScore.OptimizationParameters, ModelScorer.PhyloTree.LeafCollection);
//Score condScore = ModelScorer.MaximizeLikelihood(altMessageInitializer);
List <Score> nullScores = new List <Score>();
if (_includePredictorInScore)
{
int[] predFisherCounts = new int[] { realFisherCounts[0], realFisherCounts[2], realFisherCounts[1], realFisherCounts[3] };
Score predNullScore = ComputeSingleVariableScore(targetMap, predictorMap, NullDistn, predFisherCounts, out isInvariant);
nullScores.Add(predNullScore);
// conditional model altScore doesn't include predLL. If we're here, we want to add it to make it comparable to joint or reverseConditional
altScore = Score.GetInstance(altScore.Loglikelihood + predNullScore.Loglikelihood, altScore.OptimizationParameters, altScore.Distribution);
}
nullScores.Add(nullScoreTarg);
evalResults = EvaluationResultsDiscrete.GetInstance(this, nullScores, altScore, realFisherCounts, ChiSquareDegreesOfFreedom);
#if DEBUG
MessageInitializerDiscrete nullMessageInitializer = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap, NullDistn, fisherCounts, ModelScorer.PhyloTree.LeafCollection);
MessageInitializerDiscrete altMessageInitializer = MessageInitializerDiscrete.GetInstance(predictorMap, targetMap, (DistributionDiscreteConditional)AltDistn, nullScoreTarg.OptimizationParameters, ModelScorer.PhyloTree.LeafCollection);
double nullLL = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializer, nullScoreTarg.OptimizationParameters);
double altLL = ModelScorer.ComputeLogLikelihoodModelGivenData(altMessageInitializer, altScore.OptimizationParameters);
if (_includePredictorInScore)
{
int[] predFisherCounts = new int[] { realFisherCounts[0], realFisherCounts[2], realFisherCounts[1], realFisherCounts[3] };
MessageInitializerDiscrete nullMessageInitializerPred = MessageInitializerDiscrete.GetInstance(targetMap, predictorMap, NullDistn, predFisherCounts, ModelScorer.PhyloTree.LeafCollection);
double nullLLPred = ModelScorer.ComputeLogLikelihoodModelGivenData(nullMessageInitializerPred, nullScores[0].OptimizationParameters);
altLL += nullLLPred;
}
EvaluationResults evalResults2 = EvaluateModelOnDataGivenParams(predictorMap, targetMap, evalResults);
double eps = 1E-10;
Debug.Assert(ComplexNumber.ApproxEqual(nullLL, nullScoreTarg.Loglikelihood, eps));
Debug.Assert(ComplexNumber.ApproxEqual(altLL, altScore.Loglikelihood, eps));
Debug.Assert(ComplexNumber.ApproxEqual(evalResults.NullLL, evalResults2.NullLL, eps) && ComplexNumber.ApproxEqual(evalResults.AltLL, evalResults2.AltLL, eps), "In ModelEvaluatorCond, results of maximizing LL and computing LL from same params are not the same.");
#endif
return(evalResults);
}
protected EvaluationResultsTestGivenTrain(ModelEvaluator modelEval, EvaluationResults testAndTrain, List <Score> testGiveTrainNullScores, Score testGivenTrainAltScore)
:
base(modelEval, testGiveTrainNullScores, testGivenTrainAltScore, testAndTrain.ChiSquareDegreesOfFreedom, testAndTrain.GlobalNonMissingCount)
{
_testAndTrain = testAndTrain;
}