public override EvaluationResults EvaluateModelOnData(Converter <Leaf, SufficientStatistics> v1, Converter <Leaf, SufficientStatistics> v2) { List <Leaf> nonMissingLeaves = new List <Leaf>(100); int seed = 0; foreach (Leaf leaf in ModelScorer.PhyloTree.LeafCollection) { SufficientStatistics class1 = v1(leaf); SufficientStatistics class2 = v2(leaf); if (!class1.IsMissing() && !class2.IsMissing()) { nonMissingLeaves.Add(leaf); seed ^= (leaf.CaseName + class1.ToString() + class2.ToString()).GetHashCode(); } } Random rand = new Random(seed); nonMissingLeaves = SpecialFunctions.Shuffle(nonMissingLeaves, ref rand); int groupSize = nonMissingLeaves.Count / _crossValidateCount; EvaluationResultsCrossValidate combinedResults = null; double testAltLLSum = 0; // for debugging double testNullLLSum = 0; // for debugging for (int i = 0; i < _crossValidateCount; i++) { int testStart = i * groupSize; int trainStart = testStart + groupSize; Set <Leaf> trainSet = new Set <Leaf>(SpecialFunctions.SubList(nonMissingLeaves, trainStart, nonMissingLeaves.Count - trainStart)); trainSet.AddNewRange(SpecialFunctions.SubList(nonMissingLeaves, 0, testStart)); Converter <Leaf, SufficientStatistics> v1Train = CreateFilteredMap(v1, trainSet); Converter <Leaf, SufficientStatistics> v2Train = CreateFilteredMap(v2, trainSet); EvaluationResults trainingResults = InternalEvaluator.EvaluateModelOnData(v1Train, v2Train); EvaluationResults testAndTrainResult = InternalEvaluator.EvaluateModelOnDataGivenParams(v1, v2, trainingResults); EvaluationResultsTestGivenTrain testGivenTrainResult = EvaluationResultsTestGivenTrain.GetInstance(this, trainingResults, testAndTrainResult); if (combinedResults == null) { combinedResults = EvaluationResultsCrossValidate.GetInstance(this, testGivenTrainResult); } else { combinedResults = combinedResults.AddNewResults(testGivenTrainResult); } if (double.IsInfinity(combinedResults.AltLL)) // no point in continuing...infinity will kill everything. { break; } #if DEBUG double eps = 1E-10; EvaluationResults testTrainingResults = InternalEvaluator.EvaluateModelOnDataGivenParams(v1Train, v2Train, trainingResults); Debug.Assert(ComplexNumber.ApproxEqual(testTrainingResults.AltLL, trainingResults.AltLL, eps) && ComplexNumber.ApproxEqual(testTrainingResults.NullLL, trainingResults.NullLL, eps)); //Debug.Assert(testTrainingResults.Equals(trainingResults)); double newNullLL = testAndTrainResult.NullLL - trainingResults.NullLL; double newAltLL = testAndTrainResult.AltLL - trainingResults.AltLL; Debug.Assert(ComplexNumber.ApproxEqual(newNullLL, testGivenTrainResult.NullLL, eps)); Debug.Assert(ComplexNumber.ApproxEqual(newAltLL, testGivenTrainResult.AltLL, eps)); testNullLLSum += newNullLL; testAltLLSum += newAltLL; Debug.Assert(ComplexNumber.ApproxEqual(testNullLLSum, combinedResults.NullLL, eps), "Combined result has wrong NullLL"); Debug.Assert(ComplexNumber.ApproxEqual(testAltLLSum, combinedResults.AltLL, eps), "Combined result has wrong AltLL"); #endif } return(combinedResults); }