public void LPEarlyStoppingCriterionTest()
{
IEarlyStoppingCriterion cr = CreateEarlyStoppingCriterion("lp", "th=0.01 w=5", false);
bool isBestCandidate;
bool shouldStop;
for (int i = 0; i < 100; i++)
{
float score = 0.001f * i;
shouldStop = cr.CheckScore(score, score, out isBestCandidate);
Assert.True(isBestCandidate);
Assert.False(shouldStop);
}
for (int i = 1; i <= 10; i++)
{
shouldStop = cr.CheckScore(i, i, out isBestCandidate);
Assert.True(isBestCandidate);
Assert.False(shouldStop);
}
// At this point, average of score should be 8 and the best score should be 10.
for (int i = 0; i < 3; i++)
{
shouldStop = cr.CheckScore(0, 10f, out isBestCandidate);
Assert.False(isBestCandidate);
Assert.False(shouldStop);
}
shouldStop = cr.CheckScore(0, 10f, out isBestCandidate);
Assert.False(isBestCandidate);
Assert.True(shouldStop);
}
public void UPEarlyStoppingCriterionTest()
{
const int windowSize = 8;
IEarlyStoppingCriterion cr = CreateEarlyStoppingCriterion("up", "w=8", false);
bool isBestCandidate;
bool shouldStop;
for (int i = 0; i < 100; i++)
{
float score = 0.001f * i;
shouldStop = cr.CheckScore(score, 0, out isBestCandidate);
Assert.True(isBestCandidate);
Assert.False(shouldStop);
}
for (int i = 0; i < windowSize - 1; i++)
{
float score = 0.09f - 0.001f * i;
shouldStop = cr.CheckScore(score, 0, out isBestCandidate);
Assert.False(isBestCandidate);
Assert.False(shouldStop);
}
shouldStop = cr.CheckScore(0.0f, 0, out isBestCandidate);
Assert.True(shouldStop);
Assert.False(isBestCandidate);
}
public void GLEarlyStoppingCriterionTest()
{
IEarlyStoppingCriterion cr = CreateEarlyStoppingCriterion("gl", "th=0.01", false);
bool isBestCandidate;
bool shouldStop;
for (int i = 0; i < 100; i++)
{
float score = 0.001f * i;
shouldStop = cr.CheckScore(score, 0, out isBestCandidate);
Assert.True(isBestCandidate);
Assert.False(shouldStop);
}
shouldStop = cr.CheckScore(1.0f, 0, out isBestCandidate);
Assert.True(isBestCandidate);
Assert.False(shouldStop);
shouldStop = cr.CheckScore(0.98f, 0, out isBestCandidate);
Assert.False(isBestCandidate);
Assert.True(shouldStop);
}
protected override bool ShouldStop(IChannel ch, ref IEarlyStoppingCriterion earlyStoppingRule, ref int bestIteration)
{
if (Args.EarlyStoppingRule == null)
{
return(false);
}
ch.AssertValue(ValidTest);
ch.AssertValue(TrainTest);
var validationResult = ValidTest.ComputeTests().First();
ch.Assert(validationResult.FinalValue >= 0);
bool lowerIsBetter = validationResult.LowerIsBetter;
var trainingResult = TrainTest.ComputeTests().First();
ch.Assert(trainingResult.FinalValue >= 0);
// Create early stopping rule.
if (earlyStoppingRule == null)
{
earlyStoppingRule = Args.EarlyStoppingRule.CreateComponent(Host, lowerIsBetter);
ch.Assert(earlyStoppingRule != null);
}
bool isBestCandidate;
bool shouldStop = earlyStoppingRule.CheckScore((Float)validationResult.FinalValue,
(Float)trainingResult.FinalValue, out isBestCandidate);
if (isBestCandidate)
{
bestIteration = Ensemble.NumTrees;
}
return(shouldStop);
}
