/// <summary>
/// Helper method in order to create training before training. It also try to restore trained from
/// checkpoint file in order to continue with training
/// </summary>
/// <param name="network"></param>
/// <param name="lrParams"></param>
/// <param name="trParams"></param>
/// <param name="modelCheckPoint"></param>
/// <returns></returns>
public Trainer CreateTrainer(Function network, LearningParameters lrParams, TrainingParameters trParams, string modelCheckPoint, string historyPath)
{
try
{
//create trainer
var trainer = createTrainer(network, lrParams, trParams);
//set initial value for the evaluation value
m_PrevTrainingEval = StatMetrics.IsGoalToMinimize(trainer.EvaluationFunction()) ? double.MaxValue : double.MinValue;
m_PrevValidationEval = StatMetrics.IsGoalToMinimize(trainer.EvaluationFunction()) ? double.MaxValue : double.MinValue;
//in case modelCheckpoint is saved and user select re-training existing trainer
//first check if the checkpoint is available
if (trParams.ContinueTraining && !string.IsNullOrEmpty(modelCheckPoint) && File.Exists(modelCheckPoint))
{
//if the network model changed checkpoint state will throw exception
//in that case throw exception that re-training is not possible
try
{
trainer.RestoreFromCheckpoint(modelCheckPoint);
//load history of training in case continuation of training is requested
m_trainingHistory = MLFactory.LoadTrainingHistory(historyPath);
}
catch (Exception)
{
throw new Exception("The Trainer cannot be restored from the previous state probably because the network has changed." +
"\n Uncheck 'Continue Training' and train the model from scratch.");
throw;
}
}
else//delete checkpoint if exist in case no retraining is required,
//so the next checkpoint saving is free of previous checkpoints
{
//delete heckpoint
if (File.Exists(modelCheckPoint))
{
File.Delete(modelCheckPoint);
}
//delete history
if (File.Exists(historyPath))
{
File.Delete(historyPath);
}
}
return(trainer);
}
catch (Exception)
{
throw;
}
}
