/// <summary>
/// Train experiment using <paramref name="train"/> and validate it on <paramref name="validate"/>.
/// </summary>
/// <param name="train">train dataset.</param>
/// <param name="validate">validate dataset. this dataset is only used for hypeparameter optimization.</param>
/// <param name="reporter">train-progress reporter. default is null.</param>
/// <param name="ct">cancelation tokem, default is <see cref="CancellationToken.None"/>.</param>
/// <returns>experiment result.</returns>
public async Task <ExperimentResult> TrainAsync(IDataView train, IDataView validate, IProgress <IterationInfo> reporter = null, CancellationToken ct = default)
{
var experimentResult = new ExperimentResult();
var trainingServiceOption = new SinglePipelineTrainingService.Option()
{
ParameterSweeper = this.option.ParameterSweeper,
EvaluationMetric = this.option.EvaluateFunction,
Metrics = this.option.Metrics,
IsMaximizng = this.option.IsMaximizing,
ParameterSweepingIteration = this.option.ParameterSweeperIteration,
};
foreach (var pipelineConfig in this.option.PipelineSweeper.ProposeSweeps(this.sweepablePipeline, this.option.PipelineSweeperIteration))
{
var singleSweepablePipeline = this.sweepablePipeline.BuildFromParameters(pipelineConfig);
var trainingService = new SinglePipelineTrainingService(this.context, singleSweepablePipeline, trainingServiceOption);
var bestIteration = await trainingService.StartTrainingAsync(train, validate, ct, reporter);
this.timeLeft -= bestIteration.TrainingTime;
var bestModel = bestIteration.Model;
if (this.timeLeft < 0)
{
this.logger.Trace(MessageSensitivity.All, "experiment time out");
break;
}
experimentResult.AddRunHistory(bestIteration, bestModel);
// TODO
// Add Run history to this.option.PipelineSweeper.
}
return(experimentResult);
}
public async Task <IterationInfo> StartTrainingAsync(IDataView train, IDataView validate, CancellationToken ct = default, IProgress <IterationInfo> reporter = null)
{
// TODO check schema
var option = new SinglePipelineTrainingService.Option()
{
IsMaximizng = this.option.IsMaximizng,
ParameterSweepingIteration = this.option.ParameterSweepingIteration,
ParameterSweeper = this.context.AutoML().Serializable().Factory.CreateSweeper(this.option.ParameterSweeper),
EvaluationMetric = this.CreateEvaluateFunction(this.option.EvaluationMetric),
Metrics = this.option.Metrics.Select(metric => this.CreateEvaluateFunction(metric)).ToArray(),
};
var trainingService = new SinglePipelineTrainingService(this.context, this.Pipeline, option);
return(await trainingService.StartTrainingAsync(train, validate, ct, reporter));
}
