/// <summary> /// Train experiment using <paramref name="train"/> and validate it on <paramref name="validate"/>. /// </summary> /// <param name="train">train dataset.</param> /// <param name="validate">validate dataset. this dataset is only used for hypeparameter optimization.</param> /// <param name="reporter">train-progress reporter. default is null.</param> /// <param name="ct">cancelation tokem, default is <see cref="CancellationToken.None"/>.</param> /// <returns>experiment result.</returns> public async Task <ExperimentResult> TrainAsync(IDataView train, IDataView validate, IProgress <IterationInfo> reporter = null, CancellationToken ct = default) { var experimentResult = new ExperimentResult(); var trainingServiceOption = new SinglePipelineTrainingService.Option() { ParameterSweeper = this.option.ParameterSweeper, EvaluationMetric = this.option.EvaluateFunction, Metrics = this.option.Metrics, IsMaximizng = this.option.IsMaximizing, ParameterSweepingIteration = this.option.ParameterSweeperIteration, }; foreach (var pipelineConfig in this.option.PipelineSweeper.ProposeSweeps(this.sweepablePipeline, this.option.PipelineSweeperIteration)) { var singleSweepablePipeline = this.sweepablePipeline.BuildFromParameters(pipelineConfig); var trainingService = new SinglePipelineTrainingService(this.context, singleSweepablePipeline, trainingServiceOption); var bestIteration = await trainingService.StartTrainingAsync(train, validate, ct, reporter); this.timeLeft -= bestIteration.TrainingTime; var bestModel = bestIteration.Model; if (this.timeLeft < 0) { this.logger.Trace(MessageSensitivity.All, "experiment time out"); break; } experimentResult.AddRunHistory(bestIteration, bestModel); // TODO // Add Run history to this.option.PipelineSweeper. } return(experimentResult); }
public async Task <IterationInfo> StartTrainingAsync(IDataView train, IDataView validate, CancellationToken ct = default, IProgress <IterationInfo> reporter = null) { // TODO check schema var option = new SinglePipelineTrainingService.Option() { IsMaximizng = this.option.IsMaximizng, ParameterSweepingIteration = this.option.ParameterSweepingIteration, ParameterSweeper = this.context.AutoML().Serializable().Factory.CreateSweeper(this.option.ParameterSweeper), EvaluationMetric = this.CreateEvaluateFunction(this.option.EvaluationMetric), Metrics = this.option.Metrics.Select(metric => this.CreateEvaluateFunction(metric)).ToArray(), }; var trainingService = new SinglePipelineTrainingService(this.context, this.Pipeline, option); return(await trainingService.StartTrainingAsync(train, validate, ct, reporter)); }