public static ExperimentResult <MulticlassClassificationMetrics> RunAutoMLExperiment(
MLContext mlContext, string labelColumnName, MulticlassExperimentSettings experimentSettings,
MulticlassExperimentProgressHandler progressHandler, IDataView dataView)
{
ConsoleHelper.ConsoleWriteHeader("=============== Running AutoML experiment ===============");
Trace.WriteLine($"Running AutoML multiclass classification experiment for {experimentSettings.MaxExperimentTimeInSeconds} seconds...");
var experimentResult = mlContext.Auto()
.CreateMulticlassClassificationExperiment(experimentSettings)
.Execute(dataView, labelColumnName, progressHandler: progressHandler);
Trace.WriteLine(Environment.NewLine);
Trace.WriteLine($"num models created: {experimentResult.RunDetails.Count()}");
// Get top few runs ranked by accuracy
var topRuns = experimentResult.RunDetails
.Where(r => r.ValidationMetrics != null && !double.IsNaN(r.ValidationMetrics.MicroAccuracy))
.OrderByDescending(r => r.ValidationMetrics.MicroAccuracy).Take(3);
Trace.WriteLine("Top models ranked by accuracy --");
CreateRow($"{"",-4} {"Trainer",-35} {"MicroAccuracy",14} {"MacroAccuracy",14} {"Duration",9}", Width);
for (var i = 0; i < topRuns.Count(); i++)
{
var run = topRuns.ElementAt(i);
CreateRow($"{i,-4} {run.TrainerName,-35} {run.ValidationMetrics?.MicroAccuracy ?? double.NaN,14:F4} {run.ValidationMetrics?.MacroAccuracy ?? double.NaN,14:F4} {run.RuntimeInSeconds,9:F1}", Width);
}
return(experimentResult);
}
public static ExperimentResult <MulticlassClassificationMetrics> Train(
MLContext mlContext, string labelColumnName, MulticlassExperimentSettings experimentSettings,
MulticlassExperimentProgressHandler progressHandler, DataFilePaths paths, TextLoader textLoader)
{
var trainData = textLoader.Load(paths.TrainPath);
var validateData = textLoader.Load(paths.ValidatePath);
var experimentResult = RunAutoMLExperiment(mlContext, labelColumnName, experimentSettings, progressHandler, trainData);
EvaluateTrainedModelAndPrintMetrics(mlContext, experimentResult.BestRun.Model, experimentResult.BestRun.TrainerName, validateData);
SaveModel(mlContext, experimentResult.BestRun.Model, paths.ModelPath, trainData);
return(experimentResult);
}
