public Experiment(MLContext context, TaskKind task, OptimizingMetricInfo metricInfo, IProgress <TRunDetail> progressCallback, ExperimentSettings experimentSettings, IMetricsAgent <TMetrics> metricsAgent, IEnumerable <TrainerName> trainerAllowList, DatasetColumnInfo[] datasetColumnInfo, IRunner <TRunDetail> runner, IChannel logger) { _context = context; _history = new List <SuggestedPipelineRunDetail>(); _optimizingMetricInfo = metricInfo; _task = task; _progressCallback = progressCallback; _experimentSettings = experimentSettings; _metricsAgent = metricsAgent; _trainerAllowList = trainerAllowList; _modelDirectory = GetModelDirectory(_context.TempFilePath, _experimentSettings.CacheDirectoryName); _datasetColumnInfo = datasetColumnInfo; _runner = runner; _logger = logger; _experimentTimerExpired = false; }
protected BasicMetrics(IMetricsAgent metricsAgent, ITaskSchedulerFactory taskSchedulerFactory) { _metricsAgent = metricsAgent; _taskScheduler = taskSchedulerFactory.GetTaskScheduler(); _taskScheduler.ScheduleOnInterval(CollectMemoryAndThreadsUsage, 0, 1000); }
public static CrossValidationRunDetail <TMetrics> GetBestRun <TMetrics>(IEnumerable <CrossValidationRunDetail <TMetrics> > results, IMetricsAgent <TMetrics> metricsAgent, bool isMetricMaximizing) { results = results.Where(r => r.Results != null && r.Results.Any(x => x.ValidationMetrics != null)); if (!results.Any()) { return(null); } var scores = results.Select(r => r.Results.Average(x => metricsAgent.GetScore(x.ValidationMetrics))); var indexOfBestScore = GetIndexOfBestScore(scores, isMetricMaximizing); return(results.ElementAt(indexOfBestScore)); }
internal ExperimentBase(MLContext context, IMetricsAgent <TMetrics> metricsAgent, OptimizingMetricInfo optimizingMetricInfo, TExperimentSettings settings, TaskKind task, IEnumerable <TrainerName> trainerAllowList) { Context = context; MetricsAgent = metricsAgent; OptimizingMetricInfo = optimizingMetricInfo; Settings = settings; _logger = ((IChannelProvider)context).Start("AutoML"); _task = task; _trainerAllowList = trainerAllowList; }
public static CrossValidationRunDetail <TMetrics> GetBestRun <TMetrics>(IEnumerable <CrossValidationRunDetail <TMetrics> > results, IMetricsAgent <TMetrics> metricsAgent, bool isMetricMaximizing) { results = results.Where(r => r.Results != null && r.Results.Any(x => x.ValidationMetrics != null)); if (!results.Any()) { return(null); } var scores = results.Select(r => r.Results.Average(x => metricsAgent.GetScore(x.ValidationMetrics))); var indexOfBestScore = GetIndexOfBestScore(scores, isMetricMaximizing); // indexOfBestScore will be -1 if the optimization metric for all models is NaN. // In this case, return the first model. indexOfBestScore = indexOfBestScore != -1 ? indexOfBestScore : 0; return(results.ElementAt(indexOfBestScore)); }
public CrossValRunner(MLContext context, IDataView[] trainDatasets, IDataView[] validDatasets, IMetricsAgent <TMetrics> metricsAgent, IEstimator <ITransformer> preFeaturizer, ITransformer[] preprocessorTransforms, string labelColumn, IChannel logger) { _context = context; _trainDatasets = trainDatasets; _validDatasets = validDatasets; _metricsAgent = metricsAgent; _preFeaturizer = preFeaturizer; _preprocessorTransforms = preprocessorTransforms; _labelColumn = labelColumn; _logger = logger; _modelInputSchema = trainDatasets[0].Schema; }
public TrainValidateRunner(MLContext context, IDataView trainData, IDataView validData, string labelColumn, IMetricsAgent <TMetrics> metricsAgent, IEstimator <ITransformer> preFeaturizer, ITransformer preprocessorTransform, IChannel logger) { _context = context; _trainData = trainData; _validData = validData; _labelColumn = labelColumn; _metricsAgent = metricsAgent; _preFeaturizer = preFeaturizer; _preprocessorTransform = preprocessorTransform; _logger = logger; _modelInputSchema = trainData.Schema; }
public GameMetrics(IMetricsAgent metricsAgent, ITaskSchedulerFactory taskSchedulerFactory) : base(metricsAgent, taskSchedulerFactory) { }
private static bool IsPerfectModel <TMetrics>(IMetricsAgent <TMetrics> metricsAgent, TMetrics metrics) { var score = metricsAgent.GetScore(metrics); return(metricsAgent.IsModelPerfect(score)); }