public void Update(TrialResult result) { var trialId = result.TrialSettings.TrialId; var metric = result.Metric; metric = _minimize ? metric : -metric; if (metric < _bestMetric) { BestConfig = _searchSpace.SampleFromFeatureSpace(_configs[trialId]); _bestMetric = metric; } var cost = result.DurationInMilliseconds; int threadId = _trialProposedBy[trialId]; if (_searchThreadPool.Count == 0) { var initParameter = _searchSpace.SampleFromFeatureSpace(_configs[trialId]); _searchThreadPool[_currentThreadId] = _localSearch.CreateSearchThread(initParameter, metric, cost); _initUsed = true; UpdateAdmissibleRegion(_configs[trialId]); } else { _searchThreadPool[threadId].OnTrialComplete(trialId, metric, cost); if (_searchThreadPool[threadId].IsConverged) { _searchThreadPool.Remove(threadId); _currentThreadId += 1; _initUsed = false; } } }
public void Update(TrialSettings settings, TrialResult result) { var schema = settings.Schema; if (_tuners.TryGetValue(schema, out var tuner)) { tuner.Update(result); } }
public void ReportBestTrial(TrialResult result) { if (result is BinaryClassificationTrialResult binaryClassificationResult) { BestRun = binaryClassificationResult; } else { throw new ArgumentException($"result must be of type {typeof(BinaryClassificationTrialResult)}"); } }
public void ReportCompletedTrial(TrialResult result) { if (result is BinaryClassificationTrialResult binaryClassificationResult) { RunDetails.Add(binaryClassificationResult); OnTrialCompleted?.Invoke(this, binaryClassificationResult); } else { throw new ArgumentException($"result must be of type {typeof(BinaryClassificationTrialResult)}"); } }
public void ReportCompletedTrial(TrialResult result) { MostRecentTrial = result; CompletedTrials.Add(result); var activeRunParam = JsonSerializer.Serialize(result.TrialSettings.Parameter, new JsonSerializerOptions() { WriteIndented = false, }); TrialData.Append(new List <KeyValuePair <string, object> >() { new KeyValuePair <string, object>("Trial", result.TrialSettings.TrialId), new KeyValuePair <string, object>("Metric", result.Metric), new KeyValuePair <string, object>("Trainer", result.TrialSettings.Pipeline.ToString().Replace("Unknown=>", "")), new KeyValuePair <string, object>("Parameters", activeRunParam), }, true); ThrottledUpdate(); }
/// <summary> /// Run experiment and return the best trial result asynchronizely. The experiment returns the current best trial result if there's any trial completed when <paramref name="ct"/> get cancelled, /// and throws <see cref="TimeoutException"/> with message "Training time finished without completing a trial run" when no trial has completed. /// Another thing needs to notice is that this function won't immediately return after <paramref name="ct"/> get cancelled. Instead, it will call <see cref="MLContext.CancelExecution"/> to cancel all training process /// and wait all running trials get cancelled or completed. /// </summary> /// <returns></returns> public async Task <TrialResult> RunAsync(CancellationToken ct = default) { ValidateSettings(); var cts = new CancellationTokenSource(); _settings.CancellationToken = ct; cts.CancelAfter((int)_settings.MaxExperimentTimeInSeconds * 1000); _settings.CancellationToken.Register(() => cts.Cancel()); cts.Token.Register(() => { // only force-canceling running trials when there's completed trials. // otherwise, wait for the current running trial to be completed. if (_bestTrialResult != null) { _context.CancelExecution(); } }); InitializeServiceCollection(); var serviceProvider = _serviceCollection.BuildServiceProvider(); var monitor = serviceProvider.GetService <IMonitor>(); var trialNum = 0; var pipelineProposer = serviceProvider.GetService <PipelineProposer>(); var hyperParameterProposer = serviceProvider.GetService <HyperParameterProposer>(); var runnerFactory = serviceProvider.GetService <ITrialRunnerFactory>(); while (true) { if (cts.Token.IsCancellationRequested) { break; } var setting = new TrialSettings() { ExperimentSettings = _settings, TrialId = trialNum++, }; setting = pipelineProposer.Propose(setting); setting = hyperParameterProposer.Propose(setting); monitor.ReportRunningTrial(setting); var runner = runnerFactory.CreateTrialRunner(); try { var trialResult = runner.Run(setting, serviceProvider); monitor.ReportCompletedTrial(trialResult); hyperParameterProposer.Update(setting, trialResult); pipelineProposer.Update(setting, trialResult); var error = _settings.IsMaximizeMetric ? 1 - trialResult.Metric : trialResult.Metric; if (error < _bestError) { _bestTrialResult = trialResult; _bestError = error; monitor.ReportBestTrial(trialResult); } } catch (Exception ex) { if (cts.Token.IsCancellationRequested) { break; } else { // TODO // it's questionable on whether to abort the entire training process // for a single fail trial. We should make it an option and only exit // when error is fatal (like schema mismatch). monitor.ReportFailTrial(setting, ex); throw; } } } if (_bestTrialResult == null) { throw new TimeoutException("Training time finished without completing a trial run"); } else { return(await Task.FromResult(_bestTrialResult)); } }
public void Update(TrialResult result) { return; }
public void ReportCompletedTrial(TrialResult result) { _logger.Info($"Update Completed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {result.TrialSettings.Pipeline} - Duration: {result.DurationInMilliseconds}"); _completedTrials.Add(result); }
public void ReportBestTrial(TrialResult result) { _logger.Info($"Update Best Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {result.TrialSettings.Pipeline}"); }
public void Update(TrialSettings parameter, TrialResult result) { var schema = parameter.Schema; var error = CaculateError(result.Metric, parameter.ExperimentSettings.IsMaximizeMetric); var duration = result.DurationInMilliseconds / 1000; var pipelineIds = _multiModelPipeline.PipelineIds; var isSuccess = duration != 0; // if k1 is null, it means this is the first completed trial. // in that case, initialize k1, k2, e1, e2 in the following way: // k1: for every learner, k1[l] = c * duration where c is a ratio defined in learnerInitialCost // k2: k2 = k1, which indicates the hypothesis that it costs the same time for learners to reach the next break through. // e1: current error // e2: 1.001*e1 if (isSuccess) { if (_k1 == null) { _k1 = pipelineIds.ToDictionary(id => id, id => duration * _learnerInitialCost[id] / _learnerInitialCost[schema]); _k2 = _k1.ToDictionary(kv => kv.Key, kv => kv.Value); _e1 = pipelineIds.ToDictionary(id => id, id => error); _e2 = pipelineIds.ToDictionary(id => id, id => 1.05 * error); _globalBestError = error; } else if (error >= _e1[schema]) { // if error is larger than current best error, which means there's no improvements for // the last trial with the current learner. // In that case, simply increase the total spent time since the last best error for that learner. _k1[schema] += duration; } else { // there's an improvement. // k2 <= k1 && e2 <= e1, and update k1, e2. _k2[schema] = _k1[schema]; _k1[schema] = duration; _e2[schema] = _e1[schema]; _e1[schema] = error; // update global best error as well if (error < _globalBestError) { _globalBestError = error; } } // update eci var eci1 = Math.Max(_k1[schema], _k2[schema]); var estimatorCostForBreakThrough = 2 * (error - _globalBestError) / ((_e2[schema] - _e1[schema]) / (_k2[schema] + _k1[schema])); _eci[schema] = Math.Max(eci1, estimatorCostForBreakThrough); } else { // double eci of current trial twice of maxium ecis. _eci[schema] = _eci.Select(kv => kv.Value).Max() * 2; } // normalize eci var sum = _eci.Select(x => x.Value).Sum(); _eci = _eci.Select(x => (x.Key, x.Value / sum)).ToDictionary(x => x.Key, x => x.Item2); // TODO // save k1,k2,e1,e2,eci,bestError to training configuration return; }
public void ReportBestTrial(TrialResult result) { BestTrial = result; ThrottledUpdate(); }