public void Update(TrialResult result)
{
var trialId = result.TrialSettings.TrialId;
var metric = result.Metric;
metric = _minimize ? metric : -metric;
if (metric < _bestMetric)
{
BestConfig = _searchSpace.SampleFromFeatureSpace(_configs[trialId]);
_bestMetric = metric;
}
var cost = result.DurationInMilliseconds;
int threadId = _trialProposedBy[trialId];
if (_searchThreadPool.Count == 0)
{
var initParameter = _searchSpace.SampleFromFeatureSpace(_configs[trialId]);
_searchThreadPool[_currentThreadId] = _localSearch.CreateSearchThread(initParameter, metric, cost);
_initUsed = true;
UpdateAdmissibleRegion(_configs[trialId]);
}
else
{
_searchThreadPool[threadId].OnTrialComplete(trialId, metric, cost);
if (_searchThreadPool[threadId].IsConverged)
{
_searchThreadPool.Remove(threadId);
_currentThreadId += 1;
_initUsed = false;
}
}
}
public void Update(TrialSettings settings, TrialResult result)
{
var schema = settings.Schema;
if (_tuners.TryGetValue(schema, out var tuner))
{
tuner.Update(result);
}
}
public void ReportBestTrial(TrialResult result)
{
if (result is BinaryClassificationTrialResult binaryClassificationResult)
{
BestRun = binaryClassificationResult;
}
else
{
throw new ArgumentException($"result must be of type {typeof(BinaryClassificationTrialResult)}");
}
}
public void ReportCompletedTrial(TrialResult result)
{
if (result is BinaryClassificationTrialResult binaryClassificationResult)
{
RunDetails.Add(binaryClassificationResult);
OnTrialCompleted?.Invoke(this, binaryClassificationResult);
}
else
{
throw new ArgumentException($"result must be of type {typeof(BinaryClassificationTrialResult)}");
}
}
public void ReportCompletedTrial(TrialResult result)
{
MostRecentTrial = result;
CompletedTrials.Add(result);
var activeRunParam = JsonSerializer.Serialize(result.TrialSettings.Parameter, new JsonSerializerOptions()
{
WriteIndented = false,
});
TrialData.Append(new List <KeyValuePair <string, object> >()
{
new KeyValuePair <string, object>("Trial", result.TrialSettings.TrialId),
new KeyValuePair <string, object>("Metric", result.Metric),
new KeyValuePair <string, object>("Trainer", result.TrialSettings.Pipeline.ToString().Replace("Unknown=>", "")),
new KeyValuePair <string, object>("Parameters", activeRunParam),
}, true);
ThrottledUpdate();
}
/// <summary>
/// Run experiment and return the best trial result asynchronizely. The experiment returns the current best trial result if there's any trial completed when <paramref name="ct"/> get cancelled,
/// and throws <see cref="TimeoutException"/> with message "Training time finished without completing a trial run" when no trial has completed.
/// Another thing needs to notice is that this function won't immediately return after <paramref name="ct"/> get cancelled. Instead, it will call <see cref="MLContext.CancelExecution"/> to cancel all training process
/// and wait all running trials get cancelled or completed.
/// </summary>
/// <returns></returns>
public async Task <TrialResult> RunAsync(CancellationToken ct = default)
{
ValidateSettings();
var cts = new CancellationTokenSource();
_settings.CancellationToken = ct;
cts.CancelAfter((int)_settings.MaxExperimentTimeInSeconds * 1000);
_settings.CancellationToken.Register(() => cts.Cancel());
cts.Token.Register(() =>
{
// only force-canceling running trials when there's completed trials.
// otherwise, wait for the current running trial to be completed.
if (_bestTrialResult != null)
{
_context.CancelExecution();
}
});
InitializeServiceCollection();
var serviceProvider = _serviceCollection.BuildServiceProvider();
var monitor = serviceProvider.GetService <IMonitor>();
var trialNum = 0;
var pipelineProposer = serviceProvider.GetService <PipelineProposer>();
var hyperParameterProposer = serviceProvider.GetService <HyperParameterProposer>();
var runnerFactory = serviceProvider.GetService <ITrialRunnerFactory>();
while (true)
{
if (cts.Token.IsCancellationRequested)
{
break;
}
var setting = new TrialSettings()
{
ExperimentSettings = _settings,
TrialId = trialNum++,
};
setting = pipelineProposer.Propose(setting);
setting = hyperParameterProposer.Propose(setting);
monitor.ReportRunningTrial(setting);
var runner = runnerFactory.CreateTrialRunner();
try
{
var trialResult = runner.Run(setting, serviceProvider);
monitor.ReportCompletedTrial(trialResult);
hyperParameterProposer.Update(setting, trialResult);
pipelineProposer.Update(setting, trialResult);
var error = _settings.IsMaximizeMetric ? 1 - trialResult.Metric : trialResult.Metric;
if (error < _bestError)
{
_bestTrialResult = trialResult;
_bestError = error;
monitor.ReportBestTrial(trialResult);
}
}
catch (Exception ex)
{
if (cts.Token.IsCancellationRequested)
{
break;
}
else
{
// TODO
// it's questionable on whether to abort the entire training process
// for a single fail trial. We should make it an option and only exit
// when error is fatal (like schema mismatch).
monitor.ReportFailTrial(setting, ex);
throw;
}
}
}
if (_bestTrialResult == null)
{
throw new TimeoutException("Training time finished without completing a trial run");
}
else
{
return(await Task.FromResult(_bestTrialResult));
}
}
public void Update(TrialResult result)
{
return;
}
public void ReportCompletedTrial(TrialResult result)
{
_logger.Info($"Update Completed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {result.TrialSettings.Pipeline} - Duration: {result.DurationInMilliseconds}");
_completedTrials.Add(result);
}
public void ReportBestTrial(TrialResult result)
{
_logger.Info($"Update Best Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {result.TrialSettings.Pipeline}");
}
public void Update(TrialSettings parameter, TrialResult result)
{
var schema = parameter.Schema;
var error = CaculateError(result.Metric, parameter.ExperimentSettings.IsMaximizeMetric);
var duration = result.DurationInMilliseconds / 1000;
var pipelineIds = _multiModelPipeline.PipelineIds;
var isSuccess = duration != 0;
// if k1 is null, it means this is the first completed trial.
// in that case, initialize k1, k2, e1, e2 in the following way:
// k1: for every learner, k1[l] = c * duration where c is a ratio defined in learnerInitialCost
// k2: k2 = k1, which indicates the hypothesis that it costs the same time for learners to reach the next break through.
// e1: current error
// e2: 1.001*e1
if (isSuccess)
{
if (_k1 == null)
{
_k1 = pipelineIds.ToDictionary(id => id, id => duration * _learnerInitialCost[id] / _learnerInitialCost[schema]);
_k2 = _k1.ToDictionary(kv => kv.Key, kv => kv.Value);
_e1 = pipelineIds.ToDictionary(id => id, id => error);
_e2 = pipelineIds.ToDictionary(id => id, id => 1.05 * error);
_globalBestError = error;
}
else if (error >= _e1[schema])
{
// if error is larger than current best error, which means there's no improvements for
// the last trial with the current learner.
// In that case, simply increase the total spent time since the last best error for that learner.
_k1[schema] += duration;
}
else
{
// there's an improvement.
// k2 <= k1 && e2 <= e1, and update k1, e2.
_k2[schema] = _k1[schema];
_k1[schema] = duration;
_e2[schema] = _e1[schema];
_e1[schema] = error;
// update global best error as well
if (error < _globalBestError)
{
_globalBestError = error;
}
}
// update eci
var eci1 = Math.Max(_k1[schema], _k2[schema]);
var estimatorCostForBreakThrough = 2 * (error - _globalBestError) / ((_e2[schema] - _e1[schema]) / (_k2[schema] + _k1[schema]));
_eci[schema] = Math.Max(eci1, estimatorCostForBreakThrough);
}
else
{
// double eci of current trial twice of maxium ecis.
_eci[schema] = _eci.Select(kv => kv.Value).Max() * 2;
}
// normalize eci
var sum = _eci.Select(x => x.Value).Sum();
_eci = _eci.Select(x => (x.Key, x.Value / sum)).ToDictionary(x => x.Key, x => x.Item2);
// TODO
// save k1,k2,e1,e2,eci,bestError to training configuration
return;
}
public void ReportBestTrial(TrialResult result)
{
BestTrial = result;
ThrottledUpdate();
}