private RegressionForestModel FitModel(IEnumerable <OptimizerResult> validParameterSets)
{
var observations = validParameterSets.Select(v => v.ParameterSet).ToList().ToF64Matrix();
var targets = validParameterSets.Select(v => v.Error).ToArray();
return(m_learner.Learn(observations, targets));
}
double RegressionExtremelyRandomizedTreesLearner_Learn_Aptitude(int trees, double subSampleRatio = 1.0)
{
var(observations, targets) = DataSetUtilities.LoadAptitudeDataSet();
var sut = new RegressionExtremelyRandomizedTreesLearner(trees, 1, 100, 1, 0.0001, subSampleRatio, 42, false);
var model = sut.Learn(observations, targets);
var predictions = model.Predict(observations);
var evaluator = new MeanSquaredErrorRegressionMetric();
var error = evaluator.Error(targets, predictions);
return(error);
}
public void RegressionExtremelyRandomizedTreesLearnerTest_Learn_Glass_100_Trees_Parallel()
{
var(observations, targets) = DataSetUtilities.LoadGlassDataSet();
var sut = new RegressionExtremelyRandomizedTreesLearner(100, 1, 100, 1, 0.0001, 1.0, 42, true);
var model = sut.Learn(observations, targets);
var predictions = model.Predict(observations);
var evaluator = new MeanSquaredErrorRegressionMetric();
var error = evaluator.Error(targets, predictions);
Assert.AreEqual(0.33450356466453129, error, m_delta);
}
double RegressionExtremelyRandomizedTreesLearner_Learn_Aptitude(int trees, double subSampleRatio = 1.0)
{
var parser = new CsvParser(() => new StringReader(Resources.AptitudeData));
var observations = parser.EnumerateRows(v => v != "Pass").ToF64Matrix();
var targets = parser.EnumerateRows("Pass").ToF64Vector();
var rows = targets.Length;
var sut = new RegressionExtremelyRandomizedTreesLearner(trees, 1, 100, 1, 0.0001, subSampleRatio, 42, false);
var model = sut.Learn(observations, targets);
var predictions = model.Predict(observations);
var evaluator = new MeanSquaredErrorRegressionMetric();
var error = evaluator.Error(targets, predictions);
return(error);
}
public void RegressionExtremelyRandomizedTreesLearnerTest_Learn_Glass_100_Trees_Parallel()
{
var parser = new CsvParser(() => new StringReader(Resources.Glass));
var observations = parser.EnumerateRows(v => v != "Target").ToF64Matrix();
var targets = parser.EnumerateRows("Target").ToF64Vector();
var sut = new RegressionExtremelyRandomizedTreesLearner(100, 1, 100, 1, 0.0001, 1.0, 42, true);
var model = sut.Learn(observations, targets);
var predictions = model.Predict(observations);
var evaluator = new MeanSquaredErrorRegressionMetric();
var error = evaluator.Error(targets, predictions);
Assert.AreEqual(0.33450356466453129, error, m_delta);
}
public void RegressionExtremelyRandomizedTreesLearnerTest_Learn_Glass_100_Indices()
{
var(observations, targets) = DataSetUtilities.LoadGlassDataSet();
var sut = new RegressionExtremelyRandomizedTreesLearner(100, 1, 100, 1, 0.0001, 1.0, 42, false);
var indices = Enumerable.Range(0, targets.Length).ToArray();
indices.Shuffle(new Random(42));
indices = indices.Take((int)(targets.Length * 0.7))
.ToArray();
var model = sut.Learn(observations, targets, indices);
var predictions = model.Predict(observations);
var evaluator = new MeanSquaredErrorRegressionMetric();
var error = evaluator.Error(targets, predictions);
Assert.AreEqual(0.592240619161302, error, m_delta);
}
public void RegressionExtremelyRandomizedTreesLearnerTest_Learn_Glass_100_Indices()
{
var parser = new CsvParser(() => new StringReader(Resources.Glass));
var observations = parser.EnumerateRows(v => v != "Target").ToF64Matrix();
var targets = parser.EnumerateRows("Target").ToF64Vector();
var rows = targets.Length;
var sut = new RegressionExtremelyRandomizedTreesLearner(100, 1, 100, 1, 0.0001, 1.0, 42, false);
var indices = Enumerable.Range(0, targets.Length).ToArray();
indices.Shuffle(new Random(42));
indices = indices.Take((int)(targets.Length * 0.7))
.ToArray();
var model = sut.Learn(observations, targets, indices);
var predictions = model.Predict(observations);
var evaluator = new MeanSquaredErrorRegressionMetric();
var error = evaluator.Error(targets, predictions);
Assert.AreEqual(0.622380056587391, error, 0.0000001);
}
/// <summary>
/// Optimization using Sequential Model-based optimization.
/// Returns the final results ordered from best to worst (minimized).
/// </summary>
/// <param name="functionToMinimize"></param>
/// <returns></returns>
public OptimizerResult[] Optimize(Func <double[], OptimizerResult> functionToMinimize)
{
var bestParameterSet = new double[m_parameters.Length];
var bestParameterSetScore = double.MaxValue;
var parameterSets = new List <double[]>();
var parameterSetScores = new List <double>();
var usePreviousResults = m_previousParameterSetScores != null && m_previousParameterSets != null;
int iterations = 0;
if (usePreviousResults)
{
parameterSets.AddRange(m_previousParameterSets);
parameterSetScores.AddRange(m_previousParameterSetScores);
for (int i = 0; i < parameterSets.Count; i++)
{
var score = parameterSetScores[i];
if (!double.IsNaN(score))
{
if (score < bestParameterSetScore)
{
bestParameterSetScore = score;
bestParameterSet = parameterSets[i];
}
}
}
}
else
{
// initialize random starting points for the first iteration
for (int i = 0; i < m_numberOfStartingPoints; i++)
{
var set = CreateParameterSet();
var score = functionToMinimize(set).Error;
iterations++;
if (!double.IsNaN(score))
{
parameterSets.Add(set);
parameterSetScores.Add(score);
if (score < bestParameterSetScore)
{
bestParameterSetScore = score;
bestParameterSet = set;
}
}
}
}
var lastSet = new double[m_parameters.Length];
for (int iteration = 0; iteration < m_maxIterations; iteration++)
{
// fit model
var observations = parameterSets.ToF64Matrix();
var targets = parameterSetScores.ToArray();
var model = m_learner.Learn(observations, targets);
var bestScore = parameterSetScores.Min();
var candidates = FindNextCandidates(model, bestScore);
var first = true;
foreach (var candidate in candidates)
{
var parameterSet = candidate.ParameterSet;
if (Equals(lastSet, parameterSet) && !first)
{
// skip evaluation if parameters have not changed.
continue;
}
if (Equals(bestParameterSet, parameterSet))
{
// if the beset parameter set is sampled again.
// Add a new random parameter set.
parameterSet = CreateParameterSet();
}
var result = functionToMinimize(parameterSet);
iterations++;
if (!double.IsNaN(result.Error))
{
// update best
if (result.Error < bestParameterSetScore)
{
bestParameterSetScore = result.Error;
bestParameterSet = result.ParameterSet;
//System.Diagnostics.Trace.WriteLine(iterations + ";" + result.Error);
}
// add point to parameter set list for next iterations model
parameterSets.Add(result.ParameterSet);
parameterSetScores.Add(result.Error);
}
lastSet = parameterSet;
first = false;
}
}
var results = new List <OptimizerResult>();
for (int i = 0; i < parameterSets.Count; i++)
{
results.Add(new OptimizerResult(parameterSets[i], parameterSetScores[i]));
}
return(results.Where(v => !double.IsNaN(v.Error)).OrderBy(r => r.Error).ToArray());
}
/// <summary>
/// Optimization using Sequential Model-based optimization.
/// Returns all results, chronologically ordered.
/// </summary>
/// <param name="functionToMinimize"></param>
/// <returns></returns>
public OptimizerResult[] Optimize(Func <double[], OptimizerResult> functionToMinimize)
{
var parameterSets = new BlockingCollection <(double[] Parameters, double Error)>();
var usePreviousResults = m_previousParameterSetScores != null && m_previousParameterSets != null;
int iterations = 0;
if (usePreviousResults)
{
for (int i = 0; i < m_previousParameterSets.Count; i++)
{
var score = m_previousParameterSetScores[i];
if (!double.IsNaN(score))
{
parameterSets.Add((m_previousParameterSets[i], score));
}
}
}
else
{
// initialize random starting points for the first iteration
Parallel.For(0, m_randomStartingPointCount, m_parallelOptions, i =>
{
var set = RandomSearchOptimizer.SampleParameterSet(m_parameters, m_sampler);
var score = functionToMinimize(set).Error;
iterations++;
if (!double.IsNaN(score))
{
parameterSets.Add((set, score));
}
});
}
for (int iteration = 0; iteration < m_iterations; iteration++)
{
// fit model
var observations = parameterSets.Select(s => s.Parameters).ToList().ToF64Matrix();
var targets = parameterSets.Select(s => s.Error).ToArray();
var model = m_learner.Learn(observations, targets);
var bestScore = parameterSets.Min(m => m.Error);
var candidates = FindNextCandidates(model, bestScore);
m_isFirst = true;
Parallel.ForEach(candidates, m_parallelOptions, candidate =>
{
var parameterSet = candidate.ParameterSet;
// skip evaluation if parameters have not changed unless explicitly allowed
if (m_allowMultipleEvaluations || IsFirstEvaluation() || !Contains(parameterSets, parameterSet))
{
if (!m_allowMultipleEvaluations && Equals(GetBestParameterSet(parameterSets), parameterSet))
{
// if the best parameter set is sampled again.
// Add a new random parameter set.
parameterSet = RandomSearchOptimizer
.SampleParameterSet(m_parameters, m_sampler);
}
var result = functionToMinimize(parameterSet);
iterations++;
if (!double.IsNaN(result.Error))
{
// add point to parameter set list for next iterations model
parameterSets.Add((parameterSet, result.Error));
}
}
