/// <summary>
/// This method checks whether we should abort learning due to perfect prediction or worsening prediction.
/// </summary>
/// <param name="current">The current model containing the fitted features.</param>
/// <returns>True if we should abort learning, false otherwise.</returns>
protected bool abortDueError(LearningRound current)
{
if (current.validationError == 0)
{
return(true);
}
double error = 0;
if (this.MLsettings.crossValidation)
{
error = (current.learningError + current.validationError) / 2;
}
else
{
error = current.validationError;
}
if (error < this.MLsettings.abortError)
{
return(true);
}
else
{
return(false);
}
}
/// <summary>
/// Parse string representation of a learning round to a LearningRound object.
/// </summary>
/// <param name="learningRoundAsString">LearningRound as string.</param>
/// <param name="vm">Variability model the LearningRound belongs to.</param>
/// <returns>LearningRound object that has the data of the string representation.</returns>
public static LearningRound FromString(string learningRoundAsString, VariabilityModel vm)
{
LearningRound learningRound = new LearningRound();
string[] data = learningRoundAsString.Split(new char[] { ';' });
learningRound.round = int.Parse(data[0].Trim());
List<Feature> featureSetFromString = new List<Feature>();
string[] featureExpressions = data[1].Split(new char[] { '+' }, StringSplitOptions.RemoveEmptyEntries);
foreach (string featureExpression in featureExpressions)
{
Feature toAdd = new Feature(featureExpression.Split(new char[] { '*' }, 2)[1], vm);
toAdd.Constant = double.Parse(featureExpression.Split(new char[] { '*' }, 2)[0].Trim(), System.Globalization.CultureInfo.GetCultureInfo("en-us"));
featureSetFromString.Add(toAdd);
}
learningRound.featureSet = featureSetFromString;
learningRound.learningError = double.Parse(data[2].Trim(), System.Globalization.CultureInfo.GetCultureInfo("en-us"));
learningRound.learningError_relative = double.Parse(data[3].Trim(), System.Globalization.CultureInfo.GetCultureInfo("en-us"));
learningRound.validationError = double.Parse(data[4].Trim(), System.Globalization.CultureInfo.GetCultureInfo("en-us"));
learningRound.validationError_relative = double.Parse(data[5].Trim(), System.Globalization.CultureInfo.GetCultureInfo("en-us"));
learningRound.elapsedTime = TimeSpan.FromSeconds(double.Parse(data[6].Trim(), System.Globalization.CultureInfo.GetCultureInfo("en-us")));
Feature bestCandidateFromString = new Feature(data[8], vm);
learningRound.bestCandidate = bestCandidateFromString;
learningRound.bestCandidateSize = int.Parse(data[9].Trim());
try
{
learningRound.bestCandidateScore = double.Parse(data[10].Trim(), System.Globalization.CultureInfo.GetCultureInfo("en-us"));
}
catch (OverflowException overF)
{
GlobalState.logError.logLine("Error in analysing of the learning round.");
GlobalState.logError.logLine(overF.Source + " -> " + overF.Message);
learningRound.bestCandidateScore = Double.MaxValue;
}
return learningRound;
}
/// <summary>
/// Performs learning using an adapted feature-selection algorithm.
/// Learning is done in rounds, in which each round adds an additional feature (i.e., configuration option or interaction) to the current model containing all influences.
/// Abort criteria is derived from ML_Settings class, such as number of rounds, minimum error, etc.
/// </summary>
public void learn()
{
if (!allInformationAvailable())
{
return;
}
this.startTime = System.DateTime.Now;
LearningRound current = new LearningRound();
if (this.strictlyMandatoryFeatures.Count > 0)
{
current.FeatureSet.AddRange(this.strictlyMandatoryFeatures);
}
double oldRoundError = Double.MaxValue;
do
{
oldRoundError = current.validationError;
current = performForwardStep(current);
if (current == null)
{
return;
}
learningHistory.Add(current);
if (this.MLsettings.useBackward)
{
current = performBackwardStep(current);
learningHistory.Add(current);
}
} while (!abortLearning(current, oldRoundError));
updateInfluenceModel();
}
/// <summary>
/// This methods checks whether the learning procedure should be aborted. For this decision, it uses parameters of ML settings, such as the number of rounds.
/// </summary>
/// <param name="current">The current state of learning (i.e., the current model).</param>
/// <returns>True if we abort learning, false otherwise</returns>
protected bool abortLearning(LearningRound current, double oldRoundError)
{
if (current.round >= this.MLsettings.numberOfRounds)
{
return(true);
}
TimeSpan diff = DateTime.Now - this.startTime;
if (current.round > 30 && diff.Minutes > 60)
{
return(true);
}
if (abortDueError(current))
{
return(true);
}
if (current.validationError + this.MLsettings.minImprovementPerRound > oldRoundError)
{
if (this.MLsettings.withHierarchy)
{
hierachyLevel++;
return(false);
}
else
{
return(true);
}
}
return(false);
}
private void UpdateDataGridView(MachineLearning.Learning.Regression.LearningRound lastRound)
{
string[] row = new string[cmd.exp.info.mlSettings.numberOfRounds * 2];
row[0] = lastRound.round.ToString();
row[1] = lastRound.learningError.ToString();
// TODO useEpsilonTube from Ml Settings
double relativeError = cmd.exp.models[0].computeError(lastRound.FeatureSet, GlobalState.allMeasurements.Configurations, false);
row[2] = relativeError.ToString();
lastRound.learningError.ToString();
foreach (Feature f in lastRound.FeatureSet)
{
string name = f.getPureString();
if (!termToIndex.ContainsKey(name))
{
perfInfGridView.Invoke((MethodInvoker)(() => perfInfGridView.Columns[termToIndex.Count + perfInfGrid_definedColumns].Name = name));
termToIndex.Add(name, termToIndex.Count + perfInfGrid_definedColumns);
}
row[termToIndex[name]] = Math.Round(f.Constant, 2).ToString();
}
perfInfGridView.Invoke((MethodInvoker)(() => this.perfInfGridView.Rows.Add(row)));
}
void roundFinished(object sender, NotifyCollectionChangedEventArgs e)
{
//e.NewItems will be an IList of all the items that were added in the AddRange method...
MachineLearning.Learning.Regression.LearningRound lastRound = (MachineLearning.Learning.Regression.LearningRound)e.NewItems[0];
UpdateDataGridView(lastRound);
}
/// <summary>
/// Based on the given learning round, the method intantiates the influence model.
/// </summary>
/// <param name="current">The current learning round containing all determined features with their influences.</param>
private void updateInfluenceModel()
{
this.infModel.BinaryOptionsInfluence.Clear();
this.infModel.NumericOptionsInfluence.Clear();
this.infModel.InteractionInfluence.Clear();
LearningRound best = null;
double lowestError = Double.MaxValue;
foreach (LearningRound r in this.learningHistory)
{
if (r.validationError < lowestError)
{
lowestError = r.validationError;
best = r;
}
}
foreach (Feature f in best.FeatureSet)
{
//single binary option influence
if (f.participatingBoolOptions.Count == 1 && f.participatingNumOptions.Count == 0 && f.getNumberOfParticipatingOptions() == 1)
{
this.infModel.BinaryOptionsInfluence.Add(f.participatingBoolOptions.ElementAt(0), f);
continue;
}
//single numeric option influence
if (f.participatingBoolOptions.Count == 0 && f.participatingNumOptions.Count == 1 && f.getNumberOfParticipatingOptions() == 1)
{
if (this.infModel.NumericOptionsInfluence.Keys.Contains(f.participatingNumOptions.ElementAt(0)))
{
InfluenceFunction composed = new InfluenceFunction(this.infModel.NumericOptionsInfluence[f.participatingNumOptions.ElementAt(0)].ToString() + " + " + f.ToString(), f.participatingNumOptions.ElementAt(0));
this.infModel.NumericOptionsInfluence[f.participatingNumOptions.ElementAt(0)] = composed;
}
else
{
this.infModel.NumericOptionsInfluence.Add(f.participatingNumOptions.ElementAt(0), f);
}
continue;
}
//interaction influence
Interaction i = new Interaction(f);
this.infModel.InteractionInfluence.Add(i, f);
}
}
/// <summary>
/// The backward steps aims at removing already learned features from the model if they have only a small impact on the prediction accuracy.
/// This should help keeping the model simple, reducing the danger of overfitting, and leaving local optima.
/// </summary>
/// <param name="current">The model learned so far containing the features that might be removed. Strictly mandatory features will not be removed.</param>
/// <returns>A new model that might be smaller than the original one and might have a slightly worse prediction accuracy.</returns>
protected LearningRound performBackwardStep(LearningRound current)
{
if (current.round < 3 || current.FeatureSet.Count < 2)
{
return(current);
}
bool abort = false;
List <Feature> featureSet = copyCombination(current.FeatureSet);
while (!abort)
{
double roundError = Double.MaxValue;
Feature toRemove = null;
foreach (Feature toDelete in featureSet)
{
List <Feature> tempSet = copyCombination(featureSet);
tempSet.Remove(toDelete);
double relativeError = 0;
double error = computeModelError(tempSet, out relativeError);
if (error - this.MLsettings.backwardErrorDelta < current.validationError && error < roundError)
{
roundError = error;
toRemove = toDelete;
}
}
if (toRemove != null)
{
featureSet.Remove(toRemove);
}
if (featureSet.Count <= 2)
{
abort = true;
}
}
current.FeatureSet = featureSet;
return(current);
}
/// <summary>
/// Makes one further step in learning. That is, it adds a further feature to the current model.
/// </summary>
/// <param name="currentModel">This parameter holds a list of features determined as important influencing factors so far.</param>
/// <returns>Returns a new model (i.e. learning round) with an additional feature.</returns>
internal LearningRound performForwardStep(LearningRound currentModel)
{
//Error in this round (depends on crossvalidation)
double minimalRoundError = Double.MaxValue;
List <Feature> minimalErrorModel = null;
//Go through each feature of the initial set and combine them with the already present features to build new candidates
List <Feature> candidates = new List <Feature>();
foreach (Feature basicFeature in this.initialFeatures)
{
candidates.AddRange(generateCandidates(currentModel.FeatureSet, basicFeature));
}
//If we got no candidates and we perform hierachical learning, we go one step further
if (candidates.Count == 0 && this.MLsettings.withHierarchy)
{
if (this.hierachyLevel > 10)
{
return(null);
}
this.hierachyLevel++;
return(performForwardStep(currentModel));
}
Dictionary <Feature, double> errorOfFeature = new Dictionary <Feature, double>();
Feature bestCandidate = null;
//Learn for each candidate a new model and compute the error for each newly learned model
foreach (Feature candidate in candidates)
{
if (this.badFeatures.Keys.Contains(candidate) && this.badFeatures[candidate] > 0)
{
this.badFeatures[candidate]--;
continue;
}
if (errorOfFeature.Keys.Contains(candidate))
{
continue;
}
List <Feature> newModel = copyCombination(currentModel.FeatureSet);
newModel.Add(candidate);
if (!fitModel(newModel))
{
continue;
}
double temp = 0;
double errorOfModel = computeModelError(newModel, out temp);
errorOfFeature.Add(candidate, temp);
if (errorOfModel < minimalRoundError)
{
minimalRoundError = errorOfModel;
minimalErrorModel = newModel;
bestCandidate = candidate;
}
}
double relativeErrorTrain = 0;
double relativeErrorEval = 0;
addFeaturesToIgnore(errorOfFeature);
LearningRound newRound = new LearningRound(minimalErrorModel, computeLearningError(minimalErrorModel, out relativeErrorTrain), computeValidationError(minimalErrorModel, out relativeErrorEval), currentModel.round + 1);
newRound.learningError_relative = relativeErrorTrain;
newRound.validationError_relative = relativeErrorEval;
return(newRound);
}