/// <summary>
        /// This method checks whether we should abort learning due to perfect prediction or worsening prediction.
        /// </summary>
        /// <param name="current">The current model containing the fitted features.</param>
        /// <returns>True if we should abort learning, false otherwise.</returns>
        protected bool abortDueError(LearningRound current)
        {
            if (current.validationError == 0)
            {
                return(true);
            }

            double error = 0;

            if (this.MLsettings.crossValidation)
            {
                error = (current.learningError + current.validationError) / 2;
            }
            else
            {
                error = current.validationError;
            }


            if (error < this.MLsettings.abortError)
            {
                return(true);
            }
            else
            {
                return(false);
            }
        }
예제 #2
0
 /// <summary>
 /// Parse string representation of a learning round to a LearningRound object.
 /// </summary>
 /// <param name="learningRoundAsString">LearningRound as string.</param>
 /// <param name="vm">Variability model the LearningRound belongs to.</param>
 /// <returns>LearningRound object that has the data of the string representation.</returns>
 public static LearningRound FromString(string learningRoundAsString, VariabilityModel vm)
 {
     LearningRound learningRound = new LearningRound();
     string[] data = learningRoundAsString.Split(new char[] { ';' });
     learningRound.round = int.Parse(data[0].Trim());
     List<Feature> featureSetFromString = new List<Feature>();
     string[] featureExpressions = data[1].Split(new char[] { '+' }, StringSplitOptions.RemoveEmptyEntries);
     foreach (string featureExpression in featureExpressions)
     {
         Feature toAdd = new Feature(featureExpression.Split(new char[] { '*' }, 2)[1], vm);
         toAdd.Constant = double.Parse(featureExpression.Split(new char[] { '*' }, 2)[0].Trim(), System.Globalization.CultureInfo.GetCultureInfo("en-us"));
         featureSetFromString.Add(toAdd);
     }
     learningRound.featureSet = featureSetFromString;
     learningRound.learningError = double.Parse(data[2].Trim(), System.Globalization.CultureInfo.GetCultureInfo("en-us"));
     learningRound.learningError_relative = double.Parse(data[3].Trim(), System.Globalization.CultureInfo.GetCultureInfo("en-us"));
     learningRound.validationError = double.Parse(data[4].Trim(), System.Globalization.CultureInfo.GetCultureInfo("en-us"));
     learningRound.validationError_relative = double.Parse(data[5].Trim(), System.Globalization.CultureInfo.GetCultureInfo("en-us"));
     learningRound.elapsedTime = TimeSpan.FromSeconds(double.Parse(data[6].Trim(), System.Globalization.CultureInfo.GetCultureInfo("en-us")));
     Feature bestCandidateFromString = new Feature(data[8], vm);
     learningRound.bestCandidate = bestCandidateFromString;
     learningRound.bestCandidateSize = int.Parse(data[9].Trim());
     try
     {
         learningRound.bestCandidateScore = double.Parse(data[10].Trim(), System.Globalization.CultureInfo.GetCultureInfo("en-us"));
     }
     catch (OverflowException overF)
     {
         GlobalState.logError.logLine("Error in analysing of the learning round.");
         GlobalState.logError.logLine(overF.Source + " -> " + overF.Message);
         learningRound.bestCandidateScore = Double.MaxValue;
     }
     return learningRound;
 }
        /// <summary>
        /// Performs learning using an adapted feature-selection algorithm.
        /// Learning is done in rounds, in which each round adds an additional feature (i.e., configuration option or interaction) to the current model containing all influences.
        /// Abort criteria is derived from ML_Settings class, such as number of rounds, minimum error, etc.
        /// </summary>
        public void learn()
        {
            if (!allInformationAvailable())
            {
                return;
            }
            this.startTime = System.DateTime.Now;
            LearningRound current = new LearningRound();

            if (this.strictlyMandatoryFeatures.Count > 0)
            {
                current.FeatureSet.AddRange(this.strictlyMandatoryFeatures);
            }
            double oldRoundError = Double.MaxValue;

            do
            {
                oldRoundError = current.validationError;
                current       = performForwardStep(current);
                if (current == null)
                {
                    return;
                }
                learningHistory.Add(current);

                if (this.MLsettings.useBackward)
                {
                    current = performBackwardStep(current);
                    learningHistory.Add(current);
                }
            } while (!abortLearning(current, oldRoundError));
            updateInfluenceModel();
        }
        /// <summary>
        /// This methods checks whether the learning procedure should be aborted. For this decision, it uses parameters of ML settings, such as the number of rounds.
        /// </summary>
        /// <param name="current">The current state of learning (i.e., the current model).</param>
        /// <returns>True if we abort learning, false otherwise</returns>
        protected bool abortLearning(LearningRound current, double oldRoundError)
        {
            if (current.round >= this.MLsettings.numberOfRounds)
            {
                return(true);
            }
            TimeSpan diff = DateTime.Now - this.startTime;

            if (current.round > 30 && diff.Minutes > 60)
            {
                return(true);
            }
            if (abortDueError(current))
            {
                return(true);
            }
            if (current.validationError + this.MLsettings.minImprovementPerRound > oldRoundError)
            {
                if (this.MLsettings.withHierarchy)
                {
                    hierachyLevel++;
                    return(false);
                }
                else
                {
                    return(true);
                }
            }
            return(false);
        }
예제 #5
0
        private void UpdateDataGridView(MachineLearning.Learning.Regression.LearningRound lastRound)
        {
            string[] row = new string[cmd.exp.info.mlSettings.numberOfRounds * 2];
            row[0] = lastRound.round.ToString();
            row[1] = lastRound.learningError.ToString();
            // TODO useEpsilonTube from Ml Settings
            double relativeError = cmd.exp.models[0].computeError(lastRound.FeatureSet, GlobalState.allMeasurements.Configurations, false);

            row[2] = relativeError.ToString();


            lastRound.learningError.ToString();

            foreach (Feature f in lastRound.FeatureSet)
            {
                string name = f.getPureString();
                if (!termToIndex.ContainsKey(name))
                {
                    perfInfGridView.Invoke((MethodInvoker)(() => perfInfGridView.Columns[termToIndex.Count + perfInfGrid_definedColumns].Name = name));


                    termToIndex.Add(name, termToIndex.Count + perfInfGrid_definedColumns);
                }
                row[termToIndex[name]] = Math.Round(f.Constant, 2).ToString();
            }


            perfInfGridView.Invoke((MethodInvoker)(() => this.perfInfGridView.Rows.Add(row)));
        }
예제 #6
0
        void roundFinished(object sender, NotifyCollectionChangedEventArgs e)
        {
            //e.NewItems will be an IList of all the items that were added in the AddRange method...
            MachineLearning.Learning.Regression.LearningRound lastRound = (MachineLearning.Learning.Regression.LearningRound)e.NewItems[0];

            UpdateDataGridView(lastRound);
        }
        /// <summary>
        /// Based on the given learning round, the method intantiates the influence model.
        /// </summary>
        /// <param name="current">The current learning round containing all determined features with their influences.</param>
        private void updateInfluenceModel()
        {
            this.infModel.BinaryOptionsInfluence.Clear();
            this.infModel.NumericOptionsInfluence.Clear();
            this.infModel.InteractionInfluence.Clear();
            LearningRound best        = null;
            double        lowestError = Double.MaxValue;

            foreach (LearningRound r in this.learningHistory)
            {
                if (r.validationError < lowestError)
                {
                    lowestError = r.validationError;
                    best        = r;
                }
            }

            foreach (Feature f in best.FeatureSet)
            {
                //single binary option influence
                if (f.participatingBoolOptions.Count == 1 && f.participatingNumOptions.Count == 0 && f.getNumberOfParticipatingOptions() == 1)
                {
                    this.infModel.BinaryOptionsInfluence.Add(f.participatingBoolOptions.ElementAt(0), f);
                    continue;
                }
                //single numeric option influence
                if (f.participatingBoolOptions.Count == 0 && f.participatingNumOptions.Count == 1 && f.getNumberOfParticipatingOptions() == 1)
                {
                    if (this.infModel.NumericOptionsInfluence.Keys.Contains(f.participatingNumOptions.ElementAt(0)))
                    {
                        InfluenceFunction composed = new InfluenceFunction(this.infModel.NumericOptionsInfluence[f.participatingNumOptions.ElementAt(0)].ToString() + " + " + f.ToString(), f.participatingNumOptions.ElementAt(0));
                        this.infModel.NumericOptionsInfluence[f.participatingNumOptions.ElementAt(0)] = composed;
                    }
                    else
                    {
                        this.infModel.NumericOptionsInfluence.Add(f.participatingNumOptions.ElementAt(0), f);
                    }
                    continue;
                }
                //interaction influence
                Interaction i = new Interaction(f);
                this.infModel.InteractionInfluence.Add(i, f);
            }
        }
        /// <summary>
        /// The backward steps aims at removing already learned features from the model if they have only a small impact on the prediction accuracy.
        /// This should help keeping the model simple, reducing the danger of overfitting, and leaving local optima.
        /// </summary>
        /// <param name="current">The model learned so far containing the features that might be removed. Strictly mandatory features will not be removed.</param>
        /// <returns>A new model that might be smaller than the original one and might have a slightly worse prediction accuracy.</returns>
        protected LearningRound performBackwardStep(LearningRound current)
        {
            if (current.round < 3 || current.FeatureSet.Count < 2)
            {
                return(current);
            }
            bool           abort      = false;
            List <Feature> featureSet = copyCombination(current.FeatureSet);

            while (!abort)
            {
                double  roundError = Double.MaxValue;
                Feature toRemove   = null;
                foreach (Feature toDelete in featureSet)
                {
                    List <Feature> tempSet = copyCombination(featureSet);
                    tempSet.Remove(toDelete);
                    double relativeError = 0;
                    double error         = computeModelError(tempSet, out relativeError);
                    if (error - this.MLsettings.backwardErrorDelta < current.validationError && error < roundError)
                    {
                        roundError = error;
                        toRemove   = toDelete;
                    }
                }
                if (toRemove != null)
                {
                    featureSet.Remove(toRemove);
                }
                if (featureSet.Count <= 2)
                {
                    abort = true;
                }
            }
            current.FeatureSet = featureSet;
            return(current);
        }
        /// <summary>
        /// Makes one further step in learning. That is, it adds a further feature to the current model.
        /// </summary>
        /// <param name="currentModel">This parameter holds a list of features determined as important influencing factors so far.</param>
        /// <returns>Returns a new model (i.e. learning round) with an additional feature.</returns>
        internal LearningRound performForwardStep(LearningRound currentModel)
        {
            //Error in this round (depends on crossvalidation)
            double         minimalRoundError = Double.MaxValue;
            List <Feature> minimalErrorModel = null;

            //Go through each feature of the initial set and combine them with the already present features to build new candidates
            List <Feature> candidates = new List <Feature>();

            foreach (Feature basicFeature in this.initialFeatures)
            {
                candidates.AddRange(generateCandidates(currentModel.FeatureSet, basicFeature));
            }

            //If we got no candidates and we perform hierachical learning, we go one step further
            if (candidates.Count == 0 && this.MLsettings.withHierarchy)
            {
                if (this.hierachyLevel > 10)
                {
                    return(null);
                }
                this.hierachyLevel++;
                return(performForwardStep(currentModel));
            }

            Dictionary <Feature, double> errorOfFeature = new Dictionary <Feature, double>();
            Feature bestCandidate = null;

            //Learn for each candidate a new model and compute the error for each newly learned model
            foreach (Feature candidate in candidates)
            {
                if (this.badFeatures.Keys.Contains(candidate) && this.badFeatures[candidate] > 0)
                {
                    this.badFeatures[candidate]--;
                    continue;
                }
                if (errorOfFeature.Keys.Contains(candidate))
                {
                    continue;
                }
                List <Feature> newModel = copyCombination(currentModel.FeatureSet);
                newModel.Add(candidate);
                if (!fitModel(newModel))
                {
                    continue;
                }
                double temp         = 0;
                double errorOfModel = computeModelError(newModel, out temp);
                errorOfFeature.Add(candidate, temp);

                if (errorOfModel < minimalRoundError)
                {
                    minimalRoundError = errorOfModel;
                    minimalErrorModel = newModel;
                    bestCandidate     = candidate;
                }
            }
            double relativeErrorTrain = 0;
            double relativeErrorEval  = 0;

            addFeaturesToIgnore(errorOfFeature);
            LearningRound newRound = new LearningRound(minimalErrorModel, computeLearningError(minimalErrorModel, out relativeErrorTrain), computeValidationError(minimalErrorModel, out relativeErrorEval), currentModel.round + 1);

            newRound.learningError_relative   = relativeErrorTrain;
            newRound.validationError_relative = relativeErrorEval;
            return(newRound);
        }