예제 #1
0
        public async Task <ModelEvaluation> EvaluateModel()
        {
            try
            {
                //change application in run mode
                IconUri = "Images/runningmodel.png";
                RaisePropertyChangedEvent("IsRunning");

                //init empty model evaluation
                var mEval = new ModelEvaluation()
                {
                    TrainingValue        = new List <ZedGraph.PointPair>(),
                    ValidationValue      = new List <ZedGraph.PointPair>(),
                    ModelValueTraining   = new List <ZedGraph.PointPair>(),
                    ModelValueValidation = new List <ZedGraph.PointPair>(),
                    Classes        = new List <string>(),
                    ModelOutputDim = 1
                };
                var mpt = new ModelPerformance();
                mpt.DatSetName = "Training set";
                var mpv = new ModelPerformance();
                mpv.DatSetName = "Validation set";


                //check if the trained model exists
                if (string.IsNullOrEmpty(TrainingParameters.LastBestModel) || string.IsNullOrEmpty(TrainingParameters.LastBestModel.Trim(' ')))
                {
                    await Application.Current.Dispatcher.BeginInvoke(
                        DispatcherPriority.Background,
                        new Action(
                            () => MainWindow.SetCursor(false)
                            ));


                    return(mEval);
                }


                //get model full path
                var modelMLPath = Project.GetMLConfigPath(Settings, Name);
                //check if file exists
                var fi = new FileInfo(modelMLPath);
                if (!fi.Exists)
                {
                    return(mEval);
                }
                //evaluate model against training data
                var task1 = await Task.Run(() => Project.EvaluateMLConfig(modelMLPath, DataSetType.Training, EvaluationType.ResultExtended, ProcessDevice.Default));

                var resultTrain = task1;

                //evaluate model against validation data
                var task2 = await Task.Run(() => Project.EvaluateMLConfig(modelMLPath, DataSetType.Validation, EvaluationType.ResultExtended, ProcessDevice.Default));

                var resultValidation = task2;

                if (resultTrain.Actual == null && resultTrain.Actual.Count <= 0)
                {
                    return(mEval);
                }

                ////prepare evaluation result
                for (int i = 0; i < resultTrain.Actual.Count(); i++)
                {
                    mEval.TrainingValue.Add(new PointPair(i + 1, resultTrain.Actual[i]));
                }

                for (int i = 0; i < resultTrain.Predicted.Count(); i++)
                {
                    mEval.ModelValueTraining.Add(new PointPair(i + 1, resultTrain.Predicted[i]));
                }

                ////no validation set defined
                if (resultValidation.Actual != null && resultValidation.Actual.Count > 0)
                {
                    for (int i = 0; i < resultValidation.Actual.Count(); i++)
                    {
                        mEval.ValidationValue.Add(new PointPair(i + 1, resultValidation.Actual[i]));
                    }


                    for (int i = 0; i < resultValidation.Predicted.Count(); i++)
                    {
                        mEval.ModelValueValidation.Add(new PointPair(i + 1, resultValidation.Predicted[i]));
                    }
                }

                ////
                mEval.Classes        = resultTrain.OutputClasses;
                mEval.ModelOutputDim = resultTrain.OutputClasses == null ? 1 : resultTrain.OutputClasses.Count;

                //training performance result
                mpt = MLEvaluator.CalculatePerformance(resultTrain, "Training set");
                //validation performance result
                mpv = MLEvaluator.CalculatePerformance(resultValidation, "Validation set");


                mEval.TrainPerformance = mpt;
                if (mEval.Classes != null)
                {
                    mEval.TrainPerformance.Classes = mEval.Classes.ToArray();
                }
                mEval.ValidationPerformance = mpv;
                if (mEval.Classes != null)
                {
                    mEval.ValidationPerformance.Classes = mEval.Classes.ToArray();
                }
                ModelEvaluation = mEval;

                return(mEval);
            }
            catch (Exception)
            {
                throw;
            }
            finally
            {
                //change application in normal mode
                IconUri = "Images/model.png";
                RaisePropertyChangedEvent("IsRunning");
            }
        }
예제 #2
0
        public async Task <ModelEvaluation> EvaluateModel()
        {
            try
            {
                //change application in run mode
                IconUri = "Images/runningmodel.png";
                RaisePropertyChangedEvent("IsRunning");

                //init empty model evaluation
                var mEval = new ModelEvaluation()
                {
                    TrainingValue        = new List <ZedGraph.PointPair>(),
                    ValidationValue      = new List <ZedGraph.PointPair>(),
                    ModelValueTraining   = new List <ZedGraph.PointPair>(),
                    ModelValueValidation = new List <ZedGraph.PointPair>(),
                    Classes        = new List <string>(),
                    ModelOutputDim = 1
                };
                var mpt = new ModelPerformance();
                mpt.DatSetName = "Training set";
                var mpv = new ModelPerformance();
                mpv.DatSetName = "Validation set";


                //check if the trained model exists
                if (string.IsNullOrEmpty(TrainingParameters.LastBestModel) || string.IsNullOrEmpty(TrainingParameters.LastBestModel.Trim(' ')))
                {
                    return(mEval);
                }

                //save model before evaluation since there is a data must be stored into model file.
                Save();

                //get model full path
                var modelMLPath = Project.GetMLConfigPath(Settings, Name);
                //check if file exists
                var fi = new FileInfo(modelMLPath);
                if (!fi.Exists)
                {
                    return(mEval);
                }
                //evaluate model against training data
                var task1 = await Task.Run(() => Project.EvaluateModel(modelMLPath, DataProcessing.Core.DataSetType.Training, EvaluationType.ResultyExtended, ProcessDevice.Default));

                var resultTrain = task1;

                //evaluate model against validation data
                var task2 = await Task.Run(() => Project.EvaluateModel(modelMLPath, DataProcessing.Core.DataSetType.Validation, EvaluationType.ResultyExtended, ProcessDevice.Default));

                var resultValidation = task2;

                if (resultTrain.Actual == null && resultTrain.Actual.Count <= 0)
                {
                    return(mEval);
                }

                ////prepare evaluation result
                for (int i = 0; i < resultTrain.Actual.Count(); i++)
                {
                    mEval.TrainingValue.Add(new PointPair(i + 1, resultTrain.Actual[i]));
                }

                for (int i = 0; i < resultTrain.Predicted.Count(); i++)
                {
                    mEval.ModelValueTraining.Add(new PointPair(i + 1, resultTrain.Predicted[i]));
                }

                ////no validation set defined
                if (resultValidation.Actual != null && resultValidation.Actual.Count > 0)
                {
                    for (int i = 0; i < resultValidation.Actual.Count(); i++)
                    {
                        mEval.ValidationValue.Add(new PointPair(i + 1, resultValidation.Actual[i]));
                    }


                    for (int i = 0; i < resultValidation.Predicted.Count(); i++)
                    {
                        mEval.ModelValueValidation.Add(new PointPair(i + 1, resultValidation.Predicted[i]));
                    }
                }


                ////
                mEval.Classes        = resultTrain.OutputClasses;
                mEval.ModelOutputDim = resultTrain.OutputClasses == null ? 1 : resultTrain.OutputClasses.Count;

                if (mEval.ModelOutputDim == 1)
                {
                    //Training data set
                    var actTData = mEval.TrainingValue.Select(x => x.Y).ToArray();
                    var preTData = mEval.ModelValueTraining.Select(x => x.Y).ToArray();
                    mpt.SE   = (float)actTData.SE(preTData);
                    mpt.RMSE = (float)actTData.RMSE(preTData);
                    mpt.NSE  = (float)actTData.NSE(preTData);
                    mpt.PB   = (float)actTData.PBIAS(preTData);
                    mpt.CORR = (float)actTData.R(preTData);
                    mpt.DETC = (float)actTData.R2(preTData);

                    //validation data set
                    var actVData = mEval.ValidationValue.Select(x => x.Y).ToArray();
                    var preVData = mEval.ModelValueValidation.Select(x => x.Y).ToArray();
                    if (actVData != null && actVData.Length > 0)
                    {
                        mpv.SE   = (float)actVData.SE(preVData);
                        mpv.RMSE = (float)actVData.RMSE(preVData);
                        mpv.NSE  = (float)actVData.NSE(preVData);
                        mpv.PB   = (float)actVData.PBIAS(preVData);
                        mpv.CORR = (float)actVData.R(preVData);
                        mpv.DETC = (float)actVData.R2(preVData);
                    }
                }
                else if (mEval.ModelOutputDim > 1)
                {
                    var retVal = CalculatePerformance(resultTrain.ActualEx, resultTrain.PredictedEx, null, null);
                    retVal.Add("Classes", mEval.Classes.ToList <object>());
                    mpt.PerformanceData = retVal;

                    //in case validation set is defined
                    if (resultValidation.Actual != null && resultValidation.Actual.Count > 0)
                    {
                        var retValV = CalculatePerformance(resultValidation.ActualEx, resultValidation.PredictedEx, null, null);
                        retValV.Add("Classes", mEval.Classes.ToList <object>());
                        mpv.PerformanceData = retValV;
                    }
                }
                mEval.TrainPerformance = mpt;
                if (mEval.Classes != null)
                {
                    mEval.TrainPerformance.Classes = mEval.Classes.ToArray();
                }
                mEval.ValidationPerformance = mpv;
                if (mEval.Classes != null)
                {
                    mEval.ValidationPerformance.Classes = mEval.Classes.ToArray();
                }
                ModelEvaluation = mEval;

                return(mEval);
            }
            catch (Exception)
            {
                throw;
            }
            finally
            {
                //change application in normal mode
                IconUri = "Images/model.png";
                RaisePropertyChangedEvent("IsRunning");
            }
        }