Example #1
0
        /// <summary>
        /// Evaluate the model against dataset sored in the dataset file, and exports the result in csv format for further analysis
        /// </summary>
        /// <param name="mlF"> ml factory object contains members needed to evaluation process</param>
        /// <param name="mbs"> Minibatch source which provides helpers members needed for for evaluation</param>
        /// <param name="strDataSetPath"> file of dataset</param>
        /// <param name="modelPath"> models which will be evaluate</param>
        /// <param name="resultExportPath"> result file in which the result will be exported</param>
        /// <param name="device"> device for computation</param>
        public static void EvaluateModel(string mlconfigPath, string bestTrainedModelPath, DeviceDescriptor device)
        {
            //Load ML model configuration file
            var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath);

            //add full path of model folder since model file doesn't contains any absolute path
            dicMParameters.Add("root", MLFactory.GetMLConfigFolder(mlconfigPath));

            //get model daa paths
            var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]);

            //parse feature variables
            var projectValues            = dicMParameters["training"].Split(MLFactory.m_cntkSpearator, StringSplitOptions.RemoveEmptyEntries);
            var trainedModelRelativePath = MLFactory.GetParameterValue(projectValues, "TrainedModel");


            //Minibatch type
            var           mbTypestr = MLFactory.GetParameterValue(projectValues, "Type");
            MinibatchType mbType    = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true);
            //prepare MLFactory
            var f = MLFactory.CreateMLFactory(dicMParameters);

            //prepare data paths for mini-batch source
            var strTrainPath = $"{dicMParameters["root"]}\\{dicPath["Training"]}";
            var strValidPath = $"{dicMParameters["root"]}\\{dicPath["Validation"]}";
            var strResult    = $"{dicMParameters["root"]}\\{dicPath["Result"]}";

            var bestModelFullPath = $"{dicMParameters["root"]}\\{bestTrainedModelPath}";
            //decide what data to evaluate
            var dataPath = strValidPath;

            //load model
            var model = Function.Load(bestModelFullPath, device);

            //get data for evaluation by calling GetFullBatch
            var minibatchData = MinibatchSourceEx.GetFullBatch(mbType, dataPath, f.StreamConfigurations.ToArray(), device);
            //input map creation for model evaluation
            var inputMap = new Dictionary <Variable, Value>();

            foreach (var v in minibatchData)
            {
                var vv         = model.Arguments.Where(x => x.Name == v.Key.m_name).FirstOrDefault();
                var streamInfo = v.Key;
                if (vv != null)
                {
                    inputMap.Add(vv, minibatchData[streamInfo].data);
                }
            }

            //output map
            var predictedDataMap = new Dictionary <Variable, Value>();

            foreach (var outp in model.Outputs)
            {
                predictedDataMap.Add(outp, null);
            }

            //model evaluation
            model.Evaluate(inputMap, predictedDataMap, device);

            //retrieve actual and predicted values from model
            List <List <float> > actual  = new List <List <float> >();
            List <List <float> > predict = new List <List <float> >();

            foreach (var output in model.Outputs)
            {
                //label stream
                var labelStream = minibatchData.Keys.Where(x => x.m_name == output.Name).First();

                //actual values
                List <List <float> > av = MLValue.GetValues(output, minibatchData[labelStream].data);
                //predicted values
                List <List <float> > pv = MLValue.GetValues(output, predictedDataMap[output]);

                for (int i = 0; i < av.Count; i++)
                {
                    //actual
                    var act = av[i];
                    if (actual.Count <= i)
                    {
                        actual.Add(new List <float>());
                    }
                    actual[i].AddRange(act);
                    //prediction
                    var prd = pv[i];
                    if (predict.Count <= i)
                    {
                        predict.Add(new List <float>());
                    }
                    predict[i].AddRange(prd);
                }
            }


            //export result
            MLValue.ValueToFile(actual, predict, strResult);

            //
            Console.WriteLine(Environment.NewLine);
            Console.WriteLine($"*******************Model Evaluation**************");
            Console.WriteLine(Environment.NewLine);
            Console.WriteLine($"Model Evaluation successfully exported result into file {strResult}!");
            Console.WriteLine(Environment.NewLine);
        }
Example #2
0
        public static EvaluationResult EvaluateModel(string mlconfigPath, DataSetType dsType, EvaluationType evType, ProcessDevice pdevice)
        {
            var er = new EvaluationResult();

            er.Header = new List <string>();
            //device definition
            DeviceDescriptor device = MLFactory.GetDevice(pdevice);
            //Load ML model configuration file
            var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath);

            //add full path of model folder since model file doesn't contains any absolute path
            dicMParameters.Add("root", Project.GetMLConfigFolder(mlconfigPath));

            // get model data paths
            var dicPath     = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]);
            var modelName   = Project.GetParameterValue(dicMParameters["training"], "TrainedModel");
            var nnModelPath = Path.Combine(dicMParameters["root"], modelName);

            //check if model exists
            if (!MLFactory.IsFileExist(nnModelPath))
            {
                return(er);
            }


            //check if dataset files exist
            var dataPath = GetDataPath(dicMParameters, dsType);

            if (!MLFactory.IsFileExist(dataPath))
            {
                //in case validation dataset is not defiend just export traininign dataset
                if (dsType == DataSetType.Validation)
                {
                    dataPath = GetDataPath(dicMParameters, DataSetType.Training);
                }
                if (!MLFactory.IsFileExist(dataPath))
                {
                    return(er);
                }
            }

            //get output classes in case the ml problem is classification
            var strCls = dicMParameters.ContainsKey("metadata") ? dicMParameters["metadata"] : "";

            er.OutputClasses = DataDescriptor.GetOutputClasses(strCls);

            //Minibatch type
            var           mbTypestr = Project.GetParameterValue(dicMParameters["training"], "Type");
            MinibatchType mbType    = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true);
            var           mbSizetr  = Project.GetParameterValue(dicMParameters["training"], "BatchSize");

            var mf = MLFactory.CreateMLFactory(dicMParameters);
            //perform evaluation
            var evParams = new EvaluationParameters()
            {
                MinibatchSize = uint.Parse(mbSizetr),
                MBSource      = new MinibatchSourceEx(mbType, mf.StreamConfigurations.ToArray(), dataPath, null, MinibatchSource.FullDataSweep, false),
                Input         = mf.InputVariables,
                Ouptut        = mf.OutputVariables,
            };

            //evaluate model
            if (evType == EvaluationType.FeaturesOnly)
            {
                if (!dicMParameters.ContainsKey("metadata"))
                {
                    throw new Exception("The result cannot be exported to Excel, since no metadata is stored in mlconfig file.");
                }
                var desc = ParseRawDataSet(dicMParameters["metadata"]);
                er.Header  = generateHeader(desc);
                er.DataSet = FeatureAndLabels(nnModelPath, dataPath, evParams, device);

                return(er);
            }
            else if (evType == EvaluationType.Results)
            {
                //define header
                er.Header.Add(evParams.Ouptut.First().Name + "_actual");
                er.Header.Add(evParams.Ouptut.First().Name + "_predicted");

                var result = EvaluateFunction(nnModelPath, dataPath, evParams, device);
                er.Actual    = result.actual.ToList();
                er.Predicted = result.predicted.ToList();
                return(er);
            }
            else if (evType == EvaluationType.ResultyExtended)
            {
                //define header
                er.Header.Add(evParams.Ouptut.First().Name + "_actual");
                er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
                er.Actual      = new List <float>();
                er.Predicted   = new List <float>();
                er.ActualEx    = new List <List <float> >();
                er.PredictedEx = new List <List <float> >();
                //
                var resultEx = EvaluateFunctionEx(nnModelPath, dataPath, evParams, device);
                for (int i = 0; i < resultEx.actual.Count(); i++)
                {
                    var res1 = MLValue.GetResult(resultEx.actual[i]);
                    er.Actual.Add(res1);
                    var res2 = MLValue.GetResult(resultEx.predicted[i]);
                    er.Predicted.Add(res2);
                }
                er.ActualEx    = resultEx.actual;
                er.PredictedEx = resultEx.predicted;

                return(er);
            }
            else
            {
                throw new Exception("Unknown evaluation type!");
            }
        }