/// <summary> /// Returns part of mldataset with features labels columns this is needed in case Excel export is performed. /// </summary> /// <param name="fun"></param> /// <param name="evParam"></param> /// <param name="device"></param> /// <returns></returns> public static Dictionary <string, List <List <float> > > FeaturesAndLabels(Function fun, EvaluationParameters evParam, DeviceDescriptor device) { try { //declare return vars var featDic = new Dictionary <string, List <List <float> > >(); while (true) { //get one minibatch of data for training var mbData = evParam.MBSource.GetNextMinibatch(evParam.MinibatchSize, device); var mdDataEx = MinibatchSourceEx.ToMinibatchValueData(mbData, evParam.Input.Union(evParam.Ouptut).ToList()); var inMap = new Dictionary <Variable, Value>(); //input var vars = evParam.Input; for (int i = 0; i < vars.Count() /*mdDataEx.Count*/; i++) { var vv = vars.ElementAt(i); var d = mdDataEx.Where(x => x.Key.Name.Equals(vv.Name)).FirstOrDefault(); // var fv = MLValue.GetValues(d.Key, d.Value); if (featDic.ContainsKey(d.Key.Name)) { featDic[d.Key.Name].AddRange(fv); } else { featDic.Add(d.Key.Name, fv); } } //output var varso = evParam.Ouptut; for (int i = 0; i < varso.Count() /*mdDataEx.Count*/; i++) { var vv = varso.ElementAt(i); var d = mdDataEx.Where(x => x.Key.Name.Equals(vv.Name)).FirstOrDefault(); // var fv = MLValue.GetValues(d.Key, d.Value); if (vv.Shape.Dimensions.Last() == 1) { var value = fv.Select(l => new List <float>() { l.First() }).ToList(); if (featDic.ContainsKey(d.Key.Name)) { featDic[d.Key.Name].AddRange(value); } else { featDic.Add(d.Key.Name, value); } } else { var value = fv.Select(l => new List <float>() { l.IndexOf(l.Max()) }).ToList(); if (featDic.ContainsKey(d.Key.Name)) { featDic[d.Key.Name].AddRange(value); } else { featDic.Add(d.Key.Name, value); } } } // check if sweep end reached if (mbData.Any(x => x.Value.sweepEnd)) { break; } } return(featDic); } catch (Exception) { throw; } }
public static (List <List <float> > actual, List <List <float> > predicted) EvaluateFunctionEx(Function fun, EvaluationParameters evParam, DeviceDescriptor device) { try { //declare return vars List <List <float> > actualLst = new List <List <float> >(); List <List <float> > predictedLst = new List <List <float> >(); while (true) { Value predicted = null; //get one minibatch of data for training var mbData = evParam.MBSource.GetNextMinibatch(evParam.MinibatchSize, device); var mbDataEx = MinibatchSourceEx.ToMinibatchValueData(mbData, evParam.Input.Union(evParam.Ouptut).ToList()); var inMap = new Dictionary <Variable, Value>(); // var vars = fun.Arguments.Union(fun.Outputs); for (int i = 0; i < vars.Count() /* mbDataEx.Count*/; i++) { var d = mbDataEx.ElementAt(i); var v = vars.Where(x => x.Name.Equals(d.Key.Name)).First(); //skip output data if (!evParam.Ouptut.Any(x => x.Name.Equals(v.Name))) { inMap.Add(v, d.Value); } } //actual data if t is available var actualVar = mbDataEx.Keys.Where(x => x.Name.Equals(evParam.Ouptut.First().Name)).FirstOrDefault(); var act = mbDataEx[actualVar].GetDenseData <float>(actualVar).Select(l => l.ToList()); actualLst.AddRange(act); //predicted data //map variables and data var predictedDataMap = new Dictionary <Variable, Value>() { { fun, null } }; //evaluates model fun.Evaluate(inMap, predictedDataMap, device); predicted = predictedDataMap.Values.First(); var pred = predicted.GetDenseData <float>(fun).Select(l => l.ToList()); predicted.Erase(); predicted.Dispose(); predictedLst.AddRange(pred); // check if sweep end reached if (mbData.Any(x => x.Value.sweepEnd)) { break; } } return(actualLst, predictedLst); } catch (Exception) { throw; } }
/// <summary> /// Evaluate model defined in the mlconfig file /// </summary> /// <param name="mlconfigPath"></param> /// <param name="device"></param> /// <returns></returns> public static async Task <EvaluationResult> EvaluateMLConfig(string mlconfigPath, DeviceDescriptor device, DataSetType dsType, EvaluationType evType) { try { //define eval result var er = new EvaluationResult(); er.OutputClasses = new List <string>() { "" }; er.Actual = new List <float>(); er.Predicted = new List <float>(); er.Header = new List <string>(); //Load ML configuration file var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath); //add full path of model folder since model file doesn't contains any absolute path dicMParameters.Add("root", MLFactory.GetMLConfigFolder(mlconfigPath)); // get model data paths var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]); //parse feature variables var projectValues = dicMParameters["training"].Split(MLFactory.m_cntkSpearator, StringSplitOptions.RemoveEmptyEntries); var modelName = MLFactory.GetParameterValue(projectValues, "TrainedModel"); var nnModelPath = Path.Combine(dicMParameters["root"], modelName); //check if model exists if (!MLFactory.IsFileExist(nnModelPath)) { return(er); } // var dataset = MLFactory.GetDataPath(dicMParameters, dsType); if (string.IsNullOrEmpty(dataset) || string.IsNullOrEmpty(dataset) || dataset == " ") { if (dsType == DataSetType.Testing) { dataset = MLFactory.GetDataPath(dicMParameters, DataSetType.Validation); } if (string.IsNullOrEmpty(dataset) || string.IsNullOrEmpty(dataset) || dataset == " ") { return(er); } } //get output classes in case the ml problem is classification var strCls = dicMParameters.ContainsKey("metadata") ? dicMParameters["metadata"] : ""; var oc = MLFactory.GetOutputClasses(strCls); if (oc != null) { er.OutputClasses = oc; } //MInibatch var mbTypestr = MLFactory.GetParameterValue(projectValues, "Type"); MinibatchType mbType = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true); var mbSizetr = MLFactory.GetParameterValue(projectValues, "BatchSize"); var mf = MLFactory.CreateMLFactory(dicMParameters); //perform evaluation var evParams = new EvaluationParameters() { MinibatchSize = uint.Parse(mbSizetr), MBSource = new MinibatchSourceEx(mbType, mf.StreamConfigurations.ToArray(), mf.InputVariables, mf.OutputVariables, dataset, null, MinibatchSource.FullDataSweep, false, 0), Input = mf.InputVariables, Ouptut = mf.OutputVariables, }; //evaluate model if (evType == EvaluationType.FeaturesOnly) { if (!dicMParameters.ContainsKey("metadata")) { throw new Exception("The result cannot be exported to Excel, since no metadata is stored in mlconfig file."); } var desc = MLFactory.ParseRawDataSet(dicMParameters["metadata"]); er.Header = MLFactory.GenerateHeader(desc); var fun = Function.Load(nnModelPath, device); // er.DataSet = await Task.Run(() => MLEvaluator.FeaturesAndLabels(fun, evParams, device)); return(er); } else if (evType == EvaluationType.Results) { //define header er.Header.Add(evParams.Ouptut.First().Name + "_actual"); er.Header.Add(evParams.Ouptut.First().Name + "_predicted"); var fun = Function.Load(nnModelPath, device); // var result = await Task.Run(() => MLEvaluator.EvaluateFunction(fun, evParams, device)); er.Actual = result.actual.ToList(); er.Predicted = result.predicted.ToList(); if (er.OutputClasses.Count < 2 && evParams.Ouptut.First().Shape.Dimensions.Last() > 1) { var result1 = await Task.Run(() => MLEvaluator.EvaluateFunctionEx(fun, evParams, device)); er.ActualEx = result1.actual; er.PredictedEx = result1.predicted; } return(er); } else if (evType == EvaluationType.ResultExtended) { //define header er.Header.Add(evParams.Ouptut.First().Name + "_actual"); er.Header.Add(evParams.Ouptut.First().Name + "_predicted"); er.Actual = new List <float>(); er.Predicted = new List <float>(); er.ActualEx = new List <List <float> >(); er.PredictedEx = new List <List <float> >(); // var fun = Function.Load(nnModelPath, device); var resultEx = await Task.Run(() => MLEvaluator.EvaluateFunctionEx(fun, evParams, device)); //var resultEx = EvaluateFunctionEx(nnModelPath, dataPath, evParams, device); for (int i = 0; i < resultEx.actual.Count(); i++) { var res1 = MLValue.GetResult(resultEx.actual[i]); er.Actual.Add(res1); var res2 = MLValue.GetResult(resultEx.predicted[i]); er.Predicted.Add(res2); } er.ActualEx = resultEx.actual; er.PredictedEx = resultEx.predicted; return(er); } else { throw new Exception("Unknown evaluation type!"); } } catch (Exception) { throw; } }
public static (IEnumerable <float> actual, IEnumerable <float> predicted) EvaluateFunction(Function fun, EvaluationParameters evParam, DeviceDescriptor device) { try { //declare return vars List <float> actualLst = new List <float>(); List <float> predictedLst = new List <float>(); var result = EvaluateFunctionEx(fun, evParam, device); for (int i = 0; i < result.actual.Count(); i++) { var res1 = MLValue.GetResult(result.actual[i]); actualLst.Add(res1); var res2 = MLValue.GetResult(result.predicted[i]); predictedLst.Add(res2); } return(actualLst, predictedLst); } catch (Exception) { throw; } }
protected virtual ProgressData progressTraining(TrainingParameters trParams, Trainer trainer, Function network, MinibatchSourceEx mbs, int epoch, TrainingProgress progress, DeviceDescriptor device) { //calculate average training loss and evaluation var mbAvgLoss = trainer.PreviousMinibatchLossAverage(); var mbAvgEval = trainer.PreviousMinibatchEvaluationAverage(); //get training dataset double trainEval = mbAvgEval; //sometimes when the data set is huge validation model against // full training dataset could take time, so we can skip it by setting parameter 'FullTrainingSetEval' if (trParams.FullTrainingSetEval) { var evParams = new EvaluationParameters() { MinibatchSize = trParams.BatchSize, MBSource = new MinibatchSourceEx(mbs.Type, this.StreamConfigurations.ToArray(), this.InputVariables, this.OutputVariables, mbs.TrainingDataFile, null, MinibatchSource.FullDataSweep, false, 0), Ouptut = OutputVariables, Input = InputVariables, }; var result = MLEvaluator.EvaluateFunction(trainer.Model(), evParams, device); trainEval = MLEvaluator.CalculateMetrics(trainer.EvaluationFunction().Name, result.actual, result.predicted, device); ////if output has more than one dimension and when the output is not categorical but numeric with more than one value ////for now only custom mini-batch source is supported this kind of variable //if(OutputVariables.First().Shape.Dimensions.Last() > 1 && evParams.MBSource.Type== MinibatchType.Custom) //{ // var result1 = MLEvaluator.EvaluateFunctionEx(trainer.Model(), evParams, device); // trainEval = MLEvaluator.CalculateMetrics(trainer.EvaluationFunction().Name, result1.actual, result1.predicted, device); //} //else //{ // var result = MLEvaluator.EvaluateFunction(trainer.Model(), evParams, device); // trainEval = MLEvaluator.CalculateMetrics(trainer.EvaluationFunction().Name, result.actual, result.predicted, device); //} } string bestModelPath = m_bestModelPath; double validEval = 0; //in case validation data set is empty don't perform test-minibatch if (!string.IsNullOrEmpty(mbs.ValidationDataFile)) { var evParams = new EvaluationParameters() { MinibatchSize = trParams.BatchSize, //StrmsConfig = StreamConfigurations.ToArray(), MBSource = new MinibatchSourceEx(mbs.Type, this.StreamConfigurations.ToArray(), this.InputVariables, this.OutputVariables, mbs.ValidationDataFile, null, MinibatchSource.FullDataSweep, false, 0), Ouptut = OutputVariables, Input = InputVariables, }; // var result = MLEvaluator.EvaluateFunction(trainer.Model(), evParams, device); validEval = MLEvaluator.CalculateMetrics(trainer.EvaluationFunction().Name, result.actual, result.predicted, device); ////if output has more than one dimension and when the output is not categorical but numeric with more than one value ////for now only custom mini-batch source is supported this kind of variable //if (OutputVariables.First().Shape.Dimensions.Last() > 1 && evParams.MBSource.Type == MinibatchType.Custom) //{ // var result1 = MLEvaluator.EvaluateFunctionEx(trainer.Model(), evParams, device); // validEval = MLEvaluator.CalculateMetrics(trainer.EvaluationFunction().Name, result1.actual, result1.predicted, device); //} //else //{ // var result = MLEvaluator.EvaluateFunction(trainer.Model(), evParams, device); // validEval = MLEvaluator.CalculateMetrics(trainer.EvaluationFunction().Name, result.actual, result.predicted, device); //} } //here we should decide if the current model worth to be saved into temp location // depending of the Evaluation function which sometimes can be better if it is greater that previous (e.g. ClassificationAccuracy) if (isBetterThanPrevious(trainEval, validEval, StatMetrics.IsGoalToMinimize(trainer.EvaluationFunction())) && trParams.SaveModelWhileTraining) { //save model var strFilePath = $"{trParams.ModelTempLocation}\\model_at_{epoch}of{trParams.Epochs}_epochs_TimeSpan_{DateTime.Now.Ticks}"; if (!Directory.Exists(trParams.ModelTempLocation)) { Directory.CreateDirectory(trParams.ModelTempLocation); } //save temp model network.Save(strFilePath); //set training and validation evaluation to previous state m_PrevTrainingEval = trainEval; m_PrevValidationEval = validEval; bestModelPath = strFilePath; var tpl = Tuple.Create <double, double, string>(trainEval, validEval, strFilePath); m_ModelEvaluations.Add(tpl); } m_bestModelPath = bestModelPath; //create progressData object var prData = new ProgressData(); prData.EpochTotal = trParams.Epochs; prData.EpochCurrent = epoch; prData.EvaluationFunName = trainer.EvaluationFunction().Name; prData.TrainEval = trainEval; prData.ValidationEval = validEval; prData.MinibatchAverageEval = mbAvgEval; prData.MinibatchAverageLoss = mbAvgLoss; //prData.BestModel = bestModelPath; //the progress is only reported if satisfied the following condition if (progress != null && (epoch % trParams.ProgressFrequency == 0 || epoch == 1 || epoch == trParams.Epochs)) { //add info to the history m_trainingHistory.Add(new Tuple <int, float, float, float, float>(epoch, (float)mbAvgLoss, (float)mbAvgEval, (float)trainEval, (float)validEval)); //send progress progress(prData); // //Console.WriteLine($"Epoch={epoch} of {trParams.Epochs} processed."); } //return progress data return(prData); }