public static Tuple <bool, bool, bool> GetDataSetAviability(string mlConfigPath) { try { //Load ML model configuration file var dicMParameters = MLFactory.LoadMLConfiguration(mlConfigPath); //add full path of model folder since model file doesn't contains any absolute path dicMParameters.Add("root", MLFactory.GetMLConfigFolder(mlConfigPath)); //get model daa paths var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]); var training = dicPath["Training"] != " "; var validation = dicPath["Validation"] != " "; var test = dicPath["Test"] != " "; return(new Tuple <bool, bool, bool>(training, validation, test)); } catch (Exception) { throw; } }
public static string GetMLConfigId(string mlconfigPath) { try { //Load ML configuration file var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath); if (dicMParameters.ContainsKey("configid")) { return(dicMParameters["configid"].Trim(' ')); } else { return(null); } } catch (Exception) { throw; } }
/// <summary> /// /// </summary> /// <param name="trData"></param> /// <param name="bestModelFile"></param> /// <returns></returns> public static string ReplaceBestModel(TrainingParameters trData, string mlconfigPath, string bestModelFile) { try { if (!string.IsNullOrEmpty(trData.LastBestModel) && trData.LastBestModel != " ") { var mlconfigFolder = MLFactory.GetMLConfigFolder(mlconfigPath); var oldPath = Path.Combine(mlconfigFolder, trData.LastBestModel); //delete history if (File.Exists(oldPath)) { File.Delete(oldPath); } } //set new value after delete old model trData.LastBestModel = bestModelFile; return(bestModelFile); } catch (Exception) { throw; } }
/// <summary> /// Parses the mlconfig content and return dictionary of the model components /// </summary> /// <param name="pathsString"></param> /// <returns></returns> public static Dictionary <string, string> GetMLConfigComponentPaths(string pathsString) { var dicValues = new Dictionary <string, string>(); //parse feature variables var pathsValues = pathsString.Split(m_cntkSpearator, StringSplitOptions.RemoveEmptyEntries); //training path var val = MLFactory.GetParameterValue(pathsValues, "Training"); if (string.IsNullOrEmpty(val)) { throw new Exception("One of model paths (Training) is not defined!"); } dicValues.Add("Training", val); val = MLFactory.GetParameterValue(pathsValues, "Validation"); if (string.IsNullOrEmpty(val)) { dicValues.Add("Validation", " "); } else { dicValues.Add("Validation", val); } val = MLFactory.GetParameterValue(pathsValues, "Test"); if (string.IsNullOrEmpty(val)) { dicValues.Add("Test", " "); } else { dicValues.Add("Test", val); } val = MLFactory.GetParameterValue(pathsValues, "TempModels"); if (string.IsNullOrEmpty(val)) { throw new Exception("One of model paths (TempModels) is not defined!"); } dicValues.Add("TempModels", val); val = MLFactory.GetParameterValue(pathsValues, "Models"); if (string.IsNullOrEmpty(val)) { throw new Exception("One of model paths (Models) is not defined!"); } dicValues.Add("Models", val); val = MLFactory.GetParameterValue(pathsValues, "Result"); if (string.IsNullOrEmpty(val)) { throw new Exception("One of model paths (Result) is not defined!"); } dicValues.Add("Result", val); val = MLFactory.GetParameterValue(pathsValues, "Logs"); if (string.IsNullOrEmpty(val)) { throw new Exception("One of model paths (Log) is not defined!"); } dicValues.Add("Logs", val); return(dicValues); }
/// <summary> /// Test cntk model stored at 'modelPath' against array of image paths /// </summary> /// <param name="modelPath"></param> /// <param name="vector"></param> /// <param name="device"></param> /// <returns></returns> public static List <int> TestModel(string modelPath, string[] imagePaths, DeviceDescriptor device) { try { // FileInfo fi = new FileInfo(modelPath); if (!fi.Exists) { throw new Exception($"The '{fi.FullName}' does not exist. Make sure the model is places at this location."); } //load the model from disk var model = Function.Load(fi.FullName, device); //get input feature var features = model.Arguments.ToList(); var labels = model.Outputs.ToList(); var stremsConfig = MLFactory.CreateStreamConfiguration(features, labels); var mapFile = "testMapFile"; File.WriteAllLines(mapFile, imagePaths.Select(x => $"{x}\t0")); var testMB = new MinibatchSourceEx(MinibatchType.Image, stremsConfig.ToArray(), features, labels, mapFile, null, 30, false, 0); // var vars = features.Union(labels).ToList(); var retVal = new List <int>(); var mbSize = imagePaths.Count(); if (mbSize > 30) { mbSize = 30; } while (true) { bool isSweepEnd = false; var inputMap = testMB.GetNextMinibatch((uint)mbSize, ref isSweepEnd, vars, device); //prepare data for trainer //var inputMap = new Dictionary<Variable, Value>(); //inputMap.Add(features.First(), nextMB.Where(x => x.Key.m_name.Equals(features.First().Name)).Select(x => x.Value.data).FirstOrDefault()); var outputMap = new Dictionary <Variable, Value>(); outputMap.Add(labels.First(), null); //evaluate model model.Evaluate(inputMap, outputMap, device); var result = outputMap[labels.First()].GetDenseData <float>(labels.First()); //extract result foreach (var r in result) { var l = MLValue.GetResult(r); retVal.Add((int)l); } if (/*nextMB.Any(x => x.Value.sweepEnd)*/ isSweepEnd) { break; } } return(retVal); } catch (Exception) { throw; } }
/// <summary> /// Evaluate model defined in the mlconfig file /// </summary> /// <param name="mlconfigPath"></param> /// <param name="device"></param> /// <returns></returns> public static async Task <EvaluationResult> EvaluateMLConfig(string mlconfigPath, DeviceDescriptor device, DataSetType dsType, EvaluationType evType) { try { //define eval result var er = new EvaluationResult(); er.OutputClasses = new List <string>() { "" }; er.Actual = new List <float>(); er.Predicted = new List <float>(); er.Header = new List <string>(); //Load ML configuration file var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath); //add full path of model folder since model file doesn't contains any absolute path dicMParameters.Add("root", MLFactory.GetMLConfigFolder(mlconfigPath)); // get model data paths var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]); //parse feature variables var projectValues = dicMParameters["training"].Split(MLFactory.m_cntkSpearator, StringSplitOptions.RemoveEmptyEntries); var modelName = MLFactory.GetParameterValue(projectValues, "TrainedModel"); var nnModelPath = Path.Combine(dicMParameters["root"], modelName); //check if model exists if (!MLFactory.IsFileExist(nnModelPath)) { return(er); } // var dataset = MLFactory.GetDataPath(dicMParameters, dsType); if (string.IsNullOrEmpty(dataset) || string.IsNullOrEmpty(dataset) || dataset == " ") { if (dsType == DataSetType.Testing) { dataset = MLFactory.GetDataPath(dicMParameters, DataSetType.Validation); } if (string.IsNullOrEmpty(dataset) || string.IsNullOrEmpty(dataset) || dataset == " ") { return(er); } } //get output classes in case the ml problem is classification var strCls = dicMParameters.ContainsKey("metadata") ? dicMParameters["metadata"] : ""; var oc = MLFactory.GetOutputClasses(strCls); if (oc != null) { er.OutputClasses = oc; } //MInibatch var mbTypestr = MLFactory.GetParameterValue(projectValues, "Type"); MinibatchType mbType = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true); var mbSizetr = MLFactory.GetParameterValue(projectValues, "BatchSize"); var mf = MLFactory.CreateMLFactory(dicMParameters); //perform evaluation var evParams = new EvaluationParameters() { MinibatchSize = uint.Parse(mbSizetr), MBSource = new MinibatchSourceEx(mbType, mf.StreamConfigurations.ToArray(), mf.InputVariables, mf.OutputVariables, dataset, null, MinibatchSource.FullDataSweep, false, 0), Input = mf.InputVariables, Ouptut = mf.OutputVariables, }; //evaluate model if (evType == EvaluationType.FeaturesOnly) { if (!dicMParameters.ContainsKey("metadata")) { throw new Exception("The result cannot be exported to Excel, since no metadata is stored in mlconfig file."); } var desc = MLFactory.ParseRawDataSet(dicMParameters["metadata"]); er.Header = MLFactory.GenerateHeader(desc); var fun = Function.Load(nnModelPath, device); // er.DataSet = await Task.Run(() => MLEvaluator.FeaturesAndLabels(fun, evParams, device)); return(er); } else if (evType == EvaluationType.Results) { //define header er.Header.Add(evParams.Ouptut.First().Name + "_actual"); er.Header.Add(evParams.Ouptut.First().Name + "_predicted"); var fun = Function.Load(nnModelPath, device); // var result = await Task.Run(() => MLEvaluator.EvaluateFunction(fun, evParams, device)); er.Actual = result.actual.ToList(); er.Predicted = result.predicted.ToList(); if (er.OutputClasses.Count < 2 && evParams.Ouptut.First().Shape.Dimensions.Last() > 1) { var result1 = await Task.Run(() => MLEvaluator.EvaluateFunctionEx(fun, evParams, device)); er.ActualEx = result1.actual; er.PredictedEx = result1.predicted; } return(er); } else if (evType == EvaluationType.ResultExtended) { //define header er.Header.Add(evParams.Ouptut.First().Name + "_actual"); er.Header.Add(evParams.Ouptut.First().Name + "_predicted"); er.Actual = new List <float>(); er.Predicted = new List <float>(); er.ActualEx = new List <List <float> >(); er.PredictedEx = new List <List <float> >(); // var fun = Function.Load(nnModelPath, device); var resultEx = await Task.Run(() => MLEvaluator.EvaluateFunctionEx(fun, evParams, device)); //var resultEx = EvaluateFunctionEx(nnModelPath, dataPath, evParams, device); for (int i = 0; i < resultEx.actual.Count(); i++) { var res1 = MLValue.GetResult(resultEx.actual[i]); er.Actual.Add(res1); var res2 = MLValue.GetResult(resultEx.predicted[i]); er.Predicted.Add(res2); } er.ActualEx = resultEx.actual; er.PredictedEx = resultEx.predicted; return(er); } else { throw new Exception("Unknown evaluation type!"); } } catch (Exception) { throw; } }
/// <summary> /// Main method for training /// </summary> /// <param name="trainer"></param> /// <param name="network"></param> /// <param name="trParams"></param> /// <param name="miniBatchSource"></param> /// <param name="device"></param> /// <param name="token"></param> /// <param name="progress"></param> /// <param name="modelCheckPoint"></param> /// <returns></returns> public override TrainResult Train(Trainer trainer, Function network, TrainingParameters trParams, MinibatchSourceEx miniBatchSource, DeviceDescriptor device, CancellationToken token, TrainingProgress progress, string modelCheckPoint, string historyPath) { try { //create trainer result. // the variable indicate how training process is ended // completed, stopped, crashed, var trainResult = new TrainResult(); var historyFile = ""; //create training process evaluation collection //for each iteration it is stored evaluationValue for training, and validation set with the model m_ModelEvaluations = new List <Tuple <double, double, string> >(); //check what is the optimization (Minimization (error) or maximization (accuracy)) bool isMinimize = StatMetrics.IsGoalToMinimize(trainer.EvaluationFunction()); //setup first iteration if (m_trainingHistory == null) { m_trainingHistory = new List <Tuple <int, float, float, float, float> >(); } //in case of continuation of training iteration must start with the last of path previous training process int epoch = (m_trainingHistory.Count > 0)? m_trainingHistory.Last().Item1 + 1:1; //define progressData ProgressData prData = null; //define helper variable collection var vars = InputVariables.Union(OutputVariables).ToList(); //training process while (true) { //get next mini batch data //var args = miniBatchSource.GetNextMinibatch(trParams.BatchSize, device); //var isSweepEnd = args.Any(a => a.Value.sweepEnd); ////prepare the data for trainer //var arguments = MinibatchSourceEx.ToMinibatchValueData(args, vars); // var isSweepEnd = false; var arguments = miniBatchSource.GetNextMinibatch(trParams.BatchSize, ref isSweepEnd, vars, device); trainer.TrainMinibatch(arguments, isSweepEnd, device); //make progress if (isSweepEnd) { //check the progress of the training process prData = progressTraining(trParams, trainer, network, miniBatchSource, epoch, progress, device); //check if training process ends if (epoch >= trParams.Epochs) { //save training checkpoint state if (!string.IsNullOrEmpty(modelCheckPoint)) { trainer.SaveCheckpoint(modelCheckPoint); } //save training history if (!string.IsNullOrEmpty(historyPath)) { string header = $"{trainer.LossFunction().Name};{trainer.EvaluationFunction().Name};"; MLFactory.SaveTrainingHistory(m_trainingHistory, header, historyPath); } //save best or last trained model and send report last time before trainer completes var bestModelPath = saveBestModel(trParams, trainer.Model(), epoch, isMinimize); // if (progress != null) { progress(prData); } // trainResult.Iteration = epoch; trainResult.ProcessState = ProcessState.Compleated; trainResult.BestModelFile = bestModelPath; trainResult.TrainingHistoryFile = historyFile; break; } else { epoch++; } } //stop in case user request it if (token.IsCancellationRequested) { if (!string.IsNullOrEmpty(modelCheckPoint)) { trainer.SaveCheckpoint(modelCheckPoint); } //save training history if (!string.IsNullOrEmpty(historyPath)) { string header = $"{trainer.LossFunction().Name};{trainer.EvaluationFunction().Name};"; MLFactory.SaveTrainingHistory(m_trainingHistory, header, historyPath); } //sometime stopping training process can be before first epoch passed so make a incomplete progress if (prData == null)//check the progress of the training process { prData = progressTraining(trParams, trainer, network, miniBatchSource, epoch, progress, device); } //save best or last trained model and send report last time before trainer terminates var bestModelPath = saveBestModel(trParams, trainer.Model(), epoch, isMinimize); // if (progress != null) { progress(prData); } //setup training result trainResult.Iteration = prData.EpochCurrent; trainResult.ProcessState = ProcessState.Stopped; trainResult.BestModelFile = bestModelPath; trainResult.TrainingHistoryFile = historyFile; break; } } return(trainResult); } catch (Exception ex) { var ee = ex; throw; } finally { } }