public static Tuple <bool, bool, bool> GetDataSetAviability(string mlConfigPath)
{
try
{
//Load ML model configuration file
var dicMParameters = MLFactory.LoadMLConfiguration(mlConfigPath);
//add full path of model folder since model file doesn't contains any absolute path
dicMParameters.Add("root", MLFactory.GetMLConfigFolder(mlConfigPath));
//get model daa paths
var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]);
var training = dicPath["Training"] != " ";
var validation = dicPath["Validation"] != " ";
var test = dicPath["Test"] != " ";
return(new Tuple <bool, bool, bool>(training, validation, test));
}
catch (Exception)
{
throw;
}
}
public static string GetMLConfigId(string mlconfigPath)
{
try
{
//Load ML configuration file
var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath);
if (dicMParameters.ContainsKey("configid"))
{
return(dicMParameters["configid"].Trim(' '));
}
else
{
return(null);
}
}
catch (Exception)
{
throw;
}
}
/// <summary>
///
/// </summary>
/// <param name="trData"></param>
/// <param name="bestModelFile"></param>
/// <returns></returns>
public static string ReplaceBestModel(TrainingParameters trData, string mlconfigPath, string bestModelFile)
{
try
{
if (!string.IsNullOrEmpty(trData.LastBestModel) && trData.LastBestModel != " ")
{
var mlconfigFolder = MLFactory.GetMLConfigFolder(mlconfigPath);
var oldPath = Path.Combine(mlconfigFolder, trData.LastBestModel);
//delete history
if (File.Exists(oldPath))
{
File.Delete(oldPath);
}
}
//set new value after delete old model
trData.LastBestModel = bestModelFile;
return(bestModelFile);
}
catch (Exception)
{
throw;
}
}
/// <summary>
/// Parses the mlconfig content and return dictionary of the model components
/// </summary>
/// <param name="pathsString"></param>
/// <returns></returns>
public static Dictionary <string, string> GetMLConfigComponentPaths(string pathsString)
{
var dicValues = new Dictionary <string, string>();
//parse feature variables
var pathsValues = pathsString.Split(m_cntkSpearator, StringSplitOptions.RemoveEmptyEntries);
//training path
var val = MLFactory.GetParameterValue(pathsValues, "Training");
if (string.IsNullOrEmpty(val))
{
throw new Exception("One of model paths (Training) is not defined!");
}
dicValues.Add("Training", val);
val = MLFactory.GetParameterValue(pathsValues, "Validation");
if (string.IsNullOrEmpty(val))
{
dicValues.Add("Validation", " ");
}
else
{
dicValues.Add("Validation", val);
}
val = MLFactory.GetParameterValue(pathsValues, "Test");
if (string.IsNullOrEmpty(val))
{
dicValues.Add("Test", " ");
}
else
{
dicValues.Add("Test", val);
}
val = MLFactory.GetParameterValue(pathsValues, "TempModels");
if (string.IsNullOrEmpty(val))
{
throw new Exception("One of model paths (TempModels) is not defined!");
}
dicValues.Add("TempModels", val);
val = MLFactory.GetParameterValue(pathsValues, "Models");
if (string.IsNullOrEmpty(val))
{
throw new Exception("One of model paths (Models) is not defined!");
}
dicValues.Add("Models", val);
val = MLFactory.GetParameterValue(pathsValues, "Result");
if (string.IsNullOrEmpty(val))
{
throw new Exception("One of model paths (Result) is not defined!");
}
dicValues.Add("Result", val);
val = MLFactory.GetParameterValue(pathsValues, "Logs");
if (string.IsNullOrEmpty(val))
{
throw new Exception("One of model paths (Log) is not defined!");
}
dicValues.Add("Logs", val);
return(dicValues);
}
/// <summary>
/// Test cntk model stored at 'modelPath' against array of image paths
/// </summary>
/// <param name="modelPath"></param>
/// <param name="vector"></param>
/// <param name="device"></param>
/// <returns></returns>
public static List <int> TestModel(string modelPath, string[] imagePaths, DeviceDescriptor device)
{
try
{
//
FileInfo fi = new FileInfo(modelPath);
if (!fi.Exists)
{
throw new Exception($"The '{fi.FullName}' does not exist. Make sure the model is places at this location.");
}
//load the model from disk
var model = Function.Load(fi.FullName, device);
//get input feature
var features = model.Arguments.ToList();
var labels = model.Outputs.ToList();
var stremsConfig = MLFactory.CreateStreamConfiguration(features, labels);
var mapFile = "testMapFile";
File.WriteAllLines(mapFile, imagePaths.Select(x => $"{x}\t0"));
var testMB = new MinibatchSourceEx(MinibatchType.Image, stremsConfig.ToArray(), features, labels, mapFile, null, 30, false, 0);
//
var vars = features.Union(labels).ToList();
var retVal = new List <int>();
var mbSize = imagePaths.Count();
if (mbSize > 30)
{
mbSize = 30;
}
while (true)
{
bool isSweepEnd = false;
var inputMap = testMB.GetNextMinibatch((uint)mbSize, ref isSweepEnd, vars, device);
//prepare data for trainer
//var inputMap = new Dictionary<Variable, Value>();
//inputMap.Add(features.First(), nextMB.Where(x => x.Key.m_name.Equals(features.First().Name)).Select(x => x.Value.data).FirstOrDefault());
var outputMap = new Dictionary <Variable, Value>();
outputMap.Add(labels.First(), null);
//evaluate model
model.Evaluate(inputMap, outputMap, device);
var result = outputMap[labels.First()].GetDenseData <float>(labels.First());
//extract result
foreach (var r in result)
{
var l = MLValue.GetResult(r);
retVal.Add((int)l);
}
if (/*nextMB.Any(x => x.Value.sweepEnd)*/ isSweepEnd)
{
break;
}
}
return(retVal);
}
catch (Exception)
{
throw;
}
}
/// <summary>
/// Evaluate model defined in the mlconfig file
/// </summary>
/// <param name="mlconfigPath"></param>
/// <param name="device"></param>
/// <returns></returns>
public static async Task <EvaluationResult> EvaluateMLConfig(string mlconfigPath, DeviceDescriptor device, DataSetType dsType, EvaluationType evType)
{
try
{
//define eval result
var er = new EvaluationResult();
er.OutputClasses = new List <string>()
{
""
};
er.Actual = new List <float>();
er.Predicted = new List <float>();
er.Header = new List <string>();
//Load ML configuration file
var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath);
//add full path of model folder since model file doesn't contains any absolute path
dicMParameters.Add("root", MLFactory.GetMLConfigFolder(mlconfigPath));
// get model data paths
var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]);
//parse feature variables
var projectValues = dicMParameters["training"].Split(MLFactory.m_cntkSpearator, StringSplitOptions.RemoveEmptyEntries);
var modelName = MLFactory.GetParameterValue(projectValues, "TrainedModel");
var nnModelPath = Path.Combine(dicMParameters["root"], modelName);
//check if model exists
if (!MLFactory.IsFileExist(nnModelPath))
{
return(er);
}
//
var dataset = MLFactory.GetDataPath(dicMParameters, dsType);
if (string.IsNullOrEmpty(dataset) || string.IsNullOrEmpty(dataset) || dataset == " ")
{
if (dsType == DataSetType.Testing)
{
dataset = MLFactory.GetDataPath(dicMParameters, DataSetType.Validation);
}
if (string.IsNullOrEmpty(dataset) || string.IsNullOrEmpty(dataset) || dataset == " ")
{
return(er);
}
}
//get output classes in case the ml problem is classification
var strCls = dicMParameters.ContainsKey("metadata") ? dicMParameters["metadata"] : "";
var oc = MLFactory.GetOutputClasses(strCls);
if (oc != null)
{
er.OutputClasses = oc;
}
//MInibatch
var mbTypestr = MLFactory.GetParameterValue(projectValues, "Type");
MinibatchType mbType = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true);
var mbSizetr = MLFactory.GetParameterValue(projectValues, "BatchSize");
var mf = MLFactory.CreateMLFactory(dicMParameters);
//perform evaluation
var evParams = new EvaluationParameters()
{
MinibatchSize = uint.Parse(mbSizetr),
MBSource = new MinibatchSourceEx(mbType, mf.StreamConfigurations.ToArray(), mf.InputVariables, mf.OutputVariables, dataset, null, MinibatchSource.FullDataSweep, false, 0),
Input = mf.InputVariables,
Ouptut = mf.OutputVariables,
};
//evaluate model
if (evType == EvaluationType.FeaturesOnly)
{
if (!dicMParameters.ContainsKey("metadata"))
{
throw new Exception("The result cannot be exported to Excel, since no metadata is stored in mlconfig file.");
}
var desc = MLFactory.ParseRawDataSet(dicMParameters["metadata"]);
er.Header = MLFactory.GenerateHeader(desc);
var fun = Function.Load(nnModelPath, device);
//
er.DataSet = await Task.Run(() => MLEvaluator.FeaturesAndLabels(fun, evParams, device));
return(er);
}
else if (evType == EvaluationType.Results)
{
//define header
er.Header.Add(evParams.Ouptut.First().Name + "_actual");
er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
var fun = Function.Load(nnModelPath, device);
//
var result = await Task.Run(() => MLEvaluator.EvaluateFunction(fun, evParams, device));
er.Actual = result.actual.ToList();
er.Predicted = result.predicted.ToList();
if (er.OutputClasses.Count < 2 && evParams.Ouptut.First().Shape.Dimensions.Last() > 1)
{
var result1 = await Task.Run(() => MLEvaluator.EvaluateFunctionEx(fun, evParams, device));
er.ActualEx = result1.actual;
er.PredictedEx = result1.predicted;
}
return(er);
}
else if (evType == EvaluationType.ResultExtended)
{
//define header
er.Header.Add(evParams.Ouptut.First().Name + "_actual");
er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
er.Actual = new List <float>();
er.Predicted = new List <float>();
er.ActualEx = new List <List <float> >();
er.PredictedEx = new List <List <float> >();
//
var fun = Function.Load(nnModelPath, device);
var resultEx = await Task.Run(() => MLEvaluator.EvaluateFunctionEx(fun, evParams, device));
//var resultEx = EvaluateFunctionEx(nnModelPath, dataPath, evParams, device);
for (int i = 0; i < resultEx.actual.Count(); i++)
{
var res1 = MLValue.GetResult(resultEx.actual[i]);
er.Actual.Add(res1);
var res2 = MLValue.GetResult(resultEx.predicted[i]);
er.Predicted.Add(res2);
}
er.ActualEx = resultEx.actual;
er.PredictedEx = resultEx.predicted;
return(er);
}
else
{
throw new Exception("Unknown evaluation type!");
}
}
catch (Exception)
{
throw;
}
}
/// <summary>
/// Main method for training
/// </summary>
/// <param name="trainer"></param>
/// <param name="network"></param>
/// <param name="trParams"></param>
/// <param name="miniBatchSource"></param>
/// <param name="device"></param>
/// <param name="token"></param>
/// <param name="progress"></param>
/// <param name="modelCheckPoint"></param>
/// <returns></returns>
public override TrainResult Train(Trainer trainer, Function network, TrainingParameters trParams,
MinibatchSourceEx miniBatchSource, DeviceDescriptor device, CancellationToken token, TrainingProgress progress, string modelCheckPoint, string historyPath)
{
try
{
//create trainer result.
// the variable indicate how training process is ended
// completed, stopped, crashed,
var trainResult = new TrainResult();
var historyFile = "";
//create training process evaluation collection
//for each iteration it is stored evaluationValue for training, and validation set with the model
m_ModelEvaluations = new List <Tuple <double, double, string> >();
//check what is the optimization (Minimization (error) or maximization (accuracy))
bool isMinimize = StatMetrics.IsGoalToMinimize(trainer.EvaluationFunction());
//setup first iteration
if (m_trainingHistory == null)
{
m_trainingHistory = new List <Tuple <int, float, float, float, float> >();
}
//in case of continuation of training iteration must start with the last of path previous training process
int epoch = (m_trainingHistory.Count > 0)? m_trainingHistory.Last().Item1 + 1:1;
//define progressData
ProgressData prData = null;
//define helper variable collection
var vars = InputVariables.Union(OutputVariables).ToList();
//training process
while (true)
{
//get next mini batch data
//var args = miniBatchSource.GetNextMinibatch(trParams.BatchSize, device);
//var isSweepEnd = args.Any(a => a.Value.sweepEnd);
////prepare the data for trainer
//var arguments = MinibatchSourceEx.ToMinibatchValueData(args, vars);
//
var isSweepEnd = false;
var arguments = miniBatchSource.GetNextMinibatch(trParams.BatchSize, ref isSweepEnd, vars, device);
trainer.TrainMinibatch(arguments, isSweepEnd, device);
//make progress
if (isSweepEnd)
{
//check the progress of the training process
prData = progressTraining(trParams, trainer, network, miniBatchSource, epoch, progress, device);
//check if training process ends
if (epoch >= trParams.Epochs)
{
//save training checkpoint state
if (!string.IsNullOrEmpty(modelCheckPoint))
{
trainer.SaveCheckpoint(modelCheckPoint);
}
//save training history
if (!string.IsNullOrEmpty(historyPath))
{
string header = $"{trainer.LossFunction().Name};{trainer.EvaluationFunction().Name};";
MLFactory.SaveTrainingHistory(m_trainingHistory, header, historyPath);
}
//save best or last trained model and send report last time before trainer completes
var bestModelPath = saveBestModel(trParams, trainer.Model(), epoch, isMinimize);
//
if (progress != null)
{
progress(prData);
}
//
trainResult.Iteration = epoch;
trainResult.ProcessState = ProcessState.Compleated;
trainResult.BestModelFile = bestModelPath;
trainResult.TrainingHistoryFile = historyFile;
break;
}
else
{
epoch++;
}
}
//stop in case user request it
if (token.IsCancellationRequested)
{
if (!string.IsNullOrEmpty(modelCheckPoint))
{
trainer.SaveCheckpoint(modelCheckPoint);
}
//save training history
if (!string.IsNullOrEmpty(historyPath))
{
string header = $"{trainer.LossFunction().Name};{trainer.EvaluationFunction().Name};";
MLFactory.SaveTrainingHistory(m_trainingHistory, header, historyPath);
}
//sometime stopping training process can be before first epoch passed so make a incomplete progress
if (prData == null)//check the progress of the training process
{
prData = progressTraining(trParams, trainer, network, miniBatchSource, epoch, progress, device);
}
//save best or last trained model and send report last time before trainer terminates
var bestModelPath = saveBestModel(trParams, trainer.Model(), epoch, isMinimize);
//
if (progress != null)
{
progress(prData);
}
//setup training result
trainResult.Iteration = prData.EpochCurrent;
trainResult.ProcessState = ProcessState.Stopped;
trainResult.BestModelFile = bestModelPath;
trainResult.TrainingHistoryFile = historyFile;
break;
}
}
return(trainResult);
}
catch (Exception ex)
{
var ee = ex;
throw;
}
finally
{
}
}
