/// <summary>
/// Return path for specified dataset
/// </summary>
/// <param name="dicPath"></param>
/// <param name="dsType"></param>
/// <returns></returns>
private static string GetDataPath(Dictionary <string, string> dicPath, DataSetType dsType)
{
var dataPaths = MLFactory.GetMLConfigComponentPaths(dicPath["paths"]);
//
if (dsType == DataSetType.Training)
{
//
var strPath = $"{dicPath["root"]}\\{dataPaths["Training"]}";
return(strPath);
}
else if (dsType == DataSetType.Validation)
{
var strPath = $"{dicPath["root"]}\\{dataPaths["Validation"]}";
return(strPath);
}
else if (dsType == DataSetType.Testing)
{
var strPath = $"{dicPath["root"]}\\{dataPaths["Test"]}";
return(strPath);
}
else
{
return(null);
}
}
/// <summary>
/// Evaluate the model against dataset sored in the dataset file, and exports the result in csv format for further analysis
/// </summary>
/// <param name="mlF"> ml factory object contains members needed to evaluation process</param>
/// <param name="mbs"> Minibatch source which provides helpers members needed for for evaluation</param>
/// <param name="strDataSetPath"> file of dataset</param>
/// <param name="modelPath"> models which will be evaluate</param>
/// <param name="resultExportPath"> result file in which the result will be exported</param>
/// <param name="device"> device for computation</param>
public static void EvaluateModel(string mlconfigPath, string bestTrainedModelPath, DeviceDescriptor device)
{
//Load ML model configuration file
var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath);
//add full path of model folder since model file doesn't contains any absolute path
dicMParameters.Add("root", MLFactory.GetMLConfigFolder(mlconfigPath));
//get model daa paths
var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]);
//parse feature variables
var projectValues = dicMParameters["training"].Split(MLFactory.m_cntkSpearator, StringSplitOptions.RemoveEmptyEntries);
var trainedModelRelativePath = MLFactory.GetParameterValue(projectValues, "TrainedModel");
//Minibatch type
var mbTypestr = MLFactory.GetParameterValue(projectValues, "Type");
MinibatchType mbType = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true);
//prepare MLFactory
var f = MLFactory.CreateMLFactory(dicMParameters);
//prepare data paths for mini-batch source
var strTrainPath = $"{dicMParameters["root"]}\\{dicPath["Training"]}";
var strValidPath = $"{dicMParameters["root"]}\\{dicPath["Validation"]}";
var strResult = $"{dicMParameters["root"]}\\{dicPath["Result"]}";
var bestModelFullPath = $"{dicMParameters["root"]}\\{bestTrainedModelPath}";
//decide what data to evaluate
var dataPath = strValidPath;
//load model
var model = Function.Load(bestModelFullPath, device);
//get data for evaluation by calling GetFullBatch
var minibatchData = MinibatchSourceEx.GetFullBatch(mbType, dataPath, f.StreamConfigurations.ToArray(), device);
//input map creation for model evaluation
var inputMap = new Dictionary <Variable, Value>();
foreach (var v in minibatchData)
{
var vv = model.Arguments.Where(x => x.Name == v.Key.m_name).FirstOrDefault();
var streamInfo = v.Key;
if (vv != null)
{
inputMap.Add(vv, minibatchData[streamInfo].data);
}
}
//output map
var predictedDataMap = new Dictionary <Variable, Value>();
foreach (var outp in model.Outputs)
{
predictedDataMap.Add(outp, null);
}
//model evaluation
model.Evaluate(inputMap, predictedDataMap, device);
//retrieve actual and predicted values from model
List <List <float> > actual = new List <List <float> >();
List <List <float> > predict = new List <List <float> >();
foreach (var output in model.Outputs)
{
//label stream
var labelStream = minibatchData.Keys.Where(x => x.m_name == output.Name).First();
//actual values
List <List <float> > av = MLValue.GetValues(output, minibatchData[labelStream].data);
//predicted values
List <List <float> > pv = MLValue.GetValues(output, predictedDataMap[output]);
for (int i = 0; i < av.Count; i++)
{
//actual
var act = av[i];
if (actual.Count <= i)
{
actual.Add(new List <float>());
}
actual[i].AddRange(act);
//prediction
var prd = pv[i];
if (predict.Count <= i)
{
predict.Add(new List <float>());
}
predict[i].AddRange(prd);
}
}
//export result
MLValue.ValueToFile(actual, predict, strResult);
//
Console.WriteLine(Environment.NewLine);
Console.WriteLine($"*******************Model Evaluation**************");
Console.WriteLine(Environment.NewLine);
Console.WriteLine($"Model Evaluation successfully exported result into file {strResult}!");
Console.WriteLine(Environment.NewLine);
}
public static EvaluationResult EvaluateModel(string mlconfigPath, DataSetType dsType, EvaluationType evType, ProcessDevice pdevice)
{
var er = new EvaluationResult();
er.Header = new List <string>();
//device definition
DeviceDescriptor device = MLFactory.GetDevice(pdevice);
//Load ML model configuration file
var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath);
//add full path of model folder since model file doesn't contains any absolute path
dicMParameters.Add("root", Project.GetMLConfigFolder(mlconfigPath));
// get model data paths
var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]);
var modelName = Project.GetParameterValue(dicMParameters["training"], "TrainedModel");
var nnModelPath = Path.Combine(dicMParameters["root"], modelName);
//check if model exists
if (!MLFactory.IsFileExist(nnModelPath))
{
return(er);
}
//check if dataset files exist
var dataPath = GetDataPath(dicMParameters, dsType);
if (!MLFactory.IsFileExist(dataPath))
{
//in case validation dataset is not defiend just export traininign dataset
if (dsType == DataSetType.Validation)
{
dataPath = GetDataPath(dicMParameters, DataSetType.Training);
}
if (!MLFactory.IsFileExist(dataPath))
{
return(er);
}
}
//get output classes in case the ml problem is classification
var strCls = dicMParameters.ContainsKey("metadata") ? dicMParameters["metadata"] : "";
er.OutputClasses = DataDescriptor.GetOutputClasses(strCls);
//Minibatch type
var mbTypestr = Project.GetParameterValue(dicMParameters["training"], "Type");
MinibatchType mbType = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true);
var mbSizetr = Project.GetParameterValue(dicMParameters["training"], "BatchSize");
var mf = MLFactory.CreateMLFactory(dicMParameters);
//perform evaluation
var evParams = new EvaluationParameters()
{
MinibatchSize = uint.Parse(mbSizetr),
MBSource = new MinibatchSourceEx(mbType, mf.StreamConfigurations.ToArray(), dataPath, null, MinibatchSource.FullDataSweep, false),
Input = mf.InputVariables,
Ouptut = mf.OutputVariables,
};
//evaluate model
if (evType == EvaluationType.FeaturesOnly)
{
if (!dicMParameters.ContainsKey("metadata"))
{
throw new Exception("The result cannot be exported to Excel, since no metadata is stored in mlconfig file.");
}
var desc = ParseRawDataSet(dicMParameters["metadata"]);
er.Header = generateHeader(desc);
er.DataSet = FeatureAndLabels(nnModelPath, dataPath, evParams, device);
return(er);
}
else if (evType == EvaluationType.Results)
{
//define header
er.Header.Add(evParams.Ouptut.First().Name + "_actual");
er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
var result = EvaluateFunction(nnModelPath, dataPath, evParams, device);
er.Actual = result.actual.ToList();
er.Predicted = result.predicted.ToList();
return(er);
}
else if (evType == EvaluationType.ResultyExtended)
{
//define header
er.Header.Add(evParams.Ouptut.First().Name + "_actual");
er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
er.Actual = new List <float>();
er.Predicted = new List <float>();
er.ActualEx = new List <List <float> >();
er.PredictedEx = new List <List <float> >();
//
var resultEx = EvaluateFunctionEx(nnModelPath, dataPath, evParams, device);
for (int i = 0; i < resultEx.actual.Count(); i++)
{
var res1 = MLValue.GetResult(resultEx.actual[i]);
er.Actual.Add(res1);
var res2 = MLValue.GetResult(resultEx.predicted[i]);
er.Predicted.Add(res2);
}
er.ActualEx = resultEx.actual;
er.PredictedEx = resultEx.predicted;
return(er);
}
else
{
throw new Exception("Unknown evaluation type!");
}
}