/// <summary>
/// Test cntk model stored at 'modelPath' against set of vector of values
/// </summary>
/// <param name="modelPath"></param>
/// <param name="vector"></param>
/// <param name="device"></param>
/// <returns></returns>
public static List <float> TestModel(string modelPath, float[][] vector, DeviceDescriptor device)
{
try
{
var results = new List <float>();
//
FileInfo fi = new FileInfo(modelPath);
if (!fi.Exists)
{
throw new Exception($"The '{fi.FullName}' does not exist. Make sure the model is places at this location.");
}
//load the model from disk
var model = Function.Load(fi.FullName, device);
//input map creation for model evaluation
var inputMap = new Dictionary <Variable, Value>();
//output map
var predictedDataMap = new Dictionary <Variable, Value>();
foreach (var vec in vector)
{
inputMap.Clear();
predictedDataMap.Clear();
int columnSpan = 0;
foreach (var var in model.Arguments)
{
var dim = var.Shape.Dimensions.First();
Value values = Value.CreateBatch <float>(var.Shape, vec.Skip(columnSpan).Take(dim), device);
inputMap.Add(var, values);
columnSpan += dim;
}
//output map
foreach (var outp in model.Outputs)
{
predictedDataMap.Add(outp, null);
}
//evaluate the model
model.Evaluate(inputMap, predictedDataMap, device);
//extract the result as one hot vector
var outputData = predictedDataMap[model.Output].GetDenseData <float>(model.Output);
//extract result
var retVal = MLValue.GetResult(outputData.First());
results.Add(retVal);
}
return(results);
}
catch (Exception)
{
throw;
}
}
public static (IEnumerable <float> actual, IEnumerable <float> predicted) EvaluateFunction(Function fun, EvaluationParameters evParam, DeviceDescriptor device)
{
try
{
//declare return vars
List <float> actualLst = new List <float>();
List <float> predictedLst = new List <float>();
var result = EvaluateFunctionEx(fun, evParam, device);
for (int i = 0; i < result.actual.Count(); i++)
{
var res1 = MLValue.GetResult(result.actual[i]);
actualLst.Add(res1);
var res2 = MLValue.GetResult(result.predicted[i]);
predictedLst.Add(res2);
}
return(actualLst, predictedLst);
}
catch (Exception)
{
throw;
}
}
/// <summary>
/// Returns part of mldataset with features labels columns this is needed in case Excel export is performed.
/// </summary>
/// <param name="fun"></param>
/// <param name="evParam"></param>
/// <param name="device"></param>
/// <returns></returns>
public static Dictionary <string, List <List <float> > > FeaturesAndLabels(Function fun, EvaluationParameters evParam, DeviceDescriptor device)
{
try
{
//declare return vars
var featDic = new Dictionary <string, List <List <float> > >();
while (true)
{
//get one minibatch of data for training
var mbData = evParam.MBSource.GetNextMinibatch(evParam.MinibatchSize, device);
var mdDataEx = MinibatchSourceEx.ToMinibatchValueData(mbData, evParam.Input.Union(evParam.Ouptut).ToList());
var inMap = new Dictionary <Variable, Value>();
//input
var vars = evParam.Input;
for (int i = 0; i < vars.Count() /*mdDataEx.Count*/; i++)
{
var vv = vars.ElementAt(i);
var d = mdDataEx.Where(x => x.Key.Name.Equals(vv.Name)).FirstOrDefault();
//
var fv = MLValue.GetValues(d.Key, d.Value);
if (featDic.ContainsKey(d.Key.Name))
{
featDic[d.Key.Name].AddRange(fv);
}
else
{
featDic.Add(d.Key.Name, fv);
}
}
//output
var varso = evParam.Ouptut;
for (int i = 0; i < varso.Count() /*mdDataEx.Count*/; i++)
{
var vv = varso.ElementAt(i);
var d = mdDataEx.Where(x => x.Key.Name.Equals(vv.Name)).FirstOrDefault();
//
var fv = MLValue.GetValues(d.Key, d.Value);
if (vv.Shape.Dimensions.Last() == 1)
{
var value = fv.Select(l => new List <float>()
{
l.First()
}).ToList();
if (featDic.ContainsKey(d.Key.Name))
{
featDic[d.Key.Name].AddRange(value);
}
else
{
featDic.Add(d.Key.Name, value);
}
}
else
{
var value = fv.Select(l => new List <float>()
{
l.IndexOf(l.Max())
}).ToList();
if (featDic.ContainsKey(d.Key.Name))
{
featDic[d.Key.Name].AddRange(value);
}
else
{
featDic.Add(d.Key.Name, value);
}
}
}
// check if sweep end reached
if (mbData.Any(x => x.Value.sweepEnd))
{
break;
}
}
return(featDic);
}
catch (Exception)
{
throw;
}
}
/// <summary>
/// Test cntk model stored at 'modelPath' against array of image paths
/// </summary>
/// <param name="modelPath"></param>
/// <param name="vector"></param>
/// <param name="device"></param>
/// <returns></returns>
public static List <int> TestModel(string modelPath, string[] imagePaths, DeviceDescriptor device)
{
try
{
//
FileInfo fi = new FileInfo(modelPath);
if (!fi.Exists)
{
throw new Exception($"The '{fi.FullName}' does not exist. Make sure the model is places at this location.");
}
//load the model from disk
var model = Function.Load(fi.FullName, device);
//get input feature
var features = model.Arguments.ToList();
var labels = model.Outputs.ToList();
var stremsConfig = MLFactory.CreateStreamConfiguration(features, labels);
var mapFile = "testMapFile";
File.WriteAllLines(mapFile, imagePaths.Select(x => $"{x}\t0"));
var testMB = new MinibatchSourceEx(MinibatchType.Image, stremsConfig.ToArray(), features, labels, mapFile, null, 30, false, 0);
//
var vars = features.Union(labels).ToList();
var retVal = new List <int>();
var mbSize = imagePaths.Count();
if (mbSize > 30)
{
mbSize = 30;
}
while (true)
{
bool isSweepEnd = false;
var inputMap = testMB.GetNextMinibatch((uint)mbSize, ref isSweepEnd, vars, device);
//prepare data for trainer
//var inputMap = new Dictionary<Variable, Value>();
//inputMap.Add(features.First(), nextMB.Where(x => x.Key.m_name.Equals(features.First().Name)).Select(x => x.Value.data).FirstOrDefault());
var outputMap = new Dictionary <Variable, Value>();
outputMap.Add(labels.First(), null);
//evaluate model
model.Evaluate(inputMap, outputMap, device);
var result = outputMap[labels.First()].GetDenseData <float>(labels.First());
//extract result
foreach (var r in result)
{
var l = MLValue.GetResult(r);
retVal.Add((int)l);
}
if (/*nextMB.Any(x => x.Value.sweepEnd)*/ isSweepEnd)
{
break;
}
}
return(retVal);
}
catch (Exception)
{
throw;
}
}
/// <summary>
/// Evaluate model defined in the mlconfig file
/// </summary>
/// <param name="mlconfigPath"></param>
/// <param name="device"></param>
/// <returns></returns>
public static async Task <EvaluationResult> EvaluateMLConfig(string mlconfigPath, DeviceDescriptor device, DataSetType dsType, EvaluationType evType)
{
try
{
//define eval result
var er = new EvaluationResult();
er.OutputClasses = new List <string>()
{
""
};
er.Actual = new List <float>();
er.Predicted = new List <float>();
er.Header = new List <string>();
//Load ML configuration file
var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath);
//add full path of model folder since model file doesn't contains any absolute path
dicMParameters.Add("root", MLFactory.GetMLConfigFolder(mlconfigPath));
// get model data paths
var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]);
//parse feature variables
var projectValues = dicMParameters["training"].Split(MLFactory.m_cntkSpearator, StringSplitOptions.RemoveEmptyEntries);
var modelName = MLFactory.GetParameterValue(projectValues, "TrainedModel");
var nnModelPath = Path.Combine(dicMParameters["root"], modelName);
//check if model exists
if (!MLFactory.IsFileExist(nnModelPath))
{
return(er);
}
//
var dataset = MLFactory.GetDataPath(dicMParameters, dsType);
if (string.IsNullOrEmpty(dataset) || string.IsNullOrEmpty(dataset) || dataset == " ")
{
if (dsType == DataSetType.Testing)
{
dataset = MLFactory.GetDataPath(dicMParameters, DataSetType.Validation);
}
if (string.IsNullOrEmpty(dataset) || string.IsNullOrEmpty(dataset) || dataset == " ")
{
return(er);
}
}
//get output classes in case the ml problem is classification
var strCls = dicMParameters.ContainsKey("metadata") ? dicMParameters["metadata"] : "";
var oc = MLFactory.GetOutputClasses(strCls);
if (oc != null)
{
er.OutputClasses = oc;
}
//MInibatch
var mbTypestr = MLFactory.GetParameterValue(projectValues, "Type");
MinibatchType mbType = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true);
var mbSizetr = MLFactory.GetParameterValue(projectValues, "BatchSize");
var mf = MLFactory.CreateMLFactory(dicMParameters);
//perform evaluation
var evParams = new EvaluationParameters()
{
MinibatchSize = uint.Parse(mbSizetr),
MBSource = new MinibatchSourceEx(mbType, mf.StreamConfigurations.ToArray(), mf.InputVariables, mf.OutputVariables, dataset, null, MinibatchSource.FullDataSweep, false, 0),
Input = mf.InputVariables,
Ouptut = mf.OutputVariables,
};
//evaluate model
if (evType == EvaluationType.FeaturesOnly)
{
if (!dicMParameters.ContainsKey("metadata"))
{
throw new Exception("The result cannot be exported to Excel, since no metadata is stored in mlconfig file.");
}
var desc = MLFactory.ParseRawDataSet(dicMParameters["metadata"]);
er.Header = MLFactory.GenerateHeader(desc);
var fun = Function.Load(nnModelPath, device);
//
er.DataSet = await Task.Run(() => MLEvaluator.FeaturesAndLabels(fun, evParams, device));
return(er);
}
else if (evType == EvaluationType.Results)
{
//define header
er.Header.Add(evParams.Ouptut.First().Name + "_actual");
er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
var fun = Function.Load(nnModelPath, device);
//
var result = await Task.Run(() => MLEvaluator.EvaluateFunction(fun, evParams, device));
er.Actual = result.actual.ToList();
er.Predicted = result.predicted.ToList();
if (er.OutputClasses.Count < 2 && evParams.Ouptut.First().Shape.Dimensions.Last() > 1)
{
var result1 = await Task.Run(() => MLEvaluator.EvaluateFunctionEx(fun, evParams, device));
er.ActualEx = result1.actual;
er.PredictedEx = result1.predicted;
}
return(er);
}
else if (evType == EvaluationType.ResultExtended)
{
//define header
er.Header.Add(evParams.Ouptut.First().Name + "_actual");
er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
er.Actual = new List <float>();
er.Predicted = new List <float>();
er.ActualEx = new List <List <float> >();
er.PredictedEx = new List <List <float> >();
//
var fun = Function.Load(nnModelPath, device);
var resultEx = await Task.Run(() => MLEvaluator.EvaluateFunctionEx(fun, evParams, device));
//var resultEx = EvaluateFunctionEx(nnModelPath, dataPath, evParams, device);
for (int i = 0; i < resultEx.actual.Count(); i++)
{
var res1 = MLValue.GetResult(resultEx.actual[i]);
er.Actual.Add(res1);
var res2 = MLValue.GetResult(resultEx.predicted[i]);
er.Predicted.Add(res2);
}
er.ActualEx = resultEx.actual;
er.PredictedEx = resultEx.predicted;
return(er);
}
else
{
throw new Exception("Unknown evaluation type!");
}
}
catch (Exception)
{
throw;
}
}
