/// <summary> /// Test cntk model stored at 'modelPath' against set of vector of values /// </summary> /// <param name="modelPath"></param> /// <param name="vector"></param> /// <param name="device"></param> /// <returns></returns> public static List <float> TestModel(string modelPath, float[][] vector, DeviceDescriptor device) { try { var results = new List <float>(); // FileInfo fi = new FileInfo(modelPath); if (!fi.Exists) { throw new Exception($"The '{fi.FullName}' does not exist. Make sure the model is places at this location."); } //load the model from disk var model = Function.Load(fi.FullName, device); //input map creation for model evaluation var inputMap = new Dictionary <Variable, Value>(); //output map var predictedDataMap = new Dictionary <Variable, Value>(); foreach (var vec in vector) { inputMap.Clear(); predictedDataMap.Clear(); int columnSpan = 0; foreach (var var in model.Arguments) { var dim = var.Shape.Dimensions.First(); Value values = Value.CreateBatch <float>(var.Shape, vec.Skip(columnSpan).Take(dim), device); inputMap.Add(var, values); columnSpan += dim; } //output map foreach (var outp in model.Outputs) { predictedDataMap.Add(outp, null); } //evaluate the model model.Evaluate(inputMap, predictedDataMap, device); //extract the result as one hot vector var outputData = predictedDataMap[model.Output].GetDenseData <float>(model.Output); //extract result var retVal = MLValue.GetResult(outputData.First()); results.Add(retVal); } return(results); } catch (Exception) { throw; } }
public static (IEnumerable <float> actual, IEnumerable <float> predicted) EvaluateFunction(Function fun, EvaluationParameters evParam, DeviceDescriptor device) { try { //declare return vars List <float> actualLst = new List <float>(); List <float> predictedLst = new List <float>(); var result = EvaluateFunctionEx(fun, evParam, device); for (int i = 0; i < result.actual.Count(); i++) { var res1 = MLValue.GetResult(result.actual[i]); actualLst.Add(res1); var res2 = MLValue.GetResult(result.predicted[i]); predictedLst.Add(res2); } return(actualLst, predictedLst); } catch (Exception) { throw; } }
/// <summary> /// Returns part of mldataset with features labels columns this is needed in case Excel export is performed. /// </summary> /// <param name="fun"></param> /// <param name="evParam"></param> /// <param name="device"></param> /// <returns></returns> public static Dictionary <string, List <List <float> > > FeaturesAndLabels(Function fun, EvaluationParameters evParam, DeviceDescriptor device) { try { //declare return vars var featDic = new Dictionary <string, List <List <float> > >(); while (true) { //get one minibatch of data for training var mbData = evParam.MBSource.GetNextMinibatch(evParam.MinibatchSize, device); var mdDataEx = MinibatchSourceEx.ToMinibatchValueData(mbData, evParam.Input.Union(evParam.Ouptut).ToList()); var inMap = new Dictionary <Variable, Value>(); //input var vars = evParam.Input; for (int i = 0; i < vars.Count() /*mdDataEx.Count*/; i++) { var vv = vars.ElementAt(i); var d = mdDataEx.Where(x => x.Key.Name.Equals(vv.Name)).FirstOrDefault(); // var fv = MLValue.GetValues(d.Key, d.Value); if (featDic.ContainsKey(d.Key.Name)) { featDic[d.Key.Name].AddRange(fv); } else { featDic.Add(d.Key.Name, fv); } } //output var varso = evParam.Ouptut; for (int i = 0; i < varso.Count() /*mdDataEx.Count*/; i++) { var vv = varso.ElementAt(i); var d = mdDataEx.Where(x => x.Key.Name.Equals(vv.Name)).FirstOrDefault(); // var fv = MLValue.GetValues(d.Key, d.Value); if (vv.Shape.Dimensions.Last() == 1) { var value = fv.Select(l => new List <float>() { l.First() }).ToList(); if (featDic.ContainsKey(d.Key.Name)) { featDic[d.Key.Name].AddRange(value); } else { featDic.Add(d.Key.Name, value); } } else { var value = fv.Select(l => new List <float>() { l.IndexOf(l.Max()) }).ToList(); if (featDic.ContainsKey(d.Key.Name)) { featDic[d.Key.Name].AddRange(value); } else { featDic.Add(d.Key.Name, value); } } } // check if sweep end reached if (mbData.Any(x => x.Value.sweepEnd)) { break; } } return(featDic); } catch (Exception) { throw; } }
/// <summary> /// Test cntk model stored at 'modelPath' against array of image paths /// </summary> /// <param name="modelPath"></param> /// <param name="vector"></param> /// <param name="device"></param> /// <returns></returns> public static List <int> TestModel(string modelPath, string[] imagePaths, DeviceDescriptor device) { try { // FileInfo fi = new FileInfo(modelPath); if (!fi.Exists) { throw new Exception($"The '{fi.FullName}' does not exist. Make sure the model is places at this location."); } //load the model from disk var model = Function.Load(fi.FullName, device); //get input feature var features = model.Arguments.ToList(); var labels = model.Outputs.ToList(); var stremsConfig = MLFactory.CreateStreamConfiguration(features, labels); var mapFile = "testMapFile"; File.WriteAllLines(mapFile, imagePaths.Select(x => $"{x}\t0")); var testMB = new MinibatchSourceEx(MinibatchType.Image, stremsConfig.ToArray(), features, labels, mapFile, null, 30, false, 0); // var vars = features.Union(labels).ToList(); var retVal = new List <int>(); var mbSize = imagePaths.Count(); if (mbSize > 30) { mbSize = 30; } while (true) { bool isSweepEnd = false; var inputMap = testMB.GetNextMinibatch((uint)mbSize, ref isSweepEnd, vars, device); //prepare data for trainer //var inputMap = new Dictionary<Variable, Value>(); //inputMap.Add(features.First(), nextMB.Where(x => x.Key.m_name.Equals(features.First().Name)).Select(x => x.Value.data).FirstOrDefault()); var outputMap = new Dictionary <Variable, Value>(); outputMap.Add(labels.First(), null); //evaluate model model.Evaluate(inputMap, outputMap, device); var result = outputMap[labels.First()].GetDenseData <float>(labels.First()); //extract result foreach (var r in result) { var l = MLValue.GetResult(r); retVal.Add((int)l); } if (/*nextMB.Any(x => x.Value.sweepEnd)*/ isSweepEnd) { break; } } return(retVal); } catch (Exception) { throw; } }
/// <summary> /// Evaluate model defined in the mlconfig file /// </summary> /// <param name="mlconfigPath"></param> /// <param name="device"></param> /// <returns></returns> public static async Task <EvaluationResult> EvaluateMLConfig(string mlconfigPath, DeviceDescriptor device, DataSetType dsType, EvaluationType evType) { try { //define eval result var er = new EvaluationResult(); er.OutputClasses = new List <string>() { "" }; er.Actual = new List <float>(); er.Predicted = new List <float>(); er.Header = new List <string>(); //Load ML configuration file var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath); //add full path of model folder since model file doesn't contains any absolute path dicMParameters.Add("root", MLFactory.GetMLConfigFolder(mlconfigPath)); // get model data paths var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]); //parse feature variables var projectValues = dicMParameters["training"].Split(MLFactory.m_cntkSpearator, StringSplitOptions.RemoveEmptyEntries); var modelName = MLFactory.GetParameterValue(projectValues, "TrainedModel"); var nnModelPath = Path.Combine(dicMParameters["root"], modelName); //check if model exists if (!MLFactory.IsFileExist(nnModelPath)) { return(er); } // var dataset = MLFactory.GetDataPath(dicMParameters, dsType); if (string.IsNullOrEmpty(dataset) || string.IsNullOrEmpty(dataset) || dataset == " ") { if (dsType == DataSetType.Testing) { dataset = MLFactory.GetDataPath(dicMParameters, DataSetType.Validation); } if (string.IsNullOrEmpty(dataset) || string.IsNullOrEmpty(dataset) || dataset == " ") { return(er); } } //get output classes in case the ml problem is classification var strCls = dicMParameters.ContainsKey("metadata") ? dicMParameters["metadata"] : ""; var oc = MLFactory.GetOutputClasses(strCls); if (oc != null) { er.OutputClasses = oc; } //MInibatch var mbTypestr = MLFactory.GetParameterValue(projectValues, "Type"); MinibatchType mbType = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true); var mbSizetr = MLFactory.GetParameterValue(projectValues, "BatchSize"); var mf = MLFactory.CreateMLFactory(dicMParameters); //perform evaluation var evParams = new EvaluationParameters() { MinibatchSize = uint.Parse(mbSizetr), MBSource = new MinibatchSourceEx(mbType, mf.StreamConfigurations.ToArray(), mf.InputVariables, mf.OutputVariables, dataset, null, MinibatchSource.FullDataSweep, false, 0), Input = mf.InputVariables, Ouptut = mf.OutputVariables, }; //evaluate model if (evType == EvaluationType.FeaturesOnly) { if (!dicMParameters.ContainsKey("metadata")) { throw new Exception("The result cannot be exported to Excel, since no metadata is stored in mlconfig file."); } var desc = MLFactory.ParseRawDataSet(dicMParameters["metadata"]); er.Header = MLFactory.GenerateHeader(desc); var fun = Function.Load(nnModelPath, device); // er.DataSet = await Task.Run(() => MLEvaluator.FeaturesAndLabels(fun, evParams, device)); return(er); } else if (evType == EvaluationType.Results) { //define header er.Header.Add(evParams.Ouptut.First().Name + "_actual"); er.Header.Add(evParams.Ouptut.First().Name + "_predicted"); var fun = Function.Load(nnModelPath, device); // var result = await Task.Run(() => MLEvaluator.EvaluateFunction(fun, evParams, device)); er.Actual = result.actual.ToList(); er.Predicted = result.predicted.ToList(); if (er.OutputClasses.Count < 2 && evParams.Ouptut.First().Shape.Dimensions.Last() > 1) { var result1 = await Task.Run(() => MLEvaluator.EvaluateFunctionEx(fun, evParams, device)); er.ActualEx = result1.actual; er.PredictedEx = result1.predicted; } return(er); } else if (evType == EvaluationType.ResultExtended) { //define header er.Header.Add(evParams.Ouptut.First().Name + "_actual"); er.Header.Add(evParams.Ouptut.First().Name + "_predicted"); er.Actual = new List <float>(); er.Predicted = new List <float>(); er.ActualEx = new List <List <float> >(); er.PredictedEx = new List <List <float> >(); // var fun = Function.Load(nnModelPath, device); var resultEx = await Task.Run(() => MLEvaluator.EvaluateFunctionEx(fun, evParams, device)); //var resultEx = EvaluateFunctionEx(nnModelPath, dataPath, evParams, device); for (int i = 0; i < resultEx.actual.Count(); i++) { var res1 = MLValue.GetResult(resultEx.actual[i]); er.Actual.Add(res1); var res2 = MLValue.GetResult(resultEx.predicted[i]); er.Predicted.Add(res2); } er.ActualEx = resultEx.actual; er.PredictedEx = resultEx.predicted; return(er); } else { throw new Exception("Unknown evaluation type!"); } } catch (Exception) { throw; } }