/// <summary>
/// Evaluate mlconfig stored in mlconfigPath for input row stored in vector array
/// </summary>
/// <param name="mlconfigPath"></param>
/// <param name="vector"></param>
/// <param name="pdevice"></param>
/// <returns></returns>
public static object Predict(string mlconfigPath, float[] vector, ProcessDevice pdevice)
{
//device definition
DeviceDescriptor device = MLFactory.GetDevice(pdevice);
return(MLEvaluator.TestModel(mlconfigPath, vector, device));
}
protected void Application_Start()
{
AreaRegistration.RegisterAllAreas();
FilterConfig.RegisterGlobalFilters(GlobalFilters.Filters);
RouteConfig.RegisterRoutes(RouteTable.Routes);
BundleConfig.RegisterBundles(BundleTable.Bundles);
ProcessDevice.CreateDevices();
}
public ActionResult Index()
{
// Ref: http://www.asp.net/mvc/overview/getting-started/introduction/adding-a-view
ViewBag.DeviceKey1 = "Device Key for Avatar01 = "
+ ProcessDevice.GetDeviceKey("Avatar01");
ViewBag.DeviceKey2 = "Device Key for Avatar02 = "
+ ProcessDevice.GetDeviceKey("Avatar02");
ViewBag.IothubConnectionString = ProcessDevice.GetIothubConnectionString();
return(View());
}
/// <summary>
/// Evaluate mlconfig stored in mlconfigPath for input row stored in vector array
/// </summary>
/// <param name="mlconfigPath"></param>
/// <param name="vector"></param>
/// <param name="pdevice"></param>
/// <returns></returns>
public static List <int> Predict(string mlconfigPath, string[] imagePaths, ProcessDevice pdevice)
{
try
{
//device definition
DeviceDescriptor device = MLFactory.GetDevice(pdevice);
return(MLEvaluator.TestModel(mlconfigPath, imagePaths, device));
}
catch (Exception)
{
throw;
}
}
/// <summary>
/// Returns DEviceDescription from ProcessDevice enumeration
/// </summary>
/// <param name="pdevice"></param>
/// <returns></returns>
public static DeviceDescriptor GetDevice(ProcessDevice pdevice)
{
switch (pdevice)
{
case ProcessDevice.Default:
return(DeviceDescriptor.UseDefaultDevice());
case ProcessDevice.CPU:
return(DeviceDescriptor.CPUDevice);
case ProcessDevice.GPU:
return(DeviceDescriptor.GPUDevice(0));
default:
return(DeviceDescriptor.UseDefaultDevice());
}
}
private void EvaluateModel_BtnClick(object sender, RoutedEventArgs e)
{
try
{
var model = (MLConfigController)DataContext;
var testMetaData = model.TestData;
var columns = dataGrid.Columns;
//check if the trained model exists
if (string.IsNullOrEmpty(model.TrainingParameters.LastBestModel) || string.IsNullOrEmpty(model.TrainingParameters.LastBestModel.Trim(' ')))
{
throw new Exception("There is no model to test. Model must be trained first.");
}
//Load ML model configuration file
var modelPath = Project.GetMLConfigPath(model.Settings, model.Name);
var dicMParameters = Project.LoadMLConfig(modelPath);
var trainedModelRelativePath = Project.GetParameterValue(dicMParameters["training"], "TrainedModel");
//add full path of model folder since model file doesn't contains any apsolute path
dicMParameters.Add("root", Project.GetMLConfigFolder(modelPath));
var strModelToEvaluatePath = $"{dicMParameters["root"]}\\{trainedModelRelativePath}";
ProcessDevice pd = ProcessDevice.Default;
//parse each row and transform it to ML ready data in order to predict the result
for (int i = 0; i < m_TestData.Count; i++)
{
var vector = new List <float>();
for (int j = 0; j < m_TestData[0].Count; j++)
{
//check if the value valid
if (string.IsNullOrEmpty(m_TestData[i][j]))
{
throw new Exception($"Value cannot be empty.");
}
var value = m_TestData[i][j];
if (columns[j] is DataGridComboBoxColumn)
{
var combo = columns[j] as DataGridComboBoxColumn;
foreach (var itm in combo.ItemsSource)
{
var c = itm.ToString();
if (m_TestData[i][j].Equals(c, StringComparison.InvariantCultureIgnoreCase))
{
vector.Add(1);
}
else
{
vector.Add(0);
}
}
}
else
{
if (m_TestData[i][j].Contains(","))
{
throw new Exception("Decimal separator should be point.");
}
var vald = double.Parse(m_TestData[i][j], CultureInfo.InvariantCulture);
vector.Add((float)vald);
}
}
//
var val = Project.Predict(strModelToEvaluatePath, vector.ToArray(), pd);
var labelCol = testMetaData.Where(x => x.Kind == DataKind.Label).FirstOrDefault();
if (labelCol.Type == DataType.Category)
{
var ind = int.Parse(val.ToString());
listResult.Items.Add(labelCol.Classes[ind]);
}
else
{
listResult.Items.Add(val);
}
}
}
catch (Exception ex)
{
var appCnt = App.Current.MainWindow.DataContext as AppController;
if (appCnt != null)
{
appCnt.ReportException(ex);
}
}
}
/// <summary>
/// Main methods for model training
/// </summary>
/// <param name="mlconfigPath"></param>
/// <param name="token"></param>
/// <param name="trainingProgress"></param>
/// <param name="pdevice"></param>
/// <returns></returns>
public static TrainResult TrainModel(string mlconfigPath, CancellationToken token, TrainingProgress trainingProgress, ProcessDevice pdevice)
{
try
{
//device definition
DeviceDescriptor device = MLFactory.GetDevice(pdevice);
//LOad ML configuration file
var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath);
//add path of model folder
dicMParameters.Add("root", Project.GetMLConfigFolder(mlconfigPath));
//prepare NN data
var retVal = MLFactory.PrepareNNData(dicMParameters, CustomNNModels.CustomModelCallEntryPoint, device);
//create trainer
var tr = new MLTrainer(retVal.f.StreamConfigurations, retVal.f.InputVariables, retVal.f.OutputVariables);
//setup model checkpoint
string modelCheckPoint = null;
if (dicMParameters.ContainsKey("configid"))
{
modelCheckPoint = MLFactory.GetModelCheckPointPath(mlconfigPath, dicMParameters["configid"].Trim(' '));
}
//setup model checkpoint
string historyPath = null;
if (dicMParameters.ContainsKey("configid"))
{
historyPath = MLFactory.GetTrainingHistoryPath(mlconfigPath, dicMParameters["configid"].Trim(' '));
}
//create trainer
var trainer = tr.CreateTrainer(retVal.nnModel, retVal.lrData, retVal.trData, modelCheckPoint, historyPath);
//perform training
var result = tr.Train(trainer, retVal.nnModel, retVal.trData, retVal.mbs, device, token, trainingProgress, modelCheckPoint, historyPath);
return(result);
}
catch (Exception)
{
throw;
}
}
public static async Task <EvaluationResult> EvaluateMLConfig(string mlconfigPath, DataSetType dsType, EvaluationType evType, ProcessDevice pdevice)
{
//device definition
DeviceDescriptor device = MLFactory.GetDevice(pdevice);
return(await MLEvaluator.EvaluateMLConfig(mlconfigPath, device, dsType, evType));
}
public void RunModel(string mlconfigName, CancellationToken token, TrainingProgress trainingProgress, ProcessDevice pdevice)
{
Project.TrainModel(mlconfigName, token, trainingProgress, pdevice);
}
public static EvaluationResult EvaluateModel(string mlconfigPath, DataSetType dsType, EvaluationType evType, ProcessDevice pdevice)
{
var er = new EvaluationResult();
er.Header = new List <string>();
//device definition
DeviceDescriptor device = MLFactory.GetDevice(pdevice);
//Load ML model configuration file
var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath);
//add full path of model folder since model file doesn't contains any absolute path
dicMParameters.Add("root", Project.GetMLConfigFolder(mlconfigPath));
// get model data paths
var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]);
var modelName = Project.GetParameterValue(dicMParameters["training"], "TrainedModel");
var nnModelPath = Path.Combine(dicMParameters["root"], modelName);
//check if model exists
if (!MLFactory.IsFileExist(nnModelPath))
{
return(er);
}
//check if dataset files exist
var dataPath = GetDataPath(dicMParameters, dsType);
if (!MLFactory.IsFileExist(dataPath))
{
//in case validation dataset is not defiend just export traininign dataset
if (dsType == DataSetType.Validation)
{
dataPath = GetDataPath(dicMParameters, DataSetType.Training);
}
if (!MLFactory.IsFileExist(dataPath))
{
return(er);
}
}
//get output classes in case the ml problem is classification
var strCls = dicMParameters.ContainsKey("metadata") ? dicMParameters["metadata"] : "";
er.OutputClasses = DataDescriptor.GetOutputClasses(strCls);
//Minibatch type
var mbTypestr = Project.GetParameterValue(dicMParameters["training"], "Type");
MinibatchType mbType = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true);
var mbSizetr = Project.GetParameterValue(dicMParameters["training"], "BatchSize");
var mf = MLFactory.CreateMLFactory(dicMParameters);
//perform evaluation
var evParams = new EvaluationParameters()
{
MinibatchSize = uint.Parse(mbSizetr),
MBSource = new MinibatchSourceEx(mbType, mf.StreamConfigurations.ToArray(), dataPath, null, MinibatchSource.FullDataSweep, false),
Input = mf.InputVariables,
Ouptut = mf.OutputVariables,
};
//evaluate model
if (evType == EvaluationType.FeaturesOnly)
{
if (!dicMParameters.ContainsKey("metadata"))
{
throw new Exception("The result cannot be exported to Excel, since no metadata is stored in mlconfig file.");
}
var desc = ParseRawDataSet(dicMParameters["metadata"]);
er.Header = generateHeader(desc);
er.DataSet = FeatureAndLabels(nnModelPath, dataPath, evParams, device);
return(er);
}
else if (evType == EvaluationType.Results)
{
//define header
er.Header.Add(evParams.Ouptut.First().Name + "_actual");
er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
var result = EvaluateFunction(nnModelPath, dataPath, evParams, device);
er.Actual = result.actual.ToList();
er.Predicted = result.predicted.ToList();
return(er);
}
else if (evType == EvaluationType.ResultyExtended)
{
//define header
er.Header.Add(evParams.Ouptut.First().Name + "_actual");
er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
er.Actual = new List <float>();
er.Predicted = new List <float>();
er.ActualEx = new List <List <float> >();
er.PredictedEx = new List <List <float> >();
//
var resultEx = EvaluateFunctionEx(nnModelPath, dataPath, evParams, device);
for (int i = 0; i < resultEx.actual.Count(); i++)
{
var res1 = MLValue.GetResult(resultEx.actual[i]);
er.Actual.Add(res1);
var res2 = MLValue.GetResult(resultEx.predicted[i]);
er.Predicted.Add(res2);
}
er.ActualEx = resultEx.actual;
er.PredictedEx = resultEx.predicted;
return(er);
}
else
{
throw new Exception("Unknown evaluation type!");
}
}