/// <summary>
/// Returns part of mldataset with features labels columns this is needed in case Excel export is performed.
/// </summary>
/// <param name="fun"></param>
/// <param name="evParam"></param>
/// <param name="device"></param>
/// <returns></returns>
public static Dictionary <string, List <List <float> > > FeaturesAndLabels(Function fun, EvaluationParameters evParam, DeviceDescriptor device)
{
try
{
//declare return vars
var featDic = new Dictionary <string, List <List <float> > >();
while (true)
{
//get one minibatch of data for training
var mbData = evParam.MBSource.GetNextMinibatch(evParam.MinibatchSize, device);
var mdDataEx = MinibatchSourceEx.ToMinibatchValueData(mbData, evParam.Input.Union(evParam.Ouptut).ToList());
var inMap = new Dictionary <Variable, Value>();
//input
var vars = evParam.Input;
for (int i = 0; i < vars.Count() /*mdDataEx.Count*/; i++)
{
var vv = vars.ElementAt(i);
var d = mdDataEx.Where(x => x.Key.Name.Equals(vv.Name)).FirstOrDefault();
//
var fv = MLValue.GetValues(d.Key, d.Value);
if (featDic.ContainsKey(d.Key.Name))
{
featDic[d.Key.Name].AddRange(fv);
}
else
{
featDic.Add(d.Key.Name, fv);
}
}
//output
var varso = evParam.Ouptut;
for (int i = 0; i < varso.Count() /*mdDataEx.Count*/; i++)
{
var vv = varso.ElementAt(i);
var d = mdDataEx.Where(x => x.Key.Name.Equals(vv.Name)).FirstOrDefault();
//
var fv = MLValue.GetValues(d.Key, d.Value);
if (vv.Shape.Dimensions.Last() == 1)
{
var value = fv.Select(l => new List <float>()
{
l.First()
}).ToList();
if (featDic.ContainsKey(d.Key.Name))
{
featDic[d.Key.Name].AddRange(value);
}
else
{
featDic.Add(d.Key.Name, value);
}
}
else
{
var value = fv.Select(l => new List <float>()
{
l.IndexOf(l.Max())
}).ToList();
if (featDic.ContainsKey(d.Key.Name))
{
featDic[d.Key.Name].AddRange(value);
}
else
{
featDic.Add(d.Key.Name, value);
}
}
}
// check if sweep end reached
if (mbData.Any(x => x.Value.sweepEnd))
{
break;
}
}
return(featDic);
}
catch (Exception)
{
throw;
}
}
public static (List <List <float> > actual, List <List <float> > predicted) EvaluateFunctionEx(Function fun, EvaluationParameters evParam, DeviceDescriptor device)
{
try
{
//declare return vars
List <List <float> > actualLst = new List <List <float> >();
List <List <float> > predictedLst = new List <List <float> >();
while (true)
{
Value predicted = null;
//get one minibatch of data for training
var mbData = evParam.MBSource.GetNextMinibatch(evParam.MinibatchSize, device);
var mbDataEx = MinibatchSourceEx.ToMinibatchValueData(mbData, evParam.Input.Union(evParam.Ouptut).ToList());
var inMap = new Dictionary <Variable, Value>();
//
var vars = fun.Arguments.Union(fun.Outputs);
for (int i = 0; i < vars.Count() /* mbDataEx.Count*/; i++)
{
var d = mbDataEx.ElementAt(i);
var v = vars.Where(x => x.Name.Equals(d.Key.Name)).First();
//skip output data
if (!evParam.Ouptut.Any(x => x.Name.Equals(v.Name)))
{
inMap.Add(v, d.Value);
}
}
//actual data if t is available
var actualVar = mbDataEx.Keys.Where(x => x.Name.Equals(evParam.Ouptut.First().Name)).FirstOrDefault();
var act = mbDataEx[actualVar].GetDenseData <float>(actualVar).Select(l => l.ToList());
actualLst.AddRange(act);
//predicted data
//map variables and data
var predictedDataMap = new Dictionary <Variable, Value>()
{
{ fun, null }
};
//evaluates model
fun.Evaluate(inMap, predictedDataMap, device);
predicted = predictedDataMap.Values.First();
var pred = predicted.GetDenseData <float>(fun).Select(l => l.ToList());
predicted.Erase();
predicted.Dispose();
predictedLst.AddRange(pred);
// check if sweep end reached
if (mbData.Any(x => x.Value.sweepEnd))
{
break;
}
}
return(actualLst, predictedLst);
}
catch (Exception)
{
throw;
}
}
/// <summary>
/// Evaluate model defined in the mlconfig file
/// </summary>
/// <param name="mlconfigPath"></param>
/// <param name="device"></param>
/// <returns></returns>
public static async Task <EvaluationResult> EvaluateMLConfig(string mlconfigPath, DeviceDescriptor device, DataSetType dsType, EvaluationType evType)
{
try
{
//define eval result
var er = new EvaluationResult();
er.OutputClasses = new List <string>()
{
""
};
er.Actual = new List <float>();
er.Predicted = new List <float>();
er.Header = new List <string>();
//Load ML configuration file
var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath);
//add full path of model folder since model file doesn't contains any absolute path
dicMParameters.Add("root", MLFactory.GetMLConfigFolder(mlconfigPath));
// get model data paths
var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]);
//parse feature variables
var projectValues = dicMParameters["training"].Split(MLFactory.m_cntkSpearator, StringSplitOptions.RemoveEmptyEntries);
var modelName = MLFactory.GetParameterValue(projectValues, "TrainedModel");
var nnModelPath = Path.Combine(dicMParameters["root"], modelName);
//check if model exists
if (!MLFactory.IsFileExist(nnModelPath))
{
return(er);
}
//
var dataset = MLFactory.GetDataPath(dicMParameters, dsType);
if (string.IsNullOrEmpty(dataset) || string.IsNullOrEmpty(dataset) || dataset == " ")
{
if (dsType == DataSetType.Testing)
{
dataset = MLFactory.GetDataPath(dicMParameters, DataSetType.Validation);
}
if (string.IsNullOrEmpty(dataset) || string.IsNullOrEmpty(dataset) || dataset == " ")
{
return(er);
}
}
//get output classes in case the ml problem is classification
var strCls = dicMParameters.ContainsKey("metadata") ? dicMParameters["metadata"] : "";
var oc = MLFactory.GetOutputClasses(strCls);
if (oc != null)
{
er.OutputClasses = oc;
}
//MInibatch
var mbTypestr = MLFactory.GetParameterValue(projectValues, "Type");
MinibatchType mbType = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true);
var mbSizetr = MLFactory.GetParameterValue(projectValues, "BatchSize");
var mf = MLFactory.CreateMLFactory(dicMParameters);
//perform evaluation
var evParams = new EvaluationParameters()
{
MinibatchSize = uint.Parse(mbSizetr),
MBSource = new MinibatchSourceEx(mbType, mf.StreamConfigurations.ToArray(), mf.InputVariables, mf.OutputVariables, dataset, null, MinibatchSource.FullDataSweep, false, 0),
Input = mf.InputVariables,
Ouptut = mf.OutputVariables,
};
//evaluate model
if (evType == EvaluationType.FeaturesOnly)
{
if (!dicMParameters.ContainsKey("metadata"))
{
throw new Exception("The result cannot be exported to Excel, since no metadata is stored in mlconfig file.");
}
var desc = MLFactory.ParseRawDataSet(dicMParameters["metadata"]);
er.Header = MLFactory.GenerateHeader(desc);
var fun = Function.Load(nnModelPath, device);
//
er.DataSet = await Task.Run(() => MLEvaluator.FeaturesAndLabels(fun, evParams, device));
return(er);
}
else if (evType == EvaluationType.Results)
{
//define header
er.Header.Add(evParams.Ouptut.First().Name + "_actual");
er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
var fun = Function.Load(nnModelPath, device);
//
var result = await Task.Run(() => MLEvaluator.EvaluateFunction(fun, evParams, device));
er.Actual = result.actual.ToList();
er.Predicted = result.predicted.ToList();
if (er.OutputClasses.Count < 2 && evParams.Ouptut.First().Shape.Dimensions.Last() > 1)
{
var result1 = await Task.Run(() => MLEvaluator.EvaluateFunctionEx(fun, evParams, device));
er.ActualEx = result1.actual;
er.PredictedEx = result1.predicted;
}
return(er);
}
else if (evType == EvaluationType.ResultExtended)
{
//define header
er.Header.Add(evParams.Ouptut.First().Name + "_actual");
er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
er.Actual = new List <float>();
er.Predicted = new List <float>();
er.ActualEx = new List <List <float> >();
er.PredictedEx = new List <List <float> >();
//
var fun = Function.Load(nnModelPath, device);
var resultEx = await Task.Run(() => MLEvaluator.EvaluateFunctionEx(fun, evParams, device));
//var resultEx = EvaluateFunctionEx(nnModelPath, dataPath, evParams, device);
for (int i = 0; i < resultEx.actual.Count(); i++)
{
var res1 = MLValue.GetResult(resultEx.actual[i]);
er.Actual.Add(res1);
var res2 = MLValue.GetResult(resultEx.predicted[i]);
er.Predicted.Add(res2);
}
er.ActualEx = resultEx.actual;
er.PredictedEx = resultEx.predicted;
return(er);
}
else
{
throw new Exception("Unknown evaluation type!");
}
}
catch (Exception)
{
throw;
}
}
public static (IEnumerable <float> actual, IEnumerable <float> predicted) EvaluateFunction(Function fun, EvaluationParameters evParam, DeviceDescriptor device)
{
try
{
//declare return vars
List <float> actualLst = new List <float>();
List <float> predictedLst = new List <float>();
var result = EvaluateFunctionEx(fun, evParam, device);
for (int i = 0; i < result.actual.Count(); i++)
{
var res1 = MLValue.GetResult(result.actual[i]);
actualLst.Add(res1);
var res2 = MLValue.GetResult(result.predicted[i]);
predictedLst.Add(res2);
}
return(actualLst, predictedLst);
}
catch (Exception)
{
throw;
}
}
protected virtual ProgressData progressTraining(TrainingParameters trParams, Trainer trainer,
Function network, MinibatchSourceEx mbs, int epoch, TrainingProgress progress, DeviceDescriptor device)
{
//calculate average training loss and evaluation
var mbAvgLoss = trainer.PreviousMinibatchLossAverage();
var mbAvgEval = trainer.PreviousMinibatchEvaluationAverage();
//get training dataset
double trainEval = mbAvgEval;
//sometimes when the data set is huge validation model against
// full training dataset could take time, so we can skip it by setting parameter 'FullTrainingSetEval'
if (trParams.FullTrainingSetEval)
{
var evParams = new EvaluationParameters()
{
MinibatchSize = trParams.BatchSize,
MBSource = new MinibatchSourceEx(mbs.Type, this.StreamConfigurations.ToArray(), this.InputVariables, this.OutputVariables, mbs.TrainingDataFile, null, MinibatchSource.FullDataSweep, false, 0),
Ouptut = OutputVariables,
Input = InputVariables,
};
var result = MLEvaluator.EvaluateFunction(trainer.Model(), evParams, device);
trainEval = MLEvaluator.CalculateMetrics(trainer.EvaluationFunction().Name, result.actual, result.predicted, device);
////if output has more than one dimension and when the output is not categorical but numeric with more than one value
////for now only custom mini-batch source is supported this kind of variable
//if(OutputVariables.First().Shape.Dimensions.Last() > 1 && evParams.MBSource.Type== MinibatchType.Custom)
//{
// var result1 = MLEvaluator.EvaluateFunctionEx(trainer.Model(), evParams, device);
// trainEval = MLEvaluator.CalculateMetrics(trainer.EvaluationFunction().Name, result1.actual, result1.predicted, device);
//}
//else
//{
// var result = MLEvaluator.EvaluateFunction(trainer.Model(), evParams, device);
// trainEval = MLEvaluator.CalculateMetrics(trainer.EvaluationFunction().Name, result.actual, result.predicted, device);
//}
}
string bestModelPath = m_bestModelPath;
double validEval = 0;
//in case validation data set is empty don't perform test-minibatch
if (!string.IsNullOrEmpty(mbs.ValidationDataFile))
{
var evParams = new EvaluationParameters()
{
MinibatchSize = trParams.BatchSize,
//StrmsConfig = StreamConfigurations.ToArray(),
MBSource = new MinibatchSourceEx(mbs.Type, this.StreamConfigurations.ToArray(), this.InputVariables, this.OutputVariables, mbs.ValidationDataFile, null, MinibatchSource.FullDataSweep, false, 0),
Ouptut = OutputVariables,
Input = InputVariables,
};
//
var result = MLEvaluator.EvaluateFunction(trainer.Model(), evParams, device);
validEval = MLEvaluator.CalculateMetrics(trainer.EvaluationFunction().Name, result.actual, result.predicted, device);
////if output has more than one dimension and when the output is not categorical but numeric with more than one value
////for now only custom mini-batch source is supported this kind of variable
//if (OutputVariables.First().Shape.Dimensions.Last() > 1 && evParams.MBSource.Type == MinibatchType.Custom)
//{
// var result1 = MLEvaluator.EvaluateFunctionEx(trainer.Model(), evParams, device);
// validEval = MLEvaluator.CalculateMetrics(trainer.EvaluationFunction().Name, result1.actual, result1.predicted, device);
//}
//else
//{
// var result = MLEvaluator.EvaluateFunction(trainer.Model(), evParams, device);
// validEval = MLEvaluator.CalculateMetrics(trainer.EvaluationFunction().Name, result.actual, result.predicted, device);
//}
}
//here we should decide if the current model worth to be saved into temp location
// depending of the Evaluation function which sometimes can be better if it is greater that previous (e.g. ClassificationAccuracy)
if (isBetterThanPrevious(trainEval, validEval, StatMetrics.IsGoalToMinimize(trainer.EvaluationFunction())) && trParams.SaveModelWhileTraining)
{
//save model
var strFilePath = $"{trParams.ModelTempLocation}\\model_at_{epoch}of{trParams.Epochs}_epochs_TimeSpan_{DateTime.Now.Ticks}";
if (!Directory.Exists(trParams.ModelTempLocation))
{
Directory.CreateDirectory(trParams.ModelTempLocation);
}
//save temp model
network.Save(strFilePath);
//set training and validation evaluation to previous state
m_PrevTrainingEval = trainEval;
m_PrevValidationEval = validEval;
bestModelPath = strFilePath;
var tpl = Tuple.Create <double, double, string>(trainEval, validEval, strFilePath);
m_ModelEvaluations.Add(tpl);
}
m_bestModelPath = bestModelPath;
//create progressData object
var prData = new ProgressData();
prData.EpochTotal = trParams.Epochs;
prData.EpochCurrent = epoch;
prData.EvaluationFunName = trainer.EvaluationFunction().Name;
prData.TrainEval = trainEval;
prData.ValidationEval = validEval;
prData.MinibatchAverageEval = mbAvgEval;
prData.MinibatchAverageLoss = mbAvgLoss;
//prData.BestModel = bestModelPath;
//the progress is only reported if satisfied the following condition
if (progress != null && (epoch % trParams.ProgressFrequency == 0 || epoch == 1 || epoch == trParams.Epochs))
{
//add info to the history
m_trainingHistory.Add(new Tuple <int, float, float, float, float>(epoch, (float)mbAvgLoss, (float)mbAvgEval,
(float)trainEval, (float)validEval));
//send progress
progress(prData);
//
//Console.WriteLine($"Epoch={epoch} of {trParams.Epochs} processed.");
}
//return progress data
return(prData);
}
