public MinibatchSourceEx(MinibatchType type, StreamConfiguration[] streamConfigurations, List <Variable> inputVar, List <Variable> outputVar,
string trainFilePath, string validFilePath, ulong epochSize, bool randomizeBatch, int useImgAugm)
{
this.StreamConfigurations = streamConfigurations;
this.TrainingDataFile = trainFilePath;
this.ValidationDataFile = validFilePath;
Type = type;
if (Type == MinibatchType.Default)
{
// prepare the training data
defaultmb = MinibatchSource.TextFormatMinibatchSource(trainFilePath, StreamConfigurations, epochSize, randomizeBatch);
}
else if (Type == MinibatchType.Image)
{
var featVar = inputVar.First();
//
int image_width = featVar.Shape.Dimensions[0];
int image_height = featVar.Shape.Dimensions[1];
int num_channels = featVar.Shape.Dimensions[2];
//make transformation and scaling
var transforms = new List <CNTKDictionary>();
var randomSideTransform = CNTKLib.ReaderCrop("RandomSide",
new Tuple <int, int>(0, 0),
new Tuple <float, float>(0.8f, 1.0f),
new Tuple <float, float>(0.0f, 0.0f),
new Tuple <float, float>(1.0f, 1.0f),
"uniRatio");
if (useImgAugm == 1)
{
transforms.Add(randomSideTransform);
}
//scaling image comes at the end of image transformation
var scaleTransform = CNTKLib.ReaderScale(image_width, image_height, num_channels);
transforms.Add(scaleTransform);
var labelName = streamConfigurations.Last().m_streamName;
var labelDimension = streamConfigurations.Last().m_dim;
var featureName = streamConfigurations.First().m_streamName;
var imagemb = CNTKLib.ImageDeserializer(trainFilePath, labelName, (uint)labelDimension, featureName, transforms);
var mmsConfig = new CNTK.MinibatchSourceConfig(new CNTK.DictionaryVector()
{
imagemb
});
//
defaultmb = CNTKLib.CreateCompositeMinibatchSource(mmsConfig);
}
else if (Type == MinibatchType.Custom)
{
custommb = new StreamReader(trainFilePath);
}
else
{
throw new Exception("Minibatchsource type is unknown!");
}
}
public MinibatchSourceEx(MinibatchType type, StreamConfiguration[] streamConfigurations, string trainFilePath, string validFilePath, ulong epochSize, bool randomizeBatch)
{
this.StreamConfigurations = streamConfigurations;
this.TrainingDataFile = trainFilePath;
this.ValidationDataFile = validFilePath;
Type = type;
if (Type == MinibatchType.Default)
{
// prepare the training data
defaultmb = MinibatchSource.TextFormatMinibatchSource(trainFilePath, StreamConfigurations, epochSize, randomizeBatch);
}
else if (Type == MinibatchType.Custom)
{
custommb = new StreamReader(trainFilePath);
}
}
/// <summary>
/// The method is called during Evaluation of the model for specific data set which is specified as an argument
/// </summary>
/// <param name="type"></param>
/// <param name="strFilePath">dataset file path</param>
/// <param name="streamConfigurations">stream configuration which provides meta-data information</param>
/// <param name="device"></param>
/// <returns></returns>
public static UnorderedMapStreamInformationMinibatchData GetFullBatch(MinibatchType type, string strFilePath, StreamConfiguration[] streamConfigurations, DeviceDescriptor device)
{
if (type == MinibatchType.Default)
{
var mbs = MinibatchSource.TextFormatMinibatchSource(strFilePath, streamConfigurations, MinibatchSource.FullDataSweep, false);
//
var minibatchData = mbs.GetNextMinibatch(int.MaxValue, device);
//
return(minibatchData);
}
else if (type == MinibatchType.Custom)
{
using (var mbreader = new StreamReader(strFilePath))
{
var retVal = nextBatch(mbreader, streamConfigurations, -1, 1, device);
var mb = new UnorderedMapStreamInformationMinibatchData();
for (int i = 0; i < retVal.Count; i++)
{
var k = retVal.ElementAt(i);
var key = k.Key;
var si = new StreamInformation();
si.m_definesMbSize = streamConfigurations[i].m_definesMbSize;
si.m_storageFormat = k.Value.data.StorageFormat;
si.m_name = streamConfigurations[i].m_streamName;
var stream = streamConfigurations[i];
mb.Add(si, k.Value);
}
return(mb);
}
}
else
{
throw new Exception("Minibatch is not supported.");
}
}
/// <summary>
/// Convert minibatch data
/// </summary>
/// <param name="args"> data being converted</param>
/// <returns></returns>
public static UnorderedMapVariableMinibatchData ToMinibatchData(UnorderedMapStreamInformationMinibatchData args, List <Variable> vars, MinibatchType type)
{
var arguments = new UnorderedMapVariableMinibatchData();
foreach (var mbd in args)
{
var v = vars.Where(x => x.Name == mbd.Key.m_name).FirstOrDefault();
if (v == null)
{
throw new Exception("Stream is invalid!");
}
if (type == MinibatchType.Custom)
{
var mbd1 = new MinibatchData(mbd.Value.data.DeepClone(), mbd.Value.numberOfSamples, mbd.Value.sweepEnd);
arguments.Add(v, mbd1);
}
else
{
arguments.Add(v, mbd.Value);
}
}
return(arguments);
}
/// <summary>
/// Evaluate model defined in the mlconfig file
/// </summary>
/// <param name="mlconfigPath"></param>
/// <param name="device"></param>
/// <returns></returns>
public static async Task <EvaluationResult> EvaluateMLConfig(string mlconfigPath, DeviceDescriptor device, DataSetType dsType, EvaluationType evType)
{
try
{
//define eval result
var er = new EvaluationResult();
er.OutputClasses = new List <string>()
{
""
};
er.Actual = new List <float>();
er.Predicted = new List <float>();
er.Header = new List <string>();
//Load ML configuration file
var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath);
//add full path of model folder since model file doesn't contains any absolute path
dicMParameters.Add("root", MLFactory.GetMLConfigFolder(mlconfigPath));
// get model data paths
var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]);
//parse feature variables
var projectValues = dicMParameters["training"].Split(MLFactory.m_cntkSpearator, StringSplitOptions.RemoveEmptyEntries);
var modelName = MLFactory.GetParameterValue(projectValues, "TrainedModel");
var nnModelPath = Path.Combine(dicMParameters["root"], modelName);
//check if model exists
if (!MLFactory.IsFileExist(nnModelPath))
{
return(er);
}
//
var dataset = MLFactory.GetDataPath(dicMParameters, dsType);
if (string.IsNullOrEmpty(dataset) || string.IsNullOrEmpty(dataset) || dataset == " ")
{
if (dsType == DataSetType.Testing)
{
dataset = MLFactory.GetDataPath(dicMParameters, DataSetType.Validation);
}
if (string.IsNullOrEmpty(dataset) || string.IsNullOrEmpty(dataset) || dataset == " ")
{
return(er);
}
}
//get output classes in case the ml problem is classification
var strCls = dicMParameters.ContainsKey("metadata") ? dicMParameters["metadata"] : "";
var oc = MLFactory.GetOutputClasses(strCls);
if (oc != null)
{
er.OutputClasses = oc;
}
//MInibatch
var mbTypestr = MLFactory.GetParameterValue(projectValues, "Type");
MinibatchType mbType = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true);
var mbSizetr = MLFactory.GetParameterValue(projectValues, "BatchSize");
var mf = MLFactory.CreateMLFactory(dicMParameters);
//perform evaluation
var evParams = new EvaluationParameters()
{
MinibatchSize = uint.Parse(mbSizetr),
MBSource = new MinibatchSourceEx(mbType, mf.StreamConfigurations.ToArray(), mf.InputVariables, mf.OutputVariables, dataset, null, MinibatchSource.FullDataSweep, false, 0),
Input = mf.InputVariables,
Ouptut = mf.OutputVariables,
};
//evaluate model
if (evType == EvaluationType.FeaturesOnly)
{
if (!dicMParameters.ContainsKey("metadata"))
{
throw new Exception("The result cannot be exported to Excel, since no metadata is stored in mlconfig file.");
}
var desc = MLFactory.ParseRawDataSet(dicMParameters["metadata"]);
er.Header = MLFactory.GenerateHeader(desc);
var fun = Function.Load(nnModelPath, device);
//
er.DataSet = await Task.Run(() => MLEvaluator.FeaturesAndLabels(fun, evParams, device));
return(er);
}
else if (evType == EvaluationType.Results)
{
//define header
er.Header.Add(evParams.Ouptut.First().Name + "_actual");
er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
var fun = Function.Load(nnModelPath, device);
//
var result = await Task.Run(() => MLEvaluator.EvaluateFunction(fun, evParams, device));
er.Actual = result.actual.ToList();
er.Predicted = result.predicted.ToList();
if (er.OutputClasses.Count < 2 && evParams.Ouptut.First().Shape.Dimensions.Last() > 1)
{
var result1 = await Task.Run(() => MLEvaluator.EvaluateFunctionEx(fun, evParams, device));
er.ActualEx = result1.actual;
er.PredictedEx = result1.predicted;
}
return(er);
}
else if (evType == EvaluationType.ResultExtended)
{
//define header
er.Header.Add(evParams.Ouptut.First().Name + "_actual");
er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
er.Actual = new List <float>();
er.Predicted = new List <float>();
er.ActualEx = new List <List <float> >();
er.PredictedEx = new List <List <float> >();
//
var fun = Function.Load(nnModelPath, device);
var resultEx = await Task.Run(() => MLEvaluator.EvaluateFunctionEx(fun, evParams, device));
//var resultEx = EvaluateFunctionEx(nnModelPath, dataPath, evParams, device);
for (int i = 0; i < resultEx.actual.Count(); i++)
{
var res1 = MLValue.GetResult(resultEx.actual[i]);
er.Actual.Add(res1);
var res2 = MLValue.GetResult(resultEx.predicted[i]);
er.Predicted.Add(res2);
}
er.ActualEx = resultEx.actual;
er.PredictedEx = resultEx.predicted;
return(er);
}
else
{
throw new Exception("Unknown evaluation type!");
}
}
catch (Exception)
{
throw;
}
}
/// <summary>
/// Evaluate the model against dataset sored in the dataset file, and exports the result in csv format for further analysis
/// </summary>
/// <param name="mlF"> ml factory object contains members needed to evaluation process</param>
/// <param name="mbs"> Minibatch source which provides helpers members needed for for evaluation</param>
/// <param name="strDataSetPath"> file of dataset</param>
/// <param name="modelPath"> models which will be evaluate</param>
/// <param name="resultExportPath"> result file in which the result will be exported</param>
/// <param name="device"> device for computation</param>
public static void EvaluateModel(string mlconfigPath, string bestTrainedModelPath, DeviceDescriptor device)
{
//Load ML model configuration file
var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath);
//add full path of model folder since model file doesn't contains any absolute path
dicMParameters.Add("root", MLFactory.GetMLConfigFolder(mlconfigPath));
//get model daa paths
var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]);
//parse feature variables
var projectValues = dicMParameters["training"].Split(MLFactory.m_cntkSpearator, StringSplitOptions.RemoveEmptyEntries);
var trainedModelRelativePath = MLFactory.GetParameterValue(projectValues, "TrainedModel");
//Minibatch type
var mbTypestr = MLFactory.GetParameterValue(projectValues, "Type");
MinibatchType mbType = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true);
//prepare MLFactory
var f = MLFactory.CreateMLFactory(dicMParameters);
//prepare data paths for mini-batch source
var strTrainPath = $"{dicMParameters["root"]}\\{dicPath["Training"]}";
var strValidPath = $"{dicMParameters["root"]}\\{dicPath["Validation"]}";
var strResult = $"{dicMParameters["root"]}\\{dicPath["Result"]}";
var bestModelFullPath = $"{dicMParameters["root"]}\\{bestTrainedModelPath}";
//decide what data to evaluate
var dataPath = strValidPath;
//load model
var model = Function.Load(bestModelFullPath, device);
//get data for evaluation by calling GetFullBatch
var minibatchData = MinibatchSourceEx.GetFullBatch(mbType, dataPath, f.StreamConfigurations.ToArray(), device);
//input map creation for model evaluation
var inputMap = new Dictionary <Variable, Value>();
foreach (var v in minibatchData)
{
var vv = model.Arguments.Where(x => x.Name == v.Key.m_name).FirstOrDefault();
var streamInfo = v.Key;
if (vv != null)
{
inputMap.Add(vv, minibatchData[streamInfo].data);
}
}
//output map
var predictedDataMap = new Dictionary <Variable, Value>();
foreach (var outp in model.Outputs)
{
predictedDataMap.Add(outp, null);
}
//model evaluation
model.Evaluate(inputMap, predictedDataMap, device);
//retrieve actual and predicted values from model
List <List <float> > actual = new List <List <float> >();
List <List <float> > predict = new List <List <float> >();
foreach (var output in model.Outputs)
{
//label stream
var labelStream = minibatchData.Keys.Where(x => x.m_name == output.Name).First();
//actual values
List <List <float> > av = MLValue.GetValues(output, minibatchData[labelStream].data);
//predicted values
List <List <float> > pv = MLValue.GetValues(output, predictedDataMap[output]);
for (int i = 0; i < av.Count; i++)
{
//actual
var act = av[i];
if (actual.Count <= i)
{
actual.Add(new List <float>());
}
actual[i].AddRange(act);
//prediction
var prd = pv[i];
if (predict.Count <= i)
{
predict.Add(new List <float>());
}
predict[i].AddRange(prd);
}
}
//export result
MLValue.ValueToFile(actual, predict, strResult);
//
Console.WriteLine(Environment.NewLine);
Console.WriteLine($"*******************Model Evaluation**************");
Console.WriteLine(Environment.NewLine);
Console.WriteLine($"Model Evaluation successfully exported result into file {strResult}!");
Console.WriteLine(Environment.NewLine);
}
public static EvaluationResult EvaluateModel(string mlconfigPath, DataSetType dsType, EvaluationType evType, ProcessDevice pdevice)
{
var er = new EvaluationResult();
er.Header = new List <string>();
//device definition
DeviceDescriptor device = MLFactory.GetDevice(pdevice);
//Load ML model configuration file
var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath);
//add full path of model folder since model file doesn't contains any absolute path
dicMParameters.Add("root", Project.GetMLConfigFolder(mlconfigPath));
// get model data paths
var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]);
var modelName = Project.GetParameterValue(dicMParameters["training"], "TrainedModel");
var nnModelPath = Path.Combine(dicMParameters["root"], modelName);
//check if model exists
if (!MLFactory.IsFileExist(nnModelPath))
{
return(er);
}
//check if dataset files exist
var dataPath = GetDataPath(dicMParameters, dsType);
if (!MLFactory.IsFileExist(dataPath))
{
//in case validation dataset is not defiend just export traininign dataset
if (dsType == DataSetType.Validation)
{
dataPath = GetDataPath(dicMParameters, DataSetType.Training);
}
if (!MLFactory.IsFileExist(dataPath))
{
return(er);
}
}
//get output classes in case the ml problem is classification
var strCls = dicMParameters.ContainsKey("metadata") ? dicMParameters["metadata"] : "";
er.OutputClasses = DataDescriptor.GetOutputClasses(strCls);
//Minibatch type
var mbTypestr = Project.GetParameterValue(dicMParameters["training"], "Type");
MinibatchType mbType = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true);
var mbSizetr = Project.GetParameterValue(dicMParameters["training"], "BatchSize");
var mf = MLFactory.CreateMLFactory(dicMParameters);
//perform evaluation
var evParams = new EvaluationParameters()
{
MinibatchSize = uint.Parse(mbSizetr),
MBSource = new MinibatchSourceEx(mbType, mf.StreamConfigurations.ToArray(), dataPath, null, MinibatchSource.FullDataSweep, false),
Input = mf.InputVariables,
Ouptut = mf.OutputVariables,
};
//evaluate model
if (evType == EvaluationType.FeaturesOnly)
{
if (!dicMParameters.ContainsKey("metadata"))
{
throw new Exception("The result cannot be exported to Excel, since no metadata is stored in mlconfig file.");
}
var desc = ParseRawDataSet(dicMParameters["metadata"]);
er.Header = generateHeader(desc);
er.DataSet = FeatureAndLabels(nnModelPath, dataPath, evParams, device);
return(er);
}
else if (evType == EvaluationType.Results)
{
//define header
er.Header.Add(evParams.Ouptut.First().Name + "_actual");
er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
var result = EvaluateFunction(nnModelPath, dataPath, evParams, device);
er.Actual = result.actual.ToList();
er.Predicted = result.predicted.ToList();
return(er);
}
else if (evType == EvaluationType.ResultyExtended)
{
//define header
er.Header.Add(evParams.Ouptut.First().Name + "_actual");
er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
er.Actual = new List <float>();
er.Predicted = new List <float>();
er.ActualEx = new List <List <float> >();
er.PredictedEx = new List <List <float> >();
//
var resultEx = EvaluateFunctionEx(nnModelPath, dataPath, evParams, device);
for (int i = 0; i < resultEx.actual.Count(); i++)
{
var res1 = MLValue.GetResult(resultEx.actual[i]);
er.Actual.Add(res1);
var res2 = MLValue.GetResult(resultEx.predicted[i]);
er.Predicted.Add(res2);
}
er.ActualEx = resultEx.actual;
er.PredictedEx = resultEx.predicted;
return(er);
}
else
{
throw new Exception("Unknown evaluation type!");
}
}
