/// <summary>
/// Parses raw dataset and extract the list of Variables.
/// </summary>
/// <param name="metaDataValues"></param>
/// <returns></returns>
public static List <VariableDescriptor> ParseRawDataSet(string metaDataValues)
{
try
{
//parse data
var mdata = metaDataValues.Split(MLFactory.m_cntkSpearator, StringSplitOptions.RemoveEmptyEntries);
//define columns
var cols = new List <VariableDescriptor>();
//parse meta data
foreach (var c in mdata.Where(x => x.StartsWith("Column")).OrderBy(x => x))
{
VariableDescriptor col = new VariableDescriptor();
//check if double point appear more than one time. In that case raise exception
if (c.Count(x => x == ':') > 1)
{
throw new Exception("Column data contains double point ':' which is reserved char. PLease remove double point from metadata.");
}
var strData = c.Substring(c.IndexOf(":") + 1);
var colValues = strData.Split(MLFactory.m_ValueSpearator, StringSplitOptions.RemoveEmptyEntries);
col.Name = colValues[0];
col.Type = (DataType)Enum.Parse(typeof(DataType), colValues[1], true);
col.Kind = (DataKind)Enum.Parse(typeof(DataKind), colValues[2], true);
col.MissingValue = (MissingValue)Enum.Parse(typeof(MissingValue), colValues[3], true);
//
if (col.Type == DataType.Category)
{
var cl = DataDescriptor.GetColumnClasses(c);
if (cl != null)
{
col.Classes = cl.ToArray();
}
}
//add parsed column to the collection
cols.Add(col);
}
return(cols);
}
catch (Exception)
{
throw;
}
}
public void Load(string projPath)
{
try
{
var fi = new FileInfo(projPath);
if (!fi.Exists)
{
throw new Exception("File not found!");
}
//
if (Settings == null)
{
Settings = new ProjectSettings();
}
//load project tags
var dicValues = MLFactory.LoadMLConfiguration(projPath);
//parse project information
var projectMetaData = dicValues["project"];
parseProject(projectMetaData);
//setup project folder where all files will be places
Settings.ProjectFolder = fi.Directory.FullName;
Settings.ProjectFile = fi.Name;
//reset Descriptor
Descriptor = new DataDescriptor();
//parse data
var dataValues = dicValues["parser"];
parseParser(dataValues);
//parse data
var metaDataValues = dicValues["data"];
parseMetaData(metaDataValues);
}
catch (Exception)
{
throw;
}
}
public static EvaluationResult EvaluateModel(string mlconfigPath, DataSetType dsType, EvaluationType evType, ProcessDevice pdevice)
{
var er = new EvaluationResult();
er.Header = new List <string>();
//device definition
DeviceDescriptor device = MLFactory.GetDevice(pdevice);
//Load ML model configuration file
var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath);
//add full path of model folder since model file doesn't contains any absolute path
dicMParameters.Add("root", Project.GetMLConfigFolder(mlconfigPath));
// get model data paths
var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]);
var modelName = Project.GetParameterValue(dicMParameters["training"], "TrainedModel");
var nnModelPath = Path.Combine(dicMParameters["root"], modelName);
//check if model exists
if (!MLFactory.IsFileExist(nnModelPath))
{
return(er);
}
//check if dataset files exist
var dataPath = GetDataPath(dicMParameters, dsType);
if (!MLFactory.IsFileExist(dataPath))
{
//in case validation dataset is not defiend just export traininign dataset
if (dsType == DataSetType.Validation)
{
dataPath = GetDataPath(dicMParameters, DataSetType.Training);
}
if (!MLFactory.IsFileExist(dataPath))
{
return(er);
}
}
//get output classes in case the ml problem is classification
var strCls = dicMParameters.ContainsKey("metadata") ? dicMParameters["metadata"] : "";
er.OutputClasses = DataDescriptor.GetOutputClasses(strCls);
//Minibatch type
var mbTypestr = Project.GetParameterValue(dicMParameters["training"], "Type");
MinibatchType mbType = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true);
var mbSizetr = Project.GetParameterValue(dicMParameters["training"], "BatchSize");
var mf = MLFactory.CreateMLFactory(dicMParameters);
//perform evaluation
var evParams = new EvaluationParameters()
{
MinibatchSize = uint.Parse(mbSizetr),
MBSource = new MinibatchSourceEx(mbType, mf.StreamConfigurations.ToArray(), dataPath, null, MinibatchSource.FullDataSweep, false),
Input = mf.InputVariables,
Ouptut = mf.OutputVariables,
};
//evaluate model
if (evType == EvaluationType.FeaturesOnly)
{
if (!dicMParameters.ContainsKey("metadata"))
{
throw new Exception("The result cannot be exported to Excel, since no metadata is stored in mlconfig file.");
}
var desc = ParseRawDataSet(dicMParameters["metadata"]);
er.Header = generateHeader(desc);
er.DataSet = FeatureAndLabels(nnModelPath, dataPath, evParams, device);
return(er);
}
else if (evType == EvaluationType.Results)
{
//define header
er.Header.Add(evParams.Ouptut.First().Name + "_actual");
er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
var result = EvaluateFunction(nnModelPath, dataPath, evParams, device);
er.Actual = result.actual.ToList();
er.Predicted = result.predicted.ToList();
return(er);
}
else if (evType == EvaluationType.ResultyExtended)
{
//define header
er.Header.Add(evParams.Ouptut.First().Name + "_actual");
er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
er.Actual = new List <float>();
er.Predicted = new List <float>();
er.ActualEx = new List <List <float> >();
er.PredictedEx = new List <List <float> >();
//
var resultEx = EvaluateFunctionEx(nnModelPath, dataPath, evParams, device);
for (int i = 0; i < resultEx.actual.Count(); i++)
{
var res1 = MLValue.GetResult(resultEx.actual[i]);
er.Actual.Add(res1);
var res2 = MLValue.GetResult(resultEx.predicted[i]);
er.Predicted.Add(res2);
}
er.ActualEx = resultEx.actual;
er.PredictedEx = resultEx.predicted;
return(er);
}
else
{
throw new Exception("Unknown evaluation type!");
}
}
