internal async Task <bool> CreateMLConfig(MLConfigController mlconfig)
{
//save project with new created mlconfig
await Save();
//create model name
string modelName = $"MLConfig{Models.Count}";
//Define ml data file paths
var strModelFolder = Project.GetMLConfigFolder(Settings, modelName);
var strModelDataFolder = Project.GetMLConfigDataFolder(Settings, modelName);
//check if model folder exists
if (!Directory.Exists(strModelFolder))
{
Directory.CreateDirectory(strModelFolder);
}
//check if data folder exists
if (!Directory.Exists(strModelDataFolder))
{
Directory.CreateDirectory(strModelDataFolder);
}
//model name and settings
mlconfig.Name = modelName;
mlconfig.Settings = Settings;
//load project file in order to get settings information and data description needed for data transformation
var strPPath = Path.Combine(Settings.ProjectFolder, Settings.ProjectFile);
var proj = new Project();
proj.Load(strPPath);
//ToDo: optimizes code for huge dataset
if (Settings.ProjectType == ProjectType.Default)
{
createDefaultDataSets(DataSet, modelName, proj.Descriptor);
}
else if (Settings.ProjectType == ProjectType.ImageClassification)
{
createImgClassificationDataSets(DataSet, modelName, proj.Descriptor);
}
else
{
;
}
//create mlconfig file
Project.NewMLConfigFile(proj, modelName, DataSet);
//initialize model
mlconfig.Init();
//add newly created model in to Model collection of the project
Models.Add(mlconfig);
//save project with new created mlconfig
return(await Save());
}
internal void CreateMLConfig(MLConfigController mlconfig)
{
//save project with new created mlconfig
Save();
//create model name
string modelName = $"MLConfig{Models.Count}";
//Define ml data file paths
var strModelFolder = Project.GetMLConfigFolder(Settings, modelName);
var strModelDataFolder = Project.GetMLConfigDataFolder(Settings, modelName);
var strPathTrain = Project.GetDefaultMLDatasetPath(Settings, modelName, true);
var strPathValid = Project.GetDefaultMLDatasetPath(Settings, modelName, false);
//check if model folder exists
if (!Directory.Exists(strModelFolder))
{
Directory.CreateDirectory(strModelFolder);
}
//check if data folder exists
if (!Directory.Exists(strModelDataFolder))
{
Directory.CreateDirectory(strModelDataFolder);
}
//get dataset based on options
var ds = DataSet.GetDataSet(DataSet.RandomizeData);
//we want whole data set later the data will be split
ds.TestRows = 0;
//create experiment based created dataset
var exp = new Experiment(ds);
var data = ExportData.PrepareDataSet(exp);
//calculate validation and training rows
int validCount = DataSet.IsPrecentige ? (int)(DataSet.TestRows * data.Count / 100.0) : DataSet.TestRows;
//in case of empty validation data set skip file creation
if (validCount == 0)
{
strPathValid = "";
}
//create training ml ready dataset file
int trainCount = data.Count - validCount;
if (trainCount <= 0)
{
throw new Exception("Train dataset is empty. Split data set on correct parts.");
}
File.WriteAllLines(strPathTrain, data.Take(trainCount).ToList());
//in case of empty validation data set skip file creation
if (validCount > 0)
{
var d = data.Skip(trainCount).Take(validCount).ToList();
File.WriteAllLines(strPathValid, d);
}
//model name and settings
mlconfig.Name = modelName;
mlconfig.Settings = Settings;
//load project file in order to get settings information and data description needed for data transformation
var strPPath = Path.Combine(Settings.ProjectFolder, Settings.ProjectFile);
var proj = new Project();
proj.Load(strPPath);
//enumerate all column and setup column information needed for mlconfig creation
foreach (var c in proj.Descriptor.Columns.Where(x => x.Type == DataType.Category && x.Kind != DataKind.None))
{
var cc = exp.GetColumns().Where(x => x.Name == c.Name && x.ColumnDataType == ColumnType.Category).FirstOrDefault();
if (cc == null)
{
throw new Exception("Column not found!");
}
c.Classes = cc.Statistics.Categories.ToArray();
}
//create mlconfig file
Project.NewMLConfigFile(proj, modelName);
//initialize model
mlconfig.Init();
//add newly created model in to Model collection of the project
Models.Add(mlconfig);
//save project with new created mlconfig
Save();
}
//initialize project controllers with project information
internal async Task <bool> Initproject(string projectPath, ProjectType pType = ProjectType.Default)
{
try
{
var fi = new FileInfo(projectPath);
if (!fi.Exists)
{
throw new Exception("Project File not found!");
}
//load project information from file
var dicData = Project.LoadProjectData(projectPath);
//load project info
Settings = Project.CreateProjectSettings(dicData["project"]);
Settings.ProjectFolder = fi.Directory.FullName;
Settings.ProjectFile = fi.Name;
//
Name = Project.GetParameterValue(dicData["project"], "Name");
var prData = dicData["data"].Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
var dataPath = Project.GetParameterValue(prData, "RawData");
var filePath = "";
if (!string.IsNullOrEmpty(dataPath))
{
filePath = Path.Combine(Settings.ProjectFolder, Name, dataPath);
}
else
{
filePath = Path.Combine(Settings.ProjectFolder, Name, Name + "_rawdata.txt");
}
//load rawdataset
var parser = Project.CreateDataParser(dicData["parser"]);
var result = await initRawData(filePath, parser);
//create dataset
var ds = new ANNDataSet();
ds.InitMetaColumn(prData.Where(x => x.StartsWith("Column")).OrderBy(x => x));
ds.Data = result.data;
ds.IsPrecentige = Settings.PercentigeSplit;
ds.RowsToValidation = Settings.ValidationSetCount;
ds.RowsToTest = Settings.TestSetCount;
DataSet = ds;
//load existing mlconfigs
var models = Project.GetMLConfigs(dicData["project"]);
foreach (var model in models)
{
var m = new MLConfigController(activeModelChanged);
m.Settings = Settings;
//
m.Name = model;
Models.Add(m);
}
return(true);
}
catch (Exception)
{
throw;
}
}
//initialize project controllers with project information
internal void Initproject(string projectPath)
{
try
{
var fi = new FileInfo(projectPath);
if (!fi.Exists)
{
throw new Exception("Project File not found!");
}
//load project information from file
var dicData = Project.LoadProjectData(projectPath);
//load project info
Settings = Project.CreateProjectSettings(dicData["project"]);
Settings.ProjectFolder = fi.Directory.FullName;
Settings.ProjectFile = fi.Name;
//
Name = Project.GetParameterValue(dicData["project"], "Name");
//check which type the project is
var strType = Project.GetParameterValue(dicData["project"], "Type");
if (string.IsNullOrEmpty(strType))
{
Type = ProjectType.Default;
}
else
{
Type = (ProjectType)Enum.Parse(typeof(ProjectType), strType, true);
}
var prData = dicData["data"].Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
var dataPath = Project.GetParameterValue(prData, "RawData");
var filePath = "";
if (!string.IsNullOrEmpty(dataPath))
{
filePath = Path.Combine(Settings.ProjectFolder, Name, dataPath);
}
else
{
filePath = Name + "_rawdata.txt";
}
//create dataset
var ds = new ANNDataSet();
ds.InitMetaColumn(prData.Where(x => x.StartsWith("Column")).OrderBy(x => x));
var parser = Project.CreateDataParser(dicData["parser"]);
//check if raw data file exists
fi = new FileInfo(filePath);
if (fi.Exists)
{
//column separator is always ;
var result = ANNDataSet.prepareData(File.ReadAllLines(filePath), parser.ColumnSeparator, parser.FirstRowHeader);
ds.Data = result.data;
ds.IsPrecentige = Settings.PrecentigeSplit;
ds.TestRows = Settings.ValidationSetCount;
DataSet = ds;
}
//load existing mlconfigs
var models = Project.GetMLConfigs(dicData["project"]);
foreach (var model in models)
{
var m = new MLConfigController(activeModelChanged);
m.Settings = Settings;
//
m.Name = model;
Models.Add(m);
}
}
catch (Exception)
{
throw;
}
}
