internal async Task <bool> CreateMLConfig(MLConfigController mlconfig)
        {
            //save project with new created mlconfig
            await Save();

            //create model name
            string modelName = $"MLConfig{Models.Count}";

            //Define ml data file paths
            var strModelFolder     = Project.GetMLConfigFolder(Settings, modelName);
            var strModelDataFolder = Project.GetMLConfigDataFolder(Settings, modelName);


            //check if model folder exists
            if (!Directory.Exists(strModelFolder))
            {
                Directory.CreateDirectory(strModelFolder);
            }
            //check if data folder exists
            if (!Directory.Exists(strModelDataFolder))
            {
                Directory.CreateDirectory(strModelDataFolder);
            }

            //model name and settings
            mlconfig.Name     = modelName;
            mlconfig.Settings = Settings;

            //load project file in order to get settings information and data description needed for data transformation
            var strPPath = Path.Combine(Settings.ProjectFolder, Settings.ProjectFile);
            var proj     = new Project();

            proj.Load(strPPath);

            //ToDo: optimizes code for huge dataset
            if (Settings.ProjectType == ProjectType.Default)
            {
                createDefaultDataSets(DataSet, modelName, proj.Descriptor);
            }
            else if (Settings.ProjectType == ProjectType.ImageClassification)
            {
                createImgClassificationDataSets(DataSet, modelName, proj.Descriptor);
            }
            else
            {
                ;
            }


            //create mlconfig file
            Project.NewMLConfigFile(proj, modelName, DataSet);

            //initialize model
            mlconfig.Init();

            //add newly created model in to Model collection of the project
            Models.Add(mlconfig);

            //save project with new created mlconfig
            return(await Save());
        }
Exemplo n.º 2
0
        internal void CreateMLConfig(MLConfigController mlconfig)
        {
            //save project with new created mlconfig
            Save();

            //create model name
            string modelName = $"MLConfig{Models.Count}";

            //Define ml data file paths
            var strModelFolder     = Project.GetMLConfigFolder(Settings, modelName);
            var strModelDataFolder = Project.GetMLConfigDataFolder(Settings, modelName);
            var strPathTrain       = Project.GetDefaultMLDatasetPath(Settings, modelName, true);
            var strPathValid       = Project.GetDefaultMLDatasetPath(Settings, modelName, false);

            //check if model folder exists
            if (!Directory.Exists(strModelFolder))
            {
                Directory.CreateDirectory(strModelFolder);
            }
            //check if data folder exists
            if (!Directory.Exists(strModelDataFolder))
            {
                Directory.CreateDirectory(strModelDataFolder);
            }

            //get dataset based on options
            var ds = DataSet.GetDataSet(DataSet.RandomizeData);

            //we want whole data set later the data will be split
            ds.TestRows = 0;
            //create experiment based created dataset
            var exp  = new Experiment(ds);
            var data = ExportData.PrepareDataSet(exp);

            //calculate validation and training rows
            int validCount = DataSet.IsPrecentige ? (int)(DataSet.TestRows * data.Count / 100.0) : DataSet.TestRows;

            //in case of empty validation data set skip file creation
            if (validCount == 0)
            {
                strPathValid = "";
            }

            //create training ml ready dataset file
            int trainCount = data.Count - validCount;

            if (trainCount <= 0)
            {
                throw new Exception("Train dataset is empty. Split data set on correct parts.");
            }
            File.WriteAllLines(strPathTrain, data.Take(trainCount).ToList());

            //in case of empty validation data set skip file creation
            if (validCount > 0)
            {
                var d = data.Skip(trainCount).Take(validCount).ToList();
                File.WriteAllLines(strPathValid, d);
            }

            //model name and settings
            mlconfig.Name     = modelName;
            mlconfig.Settings = Settings;

            //load project file in order to get settings information and data description needed for data transformation
            var strPPath = Path.Combine(Settings.ProjectFolder, Settings.ProjectFile);
            var proj     = new Project();

            proj.Load(strPPath);

            //enumerate all column and setup column information needed for mlconfig creation
            foreach (var c in proj.Descriptor.Columns.Where(x => x.Type == DataType.Category && x.Kind != DataKind.None))
            {
                var cc = exp.GetColumns().Where(x => x.Name == c.Name && x.ColumnDataType == ColumnType.Category).FirstOrDefault();
                if (cc == null)
                {
                    throw new Exception("Column not found!");
                }

                c.Classes = cc.Statistics.Categories.ToArray();
            }

            //create mlconfig file
            Project.NewMLConfigFile(proj, modelName);

            //initialize model
            mlconfig.Init();

            //add newly created model in to Model collection of the project
            Models.Add(mlconfig);

            //save project with new created mlconfig
            Save();
        }
        //initialize project controllers with project information
        internal async Task <bool> Initproject(string projectPath, ProjectType pType = ProjectType.Default)
        {
            try
            {
                var fi = new FileInfo(projectPath);
                if (!fi.Exists)
                {
                    throw new Exception("Project File not found!");
                }

                //load project information from file
                var dicData = Project.LoadProjectData(projectPath);

                //load project info
                Settings = Project.CreateProjectSettings(dicData["project"]);
                Settings.ProjectFolder = fi.Directory.FullName;
                Settings.ProjectFile   = fi.Name;

                //
                Name = Project.GetParameterValue(dicData["project"], "Name");



                var prData   = dicData["data"].Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
                var dataPath = Project.GetParameterValue(prData, "RawData");
                var filePath = "";
                if (!string.IsNullOrEmpty(dataPath))
                {
                    filePath = Path.Combine(Settings.ProjectFolder, Name, dataPath);
                }
                else
                {
                    filePath = Path.Combine(Settings.ProjectFolder, Name, Name + "_rawdata.txt");
                }

                //load rawdataset
                var parser = Project.CreateDataParser(dicData["parser"]);
                var result = await initRawData(filePath, parser);

                //create dataset
                var ds = new ANNDataSet();
                ds.InitMetaColumn(prData.Where(x => x.StartsWith("Column")).OrderBy(x => x));
                ds.Data             = result.data;
                ds.IsPrecentige     = Settings.PercentigeSplit;
                ds.RowsToValidation = Settings.ValidationSetCount;
                ds.RowsToTest       = Settings.TestSetCount;
                DataSet             = ds;

                //load existing mlconfigs
                var models = Project.GetMLConfigs(dicData["project"]);
                foreach (var model in models)
                {
                    var m = new MLConfigController(activeModelChanged);
                    m.Settings = Settings;
                    //
                    m.Name = model;
                    Models.Add(m);
                }
                return(true);
            }
            catch (Exception)
            {
                throw;
            }
        }
Exemplo n.º 4
0
        //initialize project controllers with project information
        internal void Initproject(string projectPath)
        {
            try
            {
                var fi = new FileInfo(projectPath);
                if (!fi.Exists)
                {
                    throw new Exception("Project File not found!");
                }

                //load project information from file
                var dicData = Project.LoadProjectData(projectPath);

                //load project info
                Settings = Project.CreateProjectSettings(dicData["project"]);
                Settings.ProjectFolder = fi.Directory.FullName;
                Settings.ProjectFile   = fi.Name;

                //
                Name = Project.GetParameterValue(dicData["project"], "Name");

                //check which type the project is
                var strType = Project.GetParameterValue(dicData["project"], "Type");
                if (string.IsNullOrEmpty(strType))
                {
                    Type = ProjectType.Default;
                }
                else
                {
                    Type = (ProjectType)Enum.Parse(typeof(ProjectType), strType, true);
                }

                var prData   = dicData["data"].Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
                var dataPath = Project.GetParameterValue(prData, "RawData");
                var filePath = "";
                if (!string.IsNullOrEmpty(dataPath))
                {
                    filePath = Path.Combine(Settings.ProjectFolder, Name, dataPath);
                }
                else
                {
                    filePath = Name + "_rawdata.txt";
                }

                //create dataset
                var ds = new ANNDataSet();

                ds.InitMetaColumn(prData.Where(x => x.StartsWith("Column")).OrderBy(x => x));
                var parser = Project.CreateDataParser(dicData["parser"]);

                //check if raw data file exists
                fi = new FileInfo(filePath);
                if (fi.Exists)
                {
                    //column separator is always ;
                    var result = ANNDataSet.prepareData(File.ReadAllLines(filePath), parser.ColumnSeparator, parser.FirstRowHeader);
                    ds.Data         = result.data;
                    ds.IsPrecentige = Settings.PrecentigeSplit;
                    ds.TestRows     = Settings.ValidationSetCount;
                    DataSet         = ds;
                }

                //load existing mlconfigs
                var models = Project.GetMLConfigs(dicData["project"]);
                foreach (var model in models)
                {
                    var m = new MLConfigController(activeModelChanged);
                    m.Settings = Settings;
                    //
                    m.Name = model;
                    Models.Add(m);
                }
            }
            catch (Exception)
            {
                throw;
            }
        }