コード例 #1
0
        public ImageClassificatorModel LoadDataSet(ANNDataSet dataSet)
        {
            try
            {
                if (dataSet == null || dataSet.Data == null || dataSet.MetaData == null)
                {
                    ResetPanel();
                    return(null);
                }

                var model = loadDataToModel(dataSet);
                //set
                //txtValidationCount.Text = dataSet.TestRows.ToString();
                txtValidationCount.Text    = dataSet.RowsToValidation.ToString();
                txtTestCount.Text          = dataSet.RowsToTest.ToString();
                radionNumber.IsChecked     = !dataSet.IsPrecentige;
                radionPercentige.IsChecked = dataSet.IsPrecentige;

                return(model);
            }
            catch (Exception)
            {
                throw;
            }
        }
コード例 #2
0
ファイル: ImportData.cs プロジェクト: zaharPonimash/anndotnet
        //import time series and convert to data frame
        private void btnImportTS_Click(object sender, EventArgs e)
        {
            try
            {
                if (string.IsNullOrEmpty(textBox3.Text))
                {
                    MessageBox.Show("No file is selected!");
                    return;
                }


                var colDelimiter = GetColumDelimiter();

                if (numCtrlNumForTest.Value < 1 && numCtrlNumForTest.Value > originData.Length)
                {
                    MessageBox.Show("Invalid number of time lag. Please specify the time lag between 1 and row number.");
                    return;
                }

                if (string.IsNullOrEmpty(originData))
                {
                    return;
                }

                //
                //transform the time series into data frame
                var result = ANNDataSet.prepareTimeSeriesData(originLines, (int)numCtrlNumForTest.Value, colDelimiter, firstRowHeaderCheck.Checked);
                Header = result.header;
                Data   = result.data;
            }
            catch (Exception ex)
            {
                reportException(ex);
            }
        }
コード例 #3
0
        public ANNDataSet GetDataSet(bool omitIgnored = false)
        {
            try
            {
                var data1 = new ANNDataSet();
                //
                data1.MetaData = ParseHeader(omitIgnored);

                data1.TestRows = int.Parse(txtValidationCount.Text);
                //data1.RowsToValidation = int.Parse(txtValidationCount.Text);
                //data1.RowsToTest = int.Parse(txtTestCount.Text);

                data1.IsPrecentige = radionPercentige.IsChecked.Value;

                var strData = ParseData(data1.MetaData);
                if (strData == null)
                {
                    return(null);
                }
                data1.RandomizeData = checkRandomizeDataset.IsChecked.Value;
                //
                var sss = strData.Select(x => x.ToArray()).ToArray();
                data1.Data = sss;
                //data1.Data = strData;
                return(data1);
            }
            catch (Exception)
            {
                throw;
            }
        }
コード例 #4
0
        private ImageClassificatorModel loadDataToModel(ANNDataSet dataSet)
        {
            var model = new ImageClassificatorModel();

            if (dataSet != null && dataSet.Data.Count > 0)
            {
                //
                var row = dataSet.Data.First();
                model.Channels = int.Parse(row[3]);
                model.Height   = int.Parse(row[4]);
                model.Width    = int.Parse(row[5]);
                foreach (var r in dataSet.Data)
                {
                    var itm = new ImageLabelItem();
                    itm.Label  = r[0];
                    itm.Folder = r[1];
                    itm.Query  = r[2];
                    model.Labels.Add(itm);
                }
                return(model);
            }
            else
            {
                return(null);
            }
        }
コード例 #5
0
ファイル: ImportData.cs プロジェクト: zaharPonimash/anndotnet
        // import dataset into data frame
        private void btnImportData_Click(object sender, EventArgs e)
        {
            try
            {
                if (string.IsNullOrEmpty(textBox3.Text))
                {
                    MessageBox.Show("No file is selected!");
                    return;
                }


                var colDelimiter = GetColumDelimiter();
                //define the row
                //string[] rows = originData.Split(Environment.NewLine.ToArray(), StringSplitOptions.RemoveEmptyEntries);
                this.Cursor = Cursors.WaitCursor;
                var result = ANNDataSet.prepareData(originLines, colDelimiter, firstRowHeaderCheck.Checked, radioButton1.Checked);

                Header = result.header;
                Data   = result.data;
            }
            catch (Exception ex)
            {
                reportException(ex);
            }
            finally
            {
                this.Cursor = Cursors.Arrow;
            }
        }
コード例 #6
0
        public ANNDataSet GetDataSet()
        {
            try
            {
                var Model = this.DataContext as ImageClassificatorModel;
                if (Model != null && Model.Labels != null)
                {
                    ANNDataSet ds = new ANNDataSet();
                    ds.MetaData = new MetaColumn[]
                    {
                        new MetaColumn()
                        {
                            Id = 0, Index = 0, Name = "Image", Type = "Image", MissingValue = "None", Param = VariableType.Feature.ToString(),
                        },
                        new MetaColumn()
                        {
                            Id = 0, Index = 0, Name = "Label", Type = "Category", MissingValue = "None", Param = VariableType.Label.ToString(),
                        }
                    };
                    ds.Data = new List <List <string> >();
                    foreach (var l in Model.Labels)
                    {
                        if (string.IsNullOrEmpty(l.Label) || string.IsNullOrEmpty(l.Folder) ||
                            Model.Channels <= 0 || Model.Width <= 0 || Model.Height <= 0)
                        {
                            throw new Exception("One or more image classification data parameters has invalid value!");
                        }

                        var strRow = new List <string>()
                        {
                            l.Label, l.Folder, l.Query, Model.Channels.ToString(),
                            Model.Height.ToString(), Model.Width.ToString(), Model.DataAugmentation.ToString()
                        };
                        ds.Data.Add(strRow);
                    }

                    //data1.TestRows = int.Parse(txtValidationCount.Text);
                    ds.RowsToValidation = int.Parse(txtValidationCount.Text);
                    ds.RowsToTest       = int.Parse(txtTestCount.Text);

                    ds.IsPrecentige  = radionPercentige.IsChecked.Value;
                    ds.RandomizeData = checkRandomizeDataset.IsChecked.Value;

                    return(ds);
                }
                else
                {
                    return(null);
                }
            }
            catch (Exception)
            {
                throw;
            }
        }
コード例 #7
0
        private ANNDataSet getDataSet(ProjectType projectType)
        {
            var ds      = new ANNDataSet();
            var cntCtrl = anndotnet.wnd.App.Current.MainWindow as MainWindow;
            var tab     = FindVisualChild <TabControl>(cntCtrl.content);

            if (tab == null)
            {
                return(null);
            }

            //
            if (projectType == ProjectType.Default)
            {
                var tbItm = tab.Items[0] as TabItem;
                if (tbItm == null)
                {
                    return(null);
                }

                var expCtrl = FindVisualChild <DataPanelWPF>(tbItm.Content as Grid);
                if (expCtrl == null)
                {
                    return(null);
                }
                ds = expCtrl.GetDataSet();
            }
            else
            {
                var tbItm = tab.Items[1] as TabItem;
                if (tbItm == null)
                {
                    return(null);
                }

                var ctrl = FindVisualChild <ImageClassificator>(tbItm.Content as Grid);

                //
                if (ctrl == null)
                {
                    return(null);
                }
                //
                ds = ctrl.GetDataSet();
            }

            setCategoryEncoding(ds);
            return(ds);
        }
コード例 #8
0
        private void setCategoryEncoding(ANNDataSet dataSet)
        {
            if (dataSet == null || dataSet.MetaData == null)
            {
                return;
            }

            foreach (var col in dataSet.MetaData)
            {
                if (col.Type == ColumnType.Category.ToString())
                {
                    col.Encoding = CategoryEncoding.OneHot.ToString();
                }
            }

            //extract the categories from
        }
コード例 #9
0
ファイル: DataPanel.cs プロジェクト: softics/anndotnet
        public void SetDataSet(ANNDataSet dataSet)
        {
            if (dataSet == null || dataSet.MetaData == null)
            {
                ResetExperimentalPanel();
                return;
            }
            //header
            m_strHeader = dataSet.MetaData.Select(x => x.Name).ToArray();
            setColumn(dataSet.MetaData.ToList());
            //Data
            setData(dataSet.Data);
            //summary
            setSummary(m_strData, dataSet.MetaData.ToList());

            //set
            numCtrlNumForTest.Value = dataSet.TestRows;
            numberRadio.Checked     = !dataSet.IsPrecentige;
            presentigeRadio.Checked = dataSet.IsPrecentige;
        }
コード例 #10
0
ファイル: ImportData.cs プロジェクト: braegelno5/anndotnet
        //import data as time series
        private void button4_Click(object sender, EventArgs e)
        {
            try
            {
                if (string.IsNullOrEmpty(textBox3.Text))
                {
                    MessageBox.Show("No file is selected!");
                    return;
                }


                var colDelimiter = GetColumDelimiter();

                if (numCtrlNumForTest.Value < 1 && numCtrlNumForTest.Value > originData.Length)
                {
                    MessageBox.Show("Invalid number of time leg. PLease specify the time leg between 1 and row number.");
                    return;
                }

                if (string.IsNullOrEmpty(originData))
                {
                    return;
                }

                //
                //define the row
                string[] tdata = originData.Split(Environment.NewLine.ToArray(), StringSplitOptions.RemoveEmptyEntries);
                //transform the time series into data frame
                string[] prepData = prepareTimeSeriesData(tdata, (int)numCtrlNumForTest.Value);

                //prepare data for loading
                var result = ANNDataSet.prepareData(prepData, new char[] { ';' }, checkBox1.Checked, radioButton1.Checked);
                Header = result.header;
                Data   = result.data;
            }
            catch (Exception ex)
            {
                reportException(ex);
            }
        }
コード例 #11
0
        public void SetDataSet(ANNDataSet dataSet)
        {
            if (dataSet == null || dataSet.Data == null || dataSet.MetaData == null)
            {
                ResetExperimentalPanel();
                return;
            }
            //header
            m_strHeader = dataSet.MetaData.Select(x => x.Name).ToArray();
            setColumn(dataSet.MetaData.ToList());
            //Data
            setData(dataSet.Data.Select(x => x.ToList()).ToList());
            //summary
            setSummary(m_strData, dataSet.MetaData.ToList());

            //set
            txtValidationCount.Text = dataSet.TestRows.ToString();
            //txtValidationCount.Text = dataSet.RowsToValidation.ToString();
            //txtTestCount.Text = dataSet.RowsToTest.ToString();
            radionNumber.IsChecked     = !dataSet.IsPrecentige;
            radionPercentige.IsChecked = dataSet.IsPrecentige;
        }
コード例 #12
0
ファイル: DataPanel.cs プロジェクト: softics/anndotnet
        public ANNDataSet GetDataSet(bool omitIgnored = false)
        {
            try
            {
                var data1 = new ANNDataSet();
                //
                data1.MetaData = ParseHeader(omitIgnored);

                data1.TestRows     = (int)numCtrlNumForTest.Value;
                data1.IsPrecentige = !numberRadio.Checked;

                var strData = ParseData(data1.MetaData);
                data1.RandomizeData = randomoizeDataSet.Checked;
                //
                data1.Data = strData;
                return(data1);
            }
            catch (Exception)
            {
                throw;
            }
        }
コード例 #13
0
        private ImageClassificatorModel loadDataToModel(ANNDataSet dataSet)
        {
            try
            {
                var model = new ImageClassificatorModel();
                if (dataSet != null && dataSet.Data.Count > 0)
                {
                    //
                    var row = dataSet.Data.First();
                    model.Channels = int.Parse(row[3]);
                    model.Height   = int.Parse(row[4]);
                    model.Width    = int.Parse(row[5]);

                    model.DataAugmentation = int.Parse(row[6]);


                    //extract each image label
                    foreach (var r in dataSet.Data)
                    {
                        var itm = new ImageLabelItem();
                        itm.Label  = r[0];
                        itm.Folder = r[1];
                        itm.Query  = r[2] == "n/a"?"":r[2];
                        model.Labels.Add(itm);
                    }
                    return(model);
                }
                else
                {
                    return(null);
                }
            }
            catch (Exception)
            {
                throw;
            }
        }
コード例 #14
0
ファイル: ImportData.cs プロジェクト: braegelno5/anndotnet
        private void button2_Click(object sender, EventArgs e)
        {
            try
            {
                if (string.IsNullOrEmpty(textBox3.Text))
                {
                    MessageBox.Show("No file is selected!");
                    return;
                }


                var colDelimiter = GetColumDelimiter();
                //define the row
                string[] rows = originData.Split(Environment.NewLine.ToArray(), StringSplitOptions.RemoveEmptyEntries);

                var result = ANNDataSet.prepareData(rows, colDelimiter, checkBox1.Checked, radioButton1.Checked);
                Header = result.header;
                Data   = result.data;
            }
            catch (Exception ex)
            {
                reportException(ex);
            }
        }
コード例 #15
0
        //initialize project controllers with project information
        internal void Initproject(string projectPath)
        {
            try
            {
                var fi = new FileInfo(projectPath);
                if (!fi.Exists)
                {
                    throw new Exception("Project File not found!");
                }

                //load project information from file
                var dicData = Project.LoadProjectData(projectPath);

                //load project info
                Settings = Project.CreateProjectSettings(dicData["project"]);
                Settings.ProjectFolder = fi.Directory.FullName;
                Settings.ProjectFile   = fi.Name;

                //
                Name = Project.GetParameterValue(dicData["project"], "Name");

                //check which type the project is
                var strType = Project.GetParameterValue(dicData["project"], "Type");
                if (string.IsNullOrEmpty(strType))
                {
                    Type = ProjectType.Default;
                }
                else
                {
                    Type = (ProjectType)Enum.Parse(typeof(ProjectType), strType, true);
                }

                var prData   = dicData["data"].Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
                var dataPath = Project.GetParameterValue(prData, "RawData");
                var filePath = "";
                if (!string.IsNullOrEmpty(dataPath))
                {
                    filePath = Path.Combine(Settings.ProjectFolder, Name, dataPath);
                }
                else
                {
                    filePath = Name + "_rawdata.txt";
                }

                //create dataset
                var ds = new ANNDataSet();

                ds.InitMetaColumn(prData.Where(x => x.StartsWith("Column")).OrderBy(x => x));
                var parser = Project.CreateDataParser(dicData["parser"]);

                //check if raw data file exists
                fi = new FileInfo(filePath);
                if (fi.Exists)
                {
                    //column separator is always ;
                    var result = ANNDataSet.prepareData(File.ReadAllLines(filePath), parser.ColumnSeparator, parser.FirstRowHeader);
                    ds.Data         = result.data;
                    ds.IsPrecentige = Settings.PrecentigeSplit;
                    ds.TestRows     = Settings.ValidationSetCount;
                    DataSet         = ds;
                }

                //load existing mlconfigs
                var models = Project.GetMLConfigs(dicData["project"]);
                foreach (var model in models)
                {
                    var m = new MLConfigController(activeModelChanged);
                    m.Settings = Settings;
                    //
                    m.Name = model;
                    Models.Add(m);
                }
            }
            catch (Exception)
            {
                throw;
            }
        }
コード例 #16
0
ファイル: Project.cs プロジェクト: bhrnjica/anndotnet
        /// <summary>
        /// Creates a new ANNdotNET Model configuration file.
        /// </summary>
        /// <param name="project"></param>
        /// <param name="mlconfigName"></param>
        /// <returns></returns>
        public static bool NewMLConfigFile(Project project, string mlconfigName, ANNDataSet ds)
        {
            try
            {
                var strFeatures = $"features:";

                //first create features and labels based on data descriptor
                var numDim = project.Descriptor.Columns.Where(x => x.Kind == DataKind.Feature && x.Type == MLDataType.Numeric).Count();
                if (numDim > 0)
                {
                    strFeatures += $"{ProjectSettings.m_NumFeaturesGroupName} 1;{numDim} 0\t";
                }

                //create features for image
                var imgCol = project.Descriptor.Columns.Where(x => x.Kind == DataKind.Feature && x.Type == MLDataType.Image).FirstOrDefault();
                if (imgCol != null)
                {
                    var shape = imgCol.Shape + ";" + ds.Data.First().Last();
                    strFeatures += $"|{imgCol.Name} {shape} 0\t";
                }


                //create category features
                foreach (var c in project.Descriptor.Columns.Where(x => x.Kind == DataKind.Feature && x.Type == MLDataType.Category))
                {
                    strFeatures += $"|{c.Name} 1;{c.Classes.Length} 0\t";
                }
                //create label
                var strLabel = $"labels:";
                foreach (var c in project.Descriptor.Columns.Where(x => x.Kind == DataKind.Label && x.Type == MLDataType.Category))
                {
                    strLabel += $"|{c.Name} {c.Classes.Length} 0\t";
                }
                //
                foreach (var c in project.Descriptor.Columns.Where(x => x.Kind == DataKind.Label && x.Type == MLDataType.Numeric))
                {
                    strLabel += $"|{c.Name} {1} 0\t";
                }

                //only one Label is supported
                var countLabel = project.Descriptor.Columns.Where(x => x.Kind == DataKind.Label && x.Type != MLDataType.None).Count();
                if (countLabel == 0)
                {
                    throw new Exception("The mlconfig cannot be created, no label is defined!");
                }
                if (countLabel > 1)
                {
                    throw new Exception("The mlconfig cannot be created, more than one label is defined!");
                }
                //create rest of the ML config file.
                //later the user will be able to setup model training and other params
                var strMlCOnfig = new List <string>();

                //first we have to generate unique model identified
                var modeId = $"configid:{Guid.NewGuid()}";
                strMlCOnfig.Add(modeId);

                //then generate metadat
                var strMetaData = $"metadata:{project.Descriptor.ToMetadataString(true)}";
                strMlCOnfig.Add(strMetaData);

                //add features and labels previously generated
                strMlCOnfig.Add(strFeatures);
                strMlCOnfig.Add(strLabel);

                //empty network
                var mbType = project.Settings.ProjectType == ProjectType.ImageClassification? MinibatchType.Image : MinibatchType.Default;
                strMlCOnfig.Add("network:|Layer:Dense 1 0 0 None 0 0");
                strMlCOnfig.Add("learning:|Type:SGDLearner |LRate:0.01 |Momentum:1 |Loss:SquaredError |Eval:SquaredError");
                strMlCOnfig.Add($"training:|Type:{mbType} |BatchSize:50 |Epochs:100 |Normalization:0 |RandomizeBatch:0 |SaveWhileTraining:1 |ProgressFrequency:100  |ContinueTraining:0 |TrainedModel: ");
                var strDicts = GetDefaultMLConfigPaths(project.Settings, mlconfigName);

                //add paths for ml config
                foreach (var d in strDicts)
                {
                    strMlCOnfig.Add($"{d.Key}:{d.Value}");
                }
                //
                var mlConfigPath = Project.GetMLConfigPath(project.Settings, mlconfigName);
                File.WriteAllLines(mlConfigPath, strMlCOnfig);
            }
            catch (Exception)
            {
                throw;
            }

            return(true);
        }
コード例 #17
0
        /// <summary>
        /// Parses the information from the Raw Data and created mlready data sets for default project
        /// </summary>
        /// <param name="dataSet"></param>
        /// <param name="modelName"></param>
        /// <param name="dataDesc"></param>
        private void createDefaultDataSets(ANNDataSet dataSet, string modelName, DataDescriptor dataDesc)
        {
            var strPathTrain = Project.GetDefaultMLDatasetPath(Settings, modelName, DataSetType.Training);
            var strPathValid = Project.GetDefaultMLDatasetPath(Settings, modelName, DataSetType.Validation);
            var strPathTest  = Project.GetDefaultMLDatasetPath(Settings, modelName, DataSetType.Testing);

            //get dataset based on options
            var ds = dataSet.GetDataSet(dataSet.RandomizeData);

            //we want whole data set later the data will be split
            ds.RowsToValidation = 0;
            ds.RowsToTest       = 0;
            //create experiment based created dataset
            var exp  = new DataFrame(ds);
            var data = ExportData.PrepareDataSet(exp);

            //calculate validation and training rows
            int validCount = dataSet.IsPrecentige ? (int)(dataSet.RowsToValidation * data.Count / 100.0) : dataSet.RowsToValidation;

            //in case of empty validation data set skip file creation
            if (validCount == 0)
            {
                strPathValid = "";
            }

            //calculate testing rows
            int testCount = dataSet.IsPrecentige ? (int)(dataSet.RowsToTest * data.Count / 100.0) : dataSet.RowsToValidation;

            //in case of empty validation data set skip file creation
            if (testCount == 0)
            {
                strPathTest = "";
            }

            //create training ml ready dataset file
            int trainCount = data.Count - validCount - testCount;

            //check if the training count number valid
            if (trainCount <= 0)
            {
                throw new Exception("Train dataset is empty. Split data set on correct parts.");
            }
            File.WriteAllLines(strPathTrain, data.Take(trainCount).ToList());

            //in case of empty validation data set skip file creation
            if (validCount > 0)
            {
                var d = data.Skip(trainCount).Take(validCount).ToList();
                File.WriteAllLines(strPathValid, d);
            }
            //in case of empty validation data set skip file creation
            if (testCount > 0)
            {
                var d = data.Skip(trainCount + validCount).Take(testCount).ToList();
                File.WriteAllLines(strPathTest, d);
            }


            //enumerate all column and setup column information needed for mlconfig creation
            foreach (var c in dataDesc.Columns.Where(x => x.Type == MLDataType.Category && x.Kind != DataKind.None))
            {
                var cc = exp.GetColumns().Where(x => x.Name == c.Name && x.ColumnDataType == ColumnType.Category).FirstOrDefault();
                if (cc == null)
                {
                    throw new Exception("Column not found!");
                }

                c.Classes = cc.Statistics.Categories.ToArray();
            }
        }
コード例 #18
0
        /// <summary>
        /// Parses the information from the Raw Data and created mlready data sets for image classification project
        /// </summary>
        /// <param name="dataSet"></param>
        /// <param name="modelName"></param>
        /// <param name="descriptor"></param>
        private void createImgClassificationDataSets(ANNDataSet dataSet, string modelName, DataDescriptor descriptor)
        {
            //define daata paths
            var strPathTrain = Project.GetDefaultMLDatasetPath(Settings, modelName, DataSetType.Training);
            var strPathValid = Project.GetDefaultMLDatasetPath(Settings, modelName, DataSetType.Validation);
            var strPathTest  = Project.GetDefaultMLDatasetPath(Settings, modelName, DataSetType.Testing);

            //
            var data = DataSet.Data;

            //retrieve image information form files
            var trainingData   = new List <string>();
            var validationData = new List <string>();
            var testData       = new List <string>();
            int dataCount      = 0;
            //
            int cahannel = int.Parse(data[0][3]);
            int height   = int.Parse(data[0][4]);
            int width    = int.Parse(data[0][5]);

            //first count how many images we are handling
            for (int i = 0; i < data.Count; i++)
            {
                //ith label
                var      label  = data[i][0];
                var      folder = data[i][1];
                var      query  = data[i][2];
                string[] images = null;

                if (string.IsNullOrEmpty(query) || query == " ")
                {
                    images = Directory.GetFiles(folder);
                }
                else
                {
                    images = Directory.GetFiles(folder, query, SearchOption.TopDirectoryOnly);
                }
                //
                if (images == null || images.Length == 0)
                {
                    throw new Exception($"No images have been found for '{label}' label. Folder path must not be empty.");
                }

                dataCount += images.Length;
            }

            //create data sets
            var classesList = new List <string>();

            for (int i = 0; i < data.Count; i++)
            {
                //i-th label
                var label = data[i][0];
                //add class into list
                classesList.Add(label);
                string lblIndex = i.ToString();
                //
                var      folder = data[i][1];
                var      query  = data[i][2];
                string[] images = null;

                if (string.IsNullOrEmpty(query) || query == " ")
                {
                    images = Directory.GetFiles(folder);
                }
                else
                {
                    images = Directory.GetFiles(folder, query, SearchOption.TopDirectoryOnly);
                }
                //
                if (images == null || images.Length == 0)
                {
                    throw new Exception($"No images have been found for '{label}' label. Folder path must not be empty.");
                }

                //construct the rows of map files
                var strValue = images.Select(x => $"{x}\t{lblIndex}");

                //calculate amount of rows to put into train, valid and test data set
                var validCount = strValue.Count() * dataSet.RowsToValidation / 100.0f;
                var testCount  = strValue.Count() * dataSet.RowsToTest / 100.0f;
                var trainCount = strValue.Count() - (int)validCount - (int)testCount;

                //add to train set
                trainingData.AddRange(strValue.Take(trainCount));

                //add to valid set
                if (validCount > 0)
                {
                    validationData.AddRange(strValue.Skip(trainCount).Take((int)validCount));
                }
                //add to test set
                if (testCount > 0)
                {
                    testData.AddRange(strValue.Skip((int)trainCount + (int)validCount).Take((int)testCount));
                }
            }

            //define metadata needed for MLConfiguration creation
            //Label columm classes definition
            descriptor.Columns.Last().Classes = classesList.ToArray();
            //image shape definition
            descriptor.Columns.First().Shape = $"{cahannel};{height};{width}";


            //check if the training count number valid
            if (trainingData.Count() <= 0)
            {
                throw new Exception("Train dataset is empty. Split data set on correct parts.");
            }
            File.WriteAllLines(strPathTrain, trainingData);

            //in case of empty validation data set skip file creation
            if (validationData.Count() > 0)
            {
                File.WriteAllLines(strPathValid, validationData);
            }
            //in case of empty validation data set skip file creation
            if (testData.Count() > 0)
            {
                File.WriteAllLines(strPathTest, testData);
            }
        }
コード例 #19
0
        private async Task <(List <string> header, List <List <string> > data)> initRawData(string filePath, DataParser parser)
        {
            //check if raw data file exists
            var fi = new FileInfo(filePath);

            if (fi.Exists)
            {
                //
                var result = await Task.Run <(List <string> header, List <List <string> > data)>(() => ANNDataSet.prepareDataFromFile(filePath, parser.ColumnSeparator, parser.FirstRowHeader, loadProjectProgress));

                return(result);
            }
            else
            {
                return(null, null);
            }
        }
コード例 #20
0
        //initialize project controllers with project information
        internal async Task <bool> Initproject(string projectPath, ProjectType pType = ProjectType.Default)
        {
            try
            {
                var fi = new FileInfo(projectPath);
                if (!fi.Exists)
                {
                    throw new Exception("Project File not found!");
                }

                //load project information from file
                var dicData = Project.LoadProjectData(projectPath);

                //load project info
                Settings = Project.CreateProjectSettings(dicData["project"]);
                Settings.ProjectFolder = fi.Directory.FullName;
                Settings.ProjectFile   = fi.Name;

                //
                Name = Project.GetParameterValue(dicData["project"], "Name");



                var prData   = dicData["data"].Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
                var dataPath = Project.GetParameterValue(prData, "RawData");
                var filePath = "";
                if (!string.IsNullOrEmpty(dataPath))
                {
                    filePath = Path.Combine(Settings.ProjectFolder, Name, dataPath);
                }
                else
                {
                    filePath = Path.Combine(Settings.ProjectFolder, Name, Name + "_rawdata.txt");
                }

                //load rawdataset
                var parser = Project.CreateDataParser(dicData["parser"]);
                var result = await initRawData(filePath, parser);

                //create dataset
                var ds = new ANNDataSet();
                ds.InitMetaColumn(prData.Where(x => x.StartsWith("Column")).OrderBy(x => x));
                ds.Data             = result.data;
                ds.IsPrecentige     = Settings.PercentigeSplit;
                ds.RowsToValidation = Settings.ValidationSetCount;
                ds.RowsToTest       = Settings.TestSetCount;
                DataSet             = ds;

                //load existing mlconfigs
                var models = Project.GetMLConfigs(dicData["project"]);
                foreach (var model in models)
                {
                    var m = new MLConfigController(activeModelChanged);
                    m.Settings = Settings;
                    //
                    m.Name = model;
                    Models.Add(m);
                }
                return(true);
            }
            catch (Exception)
            {
                throw;
            }
        }