public ImageClassificatorModel LoadDataSet(ANNDataSet dataSet) { try { if (dataSet == null || dataSet.Data == null || dataSet.MetaData == null) { ResetPanel(); return(null); } var model = loadDataToModel(dataSet); //set //txtValidationCount.Text = dataSet.TestRows.ToString(); txtValidationCount.Text = dataSet.RowsToValidation.ToString(); txtTestCount.Text = dataSet.RowsToTest.ToString(); radionNumber.IsChecked = !dataSet.IsPrecentige; radionPercentige.IsChecked = dataSet.IsPrecentige; return(model); } catch (Exception) { throw; } }
//import time series and convert to data frame private void btnImportTS_Click(object sender, EventArgs e) { try { if (string.IsNullOrEmpty(textBox3.Text)) { MessageBox.Show("No file is selected!"); return; } var colDelimiter = GetColumDelimiter(); if (numCtrlNumForTest.Value < 1 && numCtrlNumForTest.Value > originData.Length) { MessageBox.Show("Invalid number of time lag. Please specify the time lag between 1 and row number."); return; } if (string.IsNullOrEmpty(originData)) { return; } // //transform the time series into data frame var result = ANNDataSet.prepareTimeSeriesData(originLines, (int)numCtrlNumForTest.Value, colDelimiter, firstRowHeaderCheck.Checked); Header = result.header; Data = result.data; } catch (Exception ex) { reportException(ex); } }
public ANNDataSet GetDataSet(bool omitIgnored = false) { try { var data1 = new ANNDataSet(); // data1.MetaData = ParseHeader(omitIgnored); data1.TestRows = int.Parse(txtValidationCount.Text); //data1.RowsToValidation = int.Parse(txtValidationCount.Text); //data1.RowsToTest = int.Parse(txtTestCount.Text); data1.IsPrecentige = radionPercentige.IsChecked.Value; var strData = ParseData(data1.MetaData); if (strData == null) { return(null); } data1.RandomizeData = checkRandomizeDataset.IsChecked.Value; // var sss = strData.Select(x => x.ToArray()).ToArray(); data1.Data = sss; //data1.Data = strData; return(data1); } catch (Exception) { throw; } }
private ImageClassificatorModel loadDataToModel(ANNDataSet dataSet) { var model = new ImageClassificatorModel(); if (dataSet != null && dataSet.Data.Count > 0) { // var row = dataSet.Data.First(); model.Channels = int.Parse(row[3]); model.Height = int.Parse(row[4]); model.Width = int.Parse(row[5]); foreach (var r in dataSet.Data) { var itm = new ImageLabelItem(); itm.Label = r[0]; itm.Folder = r[1]; itm.Query = r[2]; model.Labels.Add(itm); } return(model); } else { return(null); } }
// import dataset into data frame private void btnImportData_Click(object sender, EventArgs e) { try { if (string.IsNullOrEmpty(textBox3.Text)) { MessageBox.Show("No file is selected!"); return; } var colDelimiter = GetColumDelimiter(); //define the row //string[] rows = originData.Split(Environment.NewLine.ToArray(), StringSplitOptions.RemoveEmptyEntries); this.Cursor = Cursors.WaitCursor; var result = ANNDataSet.prepareData(originLines, colDelimiter, firstRowHeaderCheck.Checked, radioButton1.Checked); Header = result.header; Data = result.data; } catch (Exception ex) { reportException(ex); } finally { this.Cursor = Cursors.Arrow; } }
public ANNDataSet GetDataSet() { try { var Model = this.DataContext as ImageClassificatorModel; if (Model != null && Model.Labels != null) { ANNDataSet ds = new ANNDataSet(); ds.MetaData = new MetaColumn[] { new MetaColumn() { Id = 0, Index = 0, Name = "Image", Type = "Image", MissingValue = "None", Param = VariableType.Feature.ToString(), }, new MetaColumn() { Id = 0, Index = 0, Name = "Label", Type = "Category", MissingValue = "None", Param = VariableType.Label.ToString(), } }; ds.Data = new List <List <string> >(); foreach (var l in Model.Labels) { if (string.IsNullOrEmpty(l.Label) || string.IsNullOrEmpty(l.Folder) || Model.Channels <= 0 || Model.Width <= 0 || Model.Height <= 0) { throw new Exception("One or more image classification data parameters has invalid value!"); } var strRow = new List <string>() { l.Label, l.Folder, l.Query, Model.Channels.ToString(), Model.Height.ToString(), Model.Width.ToString(), Model.DataAugmentation.ToString() }; ds.Data.Add(strRow); } //data1.TestRows = int.Parse(txtValidationCount.Text); ds.RowsToValidation = int.Parse(txtValidationCount.Text); ds.RowsToTest = int.Parse(txtTestCount.Text); ds.IsPrecentige = radionPercentige.IsChecked.Value; ds.RandomizeData = checkRandomizeDataset.IsChecked.Value; return(ds); } else { return(null); } } catch (Exception) { throw; } }
private ANNDataSet getDataSet(ProjectType projectType) { var ds = new ANNDataSet(); var cntCtrl = anndotnet.wnd.App.Current.MainWindow as MainWindow; var tab = FindVisualChild <TabControl>(cntCtrl.content); if (tab == null) { return(null); } // if (projectType == ProjectType.Default) { var tbItm = tab.Items[0] as TabItem; if (tbItm == null) { return(null); } var expCtrl = FindVisualChild <DataPanelWPF>(tbItm.Content as Grid); if (expCtrl == null) { return(null); } ds = expCtrl.GetDataSet(); } else { var tbItm = tab.Items[1] as TabItem; if (tbItm == null) { return(null); } var ctrl = FindVisualChild <ImageClassificator>(tbItm.Content as Grid); // if (ctrl == null) { return(null); } // ds = ctrl.GetDataSet(); } setCategoryEncoding(ds); return(ds); }
private void setCategoryEncoding(ANNDataSet dataSet) { if (dataSet == null || dataSet.MetaData == null) { return; } foreach (var col in dataSet.MetaData) { if (col.Type == ColumnType.Category.ToString()) { col.Encoding = CategoryEncoding.OneHot.ToString(); } } //extract the categories from }
public void SetDataSet(ANNDataSet dataSet) { if (dataSet == null || dataSet.MetaData == null) { ResetExperimentalPanel(); return; } //header m_strHeader = dataSet.MetaData.Select(x => x.Name).ToArray(); setColumn(dataSet.MetaData.ToList()); //Data setData(dataSet.Data); //summary setSummary(m_strData, dataSet.MetaData.ToList()); //set numCtrlNumForTest.Value = dataSet.TestRows; numberRadio.Checked = !dataSet.IsPrecentige; presentigeRadio.Checked = dataSet.IsPrecentige; }
//import data as time series private void button4_Click(object sender, EventArgs e) { try { if (string.IsNullOrEmpty(textBox3.Text)) { MessageBox.Show("No file is selected!"); return; } var colDelimiter = GetColumDelimiter(); if (numCtrlNumForTest.Value < 1 && numCtrlNumForTest.Value > originData.Length) { MessageBox.Show("Invalid number of time leg. PLease specify the time leg between 1 and row number."); return; } if (string.IsNullOrEmpty(originData)) { return; } // //define the row string[] tdata = originData.Split(Environment.NewLine.ToArray(), StringSplitOptions.RemoveEmptyEntries); //transform the time series into data frame string[] prepData = prepareTimeSeriesData(tdata, (int)numCtrlNumForTest.Value); //prepare data for loading var result = ANNDataSet.prepareData(prepData, new char[] { ';' }, checkBox1.Checked, radioButton1.Checked); Header = result.header; Data = result.data; } catch (Exception ex) { reportException(ex); } }
public void SetDataSet(ANNDataSet dataSet) { if (dataSet == null || dataSet.Data == null || dataSet.MetaData == null) { ResetExperimentalPanel(); return; } //header m_strHeader = dataSet.MetaData.Select(x => x.Name).ToArray(); setColumn(dataSet.MetaData.ToList()); //Data setData(dataSet.Data.Select(x => x.ToList()).ToList()); //summary setSummary(m_strData, dataSet.MetaData.ToList()); //set txtValidationCount.Text = dataSet.TestRows.ToString(); //txtValidationCount.Text = dataSet.RowsToValidation.ToString(); //txtTestCount.Text = dataSet.RowsToTest.ToString(); radionNumber.IsChecked = !dataSet.IsPrecentige; radionPercentige.IsChecked = dataSet.IsPrecentige; }
public ANNDataSet GetDataSet(bool omitIgnored = false) { try { var data1 = new ANNDataSet(); // data1.MetaData = ParseHeader(omitIgnored); data1.TestRows = (int)numCtrlNumForTest.Value; data1.IsPrecentige = !numberRadio.Checked; var strData = ParseData(data1.MetaData); data1.RandomizeData = randomoizeDataSet.Checked; // data1.Data = strData; return(data1); } catch (Exception) { throw; } }
private ImageClassificatorModel loadDataToModel(ANNDataSet dataSet) { try { var model = new ImageClassificatorModel(); if (dataSet != null && dataSet.Data.Count > 0) { // var row = dataSet.Data.First(); model.Channels = int.Parse(row[3]); model.Height = int.Parse(row[4]); model.Width = int.Parse(row[5]); model.DataAugmentation = int.Parse(row[6]); //extract each image label foreach (var r in dataSet.Data) { var itm = new ImageLabelItem(); itm.Label = r[0]; itm.Folder = r[1]; itm.Query = r[2] == "n/a"?"":r[2]; model.Labels.Add(itm); } return(model); } else { return(null); } } catch (Exception) { throw; } }
private void button2_Click(object sender, EventArgs e) { try { if (string.IsNullOrEmpty(textBox3.Text)) { MessageBox.Show("No file is selected!"); return; } var colDelimiter = GetColumDelimiter(); //define the row string[] rows = originData.Split(Environment.NewLine.ToArray(), StringSplitOptions.RemoveEmptyEntries); var result = ANNDataSet.prepareData(rows, colDelimiter, checkBox1.Checked, radioButton1.Checked); Header = result.header; Data = result.data; } catch (Exception ex) { reportException(ex); } }
//initialize project controllers with project information internal void Initproject(string projectPath) { try { var fi = new FileInfo(projectPath); if (!fi.Exists) { throw new Exception("Project File not found!"); } //load project information from file var dicData = Project.LoadProjectData(projectPath); //load project info Settings = Project.CreateProjectSettings(dicData["project"]); Settings.ProjectFolder = fi.Directory.FullName; Settings.ProjectFile = fi.Name; // Name = Project.GetParameterValue(dicData["project"], "Name"); //check which type the project is var strType = Project.GetParameterValue(dicData["project"], "Type"); if (string.IsNullOrEmpty(strType)) { Type = ProjectType.Default; } else { Type = (ProjectType)Enum.Parse(typeof(ProjectType), strType, true); } var prData = dicData["data"].Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries); var dataPath = Project.GetParameterValue(prData, "RawData"); var filePath = ""; if (!string.IsNullOrEmpty(dataPath)) { filePath = Path.Combine(Settings.ProjectFolder, Name, dataPath); } else { filePath = Name + "_rawdata.txt"; } //create dataset var ds = new ANNDataSet(); ds.InitMetaColumn(prData.Where(x => x.StartsWith("Column")).OrderBy(x => x)); var parser = Project.CreateDataParser(dicData["parser"]); //check if raw data file exists fi = new FileInfo(filePath); if (fi.Exists) { //column separator is always ; var result = ANNDataSet.prepareData(File.ReadAllLines(filePath), parser.ColumnSeparator, parser.FirstRowHeader); ds.Data = result.data; ds.IsPrecentige = Settings.PrecentigeSplit; ds.TestRows = Settings.ValidationSetCount; DataSet = ds; } //load existing mlconfigs var models = Project.GetMLConfigs(dicData["project"]); foreach (var model in models) { var m = new MLConfigController(activeModelChanged); m.Settings = Settings; // m.Name = model; Models.Add(m); } } catch (Exception) { throw; } }
/// <summary> /// Creates a new ANNdotNET Model configuration file. /// </summary> /// <param name="project"></param> /// <param name="mlconfigName"></param> /// <returns></returns> public static bool NewMLConfigFile(Project project, string mlconfigName, ANNDataSet ds) { try { var strFeatures = $"features:"; //first create features and labels based on data descriptor var numDim = project.Descriptor.Columns.Where(x => x.Kind == DataKind.Feature && x.Type == MLDataType.Numeric).Count(); if (numDim > 0) { strFeatures += $"{ProjectSettings.m_NumFeaturesGroupName} 1;{numDim} 0\t"; } //create features for image var imgCol = project.Descriptor.Columns.Where(x => x.Kind == DataKind.Feature && x.Type == MLDataType.Image).FirstOrDefault(); if (imgCol != null) { var shape = imgCol.Shape + ";" + ds.Data.First().Last(); strFeatures += $"|{imgCol.Name} {shape} 0\t"; } //create category features foreach (var c in project.Descriptor.Columns.Where(x => x.Kind == DataKind.Feature && x.Type == MLDataType.Category)) { strFeatures += $"|{c.Name} 1;{c.Classes.Length} 0\t"; } //create label var strLabel = $"labels:"; foreach (var c in project.Descriptor.Columns.Where(x => x.Kind == DataKind.Label && x.Type == MLDataType.Category)) { strLabel += $"|{c.Name} {c.Classes.Length} 0\t"; } // foreach (var c in project.Descriptor.Columns.Where(x => x.Kind == DataKind.Label && x.Type == MLDataType.Numeric)) { strLabel += $"|{c.Name} {1} 0\t"; } //only one Label is supported var countLabel = project.Descriptor.Columns.Where(x => x.Kind == DataKind.Label && x.Type != MLDataType.None).Count(); if (countLabel == 0) { throw new Exception("The mlconfig cannot be created, no label is defined!"); } if (countLabel > 1) { throw new Exception("The mlconfig cannot be created, more than one label is defined!"); } //create rest of the ML config file. //later the user will be able to setup model training and other params var strMlCOnfig = new List <string>(); //first we have to generate unique model identified var modeId = $"configid:{Guid.NewGuid()}"; strMlCOnfig.Add(modeId); //then generate metadat var strMetaData = $"metadata:{project.Descriptor.ToMetadataString(true)}"; strMlCOnfig.Add(strMetaData); //add features and labels previously generated strMlCOnfig.Add(strFeatures); strMlCOnfig.Add(strLabel); //empty network var mbType = project.Settings.ProjectType == ProjectType.ImageClassification? MinibatchType.Image : MinibatchType.Default; strMlCOnfig.Add("network:|Layer:Dense 1 0 0 None 0 0"); strMlCOnfig.Add("learning:|Type:SGDLearner |LRate:0.01 |Momentum:1 |Loss:SquaredError |Eval:SquaredError"); strMlCOnfig.Add($"training:|Type:{mbType} |BatchSize:50 |Epochs:100 |Normalization:0 |RandomizeBatch:0 |SaveWhileTraining:1 |ProgressFrequency:100 |ContinueTraining:0 |TrainedModel: "); var strDicts = GetDefaultMLConfigPaths(project.Settings, mlconfigName); //add paths for ml config foreach (var d in strDicts) { strMlCOnfig.Add($"{d.Key}:{d.Value}"); } // var mlConfigPath = Project.GetMLConfigPath(project.Settings, mlconfigName); File.WriteAllLines(mlConfigPath, strMlCOnfig); } catch (Exception) { throw; } return(true); }
/// <summary> /// Parses the information from the Raw Data and created mlready data sets for default project /// </summary> /// <param name="dataSet"></param> /// <param name="modelName"></param> /// <param name="dataDesc"></param> private void createDefaultDataSets(ANNDataSet dataSet, string modelName, DataDescriptor dataDesc) { var strPathTrain = Project.GetDefaultMLDatasetPath(Settings, modelName, DataSetType.Training); var strPathValid = Project.GetDefaultMLDatasetPath(Settings, modelName, DataSetType.Validation); var strPathTest = Project.GetDefaultMLDatasetPath(Settings, modelName, DataSetType.Testing); //get dataset based on options var ds = dataSet.GetDataSet(dataSet.RandomizeData); //we want whole data set later the data will be split ds.RowsToValidation = 0; ds.RowsToTest = 0; //create experiment based created dataset var exp = new DataFrame(ds); var data = ExportData.PrepareDataSet(exp); //calculate validation and training rows int validCount = dataSet.IsPrecentige ? (int)(dataSet.RowsToValidation * data.Count / 100.0) : dataSet.RowsToValidation; //in case of empty validation data set skip file creation if (validCount == 0) { strPathValid = ""; } //calculate testing rows int testCount = dataSet.IsPrecentige ? (int)(dataSet.RowsToTest * data.Count / 100.0) : dataSet.RowsToValidation; //in case of empty validation data set skip file creation if (testCount == 0) { strPathTest = ""; } //create training ml ready dataset file int trainCount = data.Count - validCount - testCount; //check if the training count number valid if (trainCount <= 0) { throw new Exception("Train dataset is empty. Split data set on correct parts."); } File.WriteAllLines(strPathTrain, data.Take(trainCount).ToList()); //in case of empty validation data set skip file creation if (validCount > 0) { var d = data.Skip(trainCount).Take(validCount).ToList(); File.WriteAllLines(strPathValid, d); } //in case of empty validation data set skip file creation if (testCount > 0) { var d = data.Skip(trainCount + validCount).Take(testCount).ToList(); File.WriteAllLines(strPathTest, d); } //enumerate all column and setup column information needed for mlconfig creation foreach (var c in dataDesc.Columns.Where(x => x.Type == MLDataType.Category && x.Kind != DataKind.None)) { var cc = exp.GetColumns().Where(x => x.Name == c.Name && x.ColumnDataType == ColumnType.Category).FirstOrDefault(); if (cc == null) { throw new Exception("Column not found!"); } c.Classes = cc.Statistics.Categories.ToArray(); } }
/// <summary> /// Parses the information from the Raw Data and created mlready data sets for image classification project /// </summary> /// <param name="dataSet"></param> /// <param name="modelName"></param> /// <param name="descriptor"></param> private void createImgClassificationDataSets(ANNDataSet dataSet, string modelName, DataDescriptor descriptor) { //define daata paths var strPathTrain = Project.GetDefaultMLDatasetPath(Settings, modelName, DataSetType.Training); var strPathValid = Project.GetDefaultMLDatasetPath(Settings, modelName, DataSetType.Validation); var strPathTest = Project.GetDefaultMLDatasetPath(Settings, modelName, DataSetType.Testing); // var data = DataSet.Data; //retrieve image information form files var trainingData = new List <string>(); var validationData = new List <string>(); var testData = new List <string>(); int dataCount = 0; // int cahannel = int.Parse(data[0][3]); int height = int.Parse(data[0][4]); int width = int.Parse(data[0][5]); //first count how many images we are handling for (int i = 0; i < data.Count; i++) { //ith label var label = data[i][0]; var folder = data[i][1]; var query = data[i][2]; string[] images = null; if (string.IsNullOrEmpty(query) || query == " ") { images = Directory.GetFiles(folder); } else { images = Directory.GetFiles(folder, query, SearchOption.TopDirectoryOnly); } // if (images == null || images.Length == 0) { throw new Exception($"No images have been found for '{label}' label. Folder path must not be empty."); } dataCount += images.Length; } //create data sets var classesList = new List <string>(); for (int i = 0; i < data.Count; i++) { //i-th label var label = data[i][0]; //add class into list classesList.Add(label); string lblIndex = i.ToString(); // var folder = data[i][1]; var query = data[i][2]; string[] images = null; if (string.IsNullOrEmpty(query) || query == " ") { images = Directory.GetFiles(folder); } else { images = Directory.GetFiles(folder, query, SearchOption.TopDirectoryOnly); } // if (images == null || images.Length == 0) { throw new Exception($"No images have been found for '{label}' label. Folder path must not be empty."); } //construct the rows of map files var strValue = images.Select(x => $"{x}\t{lblIndex}"); //calculate amount of rows to put into train, valid and test data set var validCount = strValue.Count() * dataSet.RowsToValidation / 100.0f; var testCount = strValue.Count() * dataSet.RowsToTest / 100.0f; var trainCount = strValue.Count() - (int)validCount - (int)testCount; //add to train set trainingData.AddRange(strValue.Take(trainCount)); //add to valid set if (validCount > 0) { validationData.AddRange(strValue.Skip(trainCount).Take((int)validCount)); } //add to test set if (testCount > 0) { testData.AddRange(strValue.Skip((int)trainCount + (int)validCount).Take((int)testCount)); } } //define metadata needed for MLConfiguration creation //Label columm classes definition descriptor.Columns.Last().Classes = classesList.ToArray(); //image shape definition descriptor.Columns.First().Shape = $"{cahannel};{height};{width}"; //check if the training count number valid if (trainingData.Count() <= 0) { throw new Exception("Train dataset is empty. Split data set on correct parts."); } File.WriteAllLines(strPathTrain, trainingData); //in case of empty validation data set skip file creation if (validationData.Count() > 0) { File.WriteAllLines(strPathValid, validationData); } //in case of empty validation data set skip file creation if (testData.Count() > 0) { File.WriteAllLines(strPathTest, testData); } }
private async Task <(List <string> header, List <List <string> > data)> initRawData(string filePath, DataParser parser) { //check if raw data file exists var fi = new FileInfo(filePath); if (fi.Exists) { // var result = await Task.Run <(List <string> header, List <List <string> > data)>(() => ANNDataSet.prepareDataFromFile(filePath, parser.ColumnSeparator, parser.FirstRowHeader, loadProjectProgress)); return(result); } else { return(null, null); } }
//initialize project controllers with project information internal async Task <bool> Initproject(string projectPath, ProjectType pType = ProjectType.Default) { try { var fi = new FileInfo(projectPath); if (!fi.Exists) { throw new Exception("Project File not found!"); } //load project information from file var dicData = Project.LoadProjectData(projectPath); //load project info Settings = Project.CreateProjectSettings(dicData["project"]); Settings.ProjectFolder = fi.Directory.FullName; Settings.ProjectFile = fi.Name; // Name = Project.GetParameterValue(dicData["project"], "Name"); var prData = dicData["data"].Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries); var dataPath = Project.GetParameterValue(prData, "RawData"); var filePath = ""; if (!string.IsNullOrEmpty(dataPath)) { filePath = Path.Combine(Settings.ProjectFolder, Name, dataPath); } else { filePath = Path.Combine(Settings.ProjectFolder, Name, Name + "_rawdata.txt"); } //load rawdataset var parser = Project.CreateDataParser(dicData["parser"]); var result = await initRawData(filePath, parser); //create dataset var ds = new ANNDataSet(); ds.InitMetaColumn(prData.Where(x => x.StartsWith("Column")).OrderBy(x => x)); ds.Data = result.data; ds.IsPrecentige = Settings.PercentigeSplit; ds.RowsToValidation = Settings.ValidationSetCount; ds.RowsToTest = Settings.TestSetCount; DataSet = ds; //load existing mlconfigs var models = Project.GetMLConfigs(dicData["project"]); foreach (var model in models) { var m = new MLConfigController(activeModelChanged); m.Settings = Settings; // m.Name = model; Models.Add(m); } return(true); } catch (Exception) { throw; } }