/// <summary> /// Parses raw dataset and extract the list of Variables. /// </summary> /// <param name="metaDataValues"></param> /// <returns></returns> public static List <VariableDescriptor> ParseRawDataSet(string metaDataValues) { try { //parse data var mdata = metaDataValues.Split(MLFactory.m_cntkSpearator, StringSplitOptions.RemoveEmptyEntries); //define columns var cols = new List <VariableDescriptor>(); //parse meta data foreach (var c in mdata.Where(x => x.StartsWith("Column")).OrderBy(x => x)) { VariableDescriptor col = new VariableDescriptor(); //check if double point appear more than one time. In that case raise exception if (c.Count(x => x == ':') > 1) { throw new Exception("Column data contains double point ':' which is reserved char. PLease remove double point from metadata."); } var strData = c.Substring(c.IndexOf(":") + 1); var colValues = strData.Split(MLFactory.m_ValueSpearator, StringSplitOptions.RemoveEmptyEntries); col.Name = colValues[0]; col.Type = (DataType)Enum.Parse(typeof(DataType), colValues[1], true); col.Kind = (DataKind)Enum.Parse(typeof(DataKind), colValues[2], true); col.MissingValue = (MissingValue)Enum.Parse(typeof(MissingValue), colValues[3], true); // if (col.Type == DataType.Category) { var cl = DataDescriptor.GetColumnClasses(c); if (cl != null) { col.Classes = cl.ToArray(); } } //add parsed column to the collection cols.Add(col); } return(cols); } catch (Exception) { throw; } }
public void Load(string projPath) { try { var fi = new FileInfo(projPath); if (!fi.Exists) { throw new Exception("File not found!"); } // if (Settings == null) { Settings = new ProjectSettings(); } //load project tags var dicValues = MLFactory.LoadMLConfiguration(projPath); //parse project information var projectMetaData = dicValues["project"]; parseProject(projectMetaData); //setup project folder where all files will be places Settings.ProjectFolder = fi.Directory.FullName; Settings.ProjectFile = fi.Name; //reset Descriptor Descriptor = new DataDescriptor(); //parse data var dataValues = dicValues["parser"]; parseParser(dataValues); //parse data var metaDataValues = dicValues["data"]; parseMetaData(metaDataValues); } catch (Exception) { throw; } }
public static EvaluationResult EvaluateModel(string mlconfigPath, DataSetType dsType, EvaluationType evType, ProcessDevice pdevice) { var er = new EvaluationResult(); er.Header = new List <string>(); //device definition DeviceDescriptor device = MLFactory.GetDevice(pdevice); //Load ML model configuration file var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath); //add full path of model folder since model file doesn't contains any absolute path dicMParameters.Add("root", Project.GetMLConfigFolder(mlconfigPath)); // get model data paths var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]); var modelName = Project.GetParameterValue(dicMParameters["training"], "TrainedModel"); var nnModelPath = Path.Combine(dicMParameters["root"], modelName); //check if model exists if (!MLFactory.IsFileExist(nnModelPath)) { return(er); } //check if dataset files exist var dataPath = GetDataPath(dicMParameters, dsType); if (!MLFactory.IsFileExist(dataPath)) { //in case validation dataset is not defiend just export traininign dataset if (dsType == DataSetType.Validation) { dataPath = GetDataPath(dicMParameters, DataSetType.Training); } if (!MLFactory.IsFileExist(dataPath)) { return(er); } } //get output classes in case the ml problem is classification var strCls = dicMParameters.ContainsKey("metadata") ? dicMParameters["metadata"] : ""; er.OutputClasses = DataDescriptor.GetOutputClasses(strCls); //Minibatch type var mbTypestr = Project.GetParameterValue(dicMParameters["training"], "Type"); MinibatchType mbType = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true); var mbSizetr = Project.GetParameterValue(dicMParameters["training"], "BatchSize"); var mf = MLFactory.CreateMLFactory(dicMParameters); //perform evaluation var evParams = new EvaluationParameters() { MinibatchSize = uint.Parse(mbSizetr), MBSource = new MinibatchSourceEx(mbType, mf.StreamConfigurations.ToArray(), dataPath, null, MinibatchSource.FullDataSweep, false), Input = mf.InputVariables, Ouptut = mf.OutputVariables, }; //evaluate model if (evType == EvaluationType.FeaturesOnly) { if (!dicMParameters.ContainsKey("metadata")) { throw new Exception("The result cannot be exported to Excel, since no metadata is stored in mlconfig file."); } var desc = ParseRawDataSet(dicMParameters["metadata"]); er.Header = generateHeader(desc); er.DataSet = FeatureAndLabels(nnModelPath, dataPath, evParams, device); return(er); } else if (evType == EvaluationType.Results) { //define header er.Header.Add(evParams.Ouptut.First().Name + "_actual"); er.Header.Add(evParams.Ouptut.First().Name + "_predicted"); var result = EvaluateFunction(nnModelPath, dataPath, evParams, device); er.Actual = result.actual.ToList(); er.Predicted = result.predicted.ToList(); return(er); } else if (evType == EvaluationType.ResultyExtended) { //define header er.Header.Add(evParams.Ouptut.First().Name + "_actual"); er.Header.Add(evParams.Ouptut.First().Name + "_predicted"); er.Actual = new List <float>(); er.Predicted = new List <float>(); er.ActualEx = new List <List <float> >(); er.PredictedEx = new List <List <float> >(); // var resultEx = EvaluateFunctionEx(nnModelPath, dataPath, evParams, device); for (int i = 0; i < resultEx.actual.Count(); i++) { var res1 = MLValue.GetResult(resultEx.actual[i]); er.Actual.Add(res1); var res2 = MLValue.GetResult(resultEx.predicted[i]); er.Predicted.Add(res2); } er.ActualEx = resultEx.actual; er.PredictedEx = resultEx.predicted; return(er); } else { throw new Exception("Unknown evaluation type!"); } }