Esempio n. 1
0
        public MinibatchSourceEx(MinibatchType type, StreamConfiguration[] streamConfigurations, List <Variable> inputVar, List <Variable> outputVar,
                                 string trainFilePath, string validFilePath, ulong epochSize, bool randomizeBatch, int useImgAugm)
        {
            this.StreamConfigurations = streamConfigurations;
            this.TrainingDataFile     = trainFilePath;
            this.ValidationDataFile   = validFilePath;
            Type = type;

            if (Type == MinibatchType.Default)
            {
                // prepare the training data
                defaultmb = MinibatchSource.TextFormatMinibatchSource(trainFilePath, StreamConfigurations, epochSize, randomizeBatch);
            }
            else if (Type == MinibatchType.Image)
            {
                var featVar = inputVar.First();
                //
                int image_width  = featVar.Shape.Dimensions[0];
                int image_height = featVar.Shape.Dimensions[1];
                int num_channels = featVar.Shape.Dimensions[2];

                //make transformation and scaling
                var transforms          = new List <CNTKDictionary>();
                var randomSideTransform = CNTKLib.ReaderCrop("RandomSide",
                                                             new Tuple <int, int>(0, 0),
                                                             new Tuple <float, float>(0.8f, 1.0f),
                                                             new Tuple <float, float>(0.0f, 0.0f),
                                                             new Tuple <float, float>(1.0f, 1.0f),
                                                             "uniRatio");
                if (useImgAugm == 1)
                {
                    transforms.Add(randomSideTransform);
                }

                //scaling image comes at the end of image transformation
                var scaleTransform = CNTKLib.ReaderScale(image_width, image_height, num_channels);
                transforms.Add(scaleTransform);


                var labelName      = streamConfigurations.Last().m_streamName;
                var labelDimension = streamConfigurations.Last().m_dim;
                var featureName    = streamConfigurations.First().m_streamName;
                var imagemb        = CNTKLib.ImageDeserializer(trainFilePath, labelName, (uint)labelDimension, featureName, transforms);
                var mmsConfig      = new CNTK.MinibatchSourceConfig(new CNTK.DictionaryVector()
                {
                    imagemb
                });

                //
                defaultmb = CNTKLib.CreateCompositeMinibatchSource(mmsConfig);
            }
            else if (Type == MinibatchType.Custom)
            {
                custommb = new StreamReader(trainFilePath);
            }
            else
            {
                throw new Exception("Minibatchsource type is unknown!");
            }
        }
Esempio n. 2
0
        public MinibatchSourceEx(MinibatchType type, StreamConfiguration[] streamConfigurations, string trainFilePath, string validFilePath, ulong epochSize, bool randomizeBatch)
        {
            this.StreamConfigurations = streamConfigurations;
            this.TrainingDataFile     = trainFilePath;
            this.ValidationDataFile   = validFilePath;
            Type = type;

            if (Type == MinibatchType.Default)
            {
                // prepare the training data
                defaultmb = MinibatchSource.TextFormatMinibatchSource(trainFilePath, StreamConfigurations, epochSize, randomizeBatch);
            }
            else if (Type == MinibatchType.Custom)
            {
                custommb = new StreamReader(trainFilePath);
            }
        }
Esempio n. 3
0
        /// <summary>
        /// The method is called during Evaluation of the model for specific data set which is specified as an argument
        /// </summary>
        /// <param name="type"></param>
        /// <param name="strFilePath">dataset file path</param>
        /// <param name="streamConfigurations">stream configuration which provides meta-data information</param>
        /// <param name="device"></param>
        /// <returns></returns>
        public static UnorderedMapStreamInformationMinibatchData GetFullBatch(MinibatchType type, string strFilePath, StreamConfiguration[] streamConfigurations, DeviceDescriptor device)
        {
            if (type == MinibatchType.Default)
            {
                var mbs = MinibatchSource.TextFormatMinibatchSource(strFilePath, streamConfigurations, MinibatchSource.FullDataSweep, false);
                //
                var minibatchData = mbs.GetNextMinibatch(int.MaxValue, device);
                //
                return(minibatchData);
            }
            else if (type == MinibatchType.Custom)
            {
                using (var mbreader = new StreamReader(strFilePath))
                {
                    var retVal = nextBatch(mbreader, streamConfigurations, -1, 1, device);
                    var mb     = new UnorderedMapStreamInformationMinibatchData();

                    for (int i = 0; i < retVal.Count; i++)
                    {
                        var k = retVal.ElementAt(i);

                        var key = k.Key;
                        var si  = new StreamInformation();
                        si.m_definesMbSize = streamConfigurations[i].m_definesMbSize;
                        si.m_storageFormat = k.Value.data.StorageFormat;
                        si.m_name          = streamConfigurations[i].m_streamName;

                        var stream = streamConfigurations[i];
                        mb.Add(si, k.Value);
                    }

                    return(mb);
                }
            }
            else
            {
                throw new Exception("Minibatch is not supported.");
            }
        }
Esempio n. 4
0
        /// <summary>
        /// Convert minibatch data
        /// </summary>
        /// <param name="args"> data being converted</param>
        /// <returns></returns>
        public static UnorderedMapVariableMinibatchData ToMinibatchData(UnorderedMapStreamInformationMinibatchData args, List <Variable> vars, MinibatchType type)
        {
            var arguments = new UnorderedMapVariableMinibatchData();

            foreach (var mbd in args)
            {
                var v = vars.Where(x => x.Name == mbd.Key.m_name).FirstOrDefault();
                if (v == null)
                {
                    throw new Exception("Stream is invalid!");
                }
                if (type == MinibatchType.Custom)
                {
                    var mbd1 = new MinibatchData(mbd.Value.data.DeepClone(), mbd.Value.numberOfSamples, mbd.Value.sweepEnd);
                    arguments.Add(v, mbd1);
                }

                else
                {
                    arguments.Add(v, mbd.Value);
                }
            }
            return(arguments);
        }
Esempio n. 5
0
        /// <summary>
        /// Evaluate model defined in the mlconfig file
        /// </summary>
        /// <param name="mlconfigPath"></param>
        /// <param name="device"></param>
        /// <returns></returns>
        public static async Task <EvaluationResult> EvaluateMLConfig(string mlconfigPath, DeviceDescriptor device, DataSetType dsType, EvaluationType evType)
        {
            try
            {
                //define eval result
                var er = new EvaluationResult();
                er.OutputClasses = new List <string>()
                {
                    ""
                };
                er.Actual    = new List <float>();
                er.Predicted = new List <float>();
                er.Header    = new List <string>();

                //Load ML configuration file
                var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath);
                //add full path of model folder since model file doesn't contains any absolute path
                dicMParameters.Add("root", MLFactory.GetMLConfigFolder(mlconfigPath));

                // get model data paths
                var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]);
                //parse feature variables
                var projectValues = dicMParameters["training"].Split(MLFactory.m_cntkSpearator, StringSplitOptions.RemoveEmptyEntries);
                var modelName     = MLFactory.GetParameterValue(projectValues, "TrainedModel");
                var nnModelPath   = Path.Combine(dicMParameters["root"], modelName);
                //check if model exists
                if (!MLFactory.IsFileExist(nnModelPath))
                {
                    return(er);
                }
                //
                var dataset = MLFactory.GetDataPath(dicMParameters, dsType);
                if (string.IsNullOrEmpty(dataset) || string.IsNullOrEmpty(dataset) || dataset == " ")
                {
                    if (dsType == DataSetType.Testing)
                    {
                        dataset = MLFactory.GetDataPath(dicMParameters, DataSetType.Validation);
                    }
                    if (string.IsNullOrEmpty(dataset) || string.IsNullOrEmpty(dataset) || dataset == " ")
                    {
                        return(er);
                    }
                }


                //get output classes in case the ml problem is classification
                var strCls = dicMParameters.ContainsKey("metadata") ? dicMParameters["metadata"] : "";
                var oc     = MLFactory.GetOutputClasses(strCls);
                if (oc != null)
                {
                    er.OutputClasses = oc;
                }

                //MInibatch
                var           mbTypestr = MLFactory.GetParameterValue(projectValues, "Type");
                MinibatchType mbType    = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true);
                var           mbSizetr  = MLFactory.GetParameterValue(projectValues, "BatchSize");

                var mf = MLFactory.CreateMLFactory(dicMParameters);


                //perform evaluation
                var evParams = new EvaluationParameters()
                {
                    MinibatchSize = uint.Parse(mbSizetr),
                    MBSource      = new MinibatchSourceEx(mbType, mf.StreamConfigurations.ToArray(), mf.InputVariables, mf.OutputVariables, dataset, null, MinibatchSource.FullDataSweep, false, 0),
                    Input         = mf.InputVariables,
                    Ouptut        = mf.OutputVariables,
                };

                //evaluate model
                if (evType == EvaluationType.FeaturesOnly)
                {
                    if (!dicMParameters.ContainsKey("metadata"))
                    {
                        throw new Exception("The result cannot be exported to Excel, since no metadata is stored in mlconfig file.");
                    }

                    var desc = MLFactory.ParseRawDataSet(dicMParameters["metadata"]);
                    er.Header = MLFactory.GenerateHeader(desc);
                    var fun = Function.Load(nnModelPath, device);
                    //
                    er.DataSet = await Task.Run(() => MLEvaluator.FeaturesAndLabels(fun, evParams, device));

                    return(er);
                }
                else if (evType == EvaluationType.Results)
                {
                    //define header
                    er.Header.Add(evParams.Ouptut.First().Name + "_actual");
                    er.Header.Add(evParams.Ouptut.First().Name + "_predicted");

                    var fun = Function.Load(nnModelPath, device);
                    //
                    var result = await Task.Run(() => MLEvaluator.EvaluateFunction(fun, evParams, device));

                    er.Actual    = result.actual.ToList();
                    er.Predicted = result.predicted.ToList();

                    if (er.OutputClasses.Count < 2 && evParams.Ouptut.First().Shape.Dimensions.Last() > 1)
                    {
                        var result1 = await Task.Run(() => MLEvaluator.EvaluateFunctionEx(fun, evParams, device));

                        er.ActualEx    = result1.actual;
                        er.PredictedEx = result1.predicted;
                    }
                    return(er);
                }
                else if (evType == EvaluationType.ResultExtended)
                {
                    //define header
                    er.Header.Add(evParams.Ouptut.First().Name + "_actual");
                    er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
                    er.Actual      = new List <float>();
                    er.Predicted   = new List <float>();
                    er.ActualEx    = new List <List <float> >();
                    er.PredictedEx = new List <List <float> >();

                    //
                    var fun      = Function.Load(nnModelPath, device);
                    var resultEx = await Task.Run(() => MLEvaluator.EvaluateFunctionEx(fun, evParams, device));

                    //var resultEx = EvaluateFunctionEx(nnModelPath, dataPath, evParams, device);
                    for (int i = 0; i < resultEx.actual.Count(); i++)
                    {
                        var res1 = MLValue.GetResult(resultEx.actual[i]);
                        er.Actual.Add(res1);
                        var res2 = MLValue.GetResult(resultEx.predicted[i]);
                        er.Predicted.Add(res2);
                    }
                    er.ActualEx    = resultEx.actual;
                    er.PredictedEx = resultEx.predicted;

                    return(er);
                }
                else
                {
                    throw new Exception("Unknown evaluation type!");
                }
            }
            catch (Exception)
            {
                throw;
            }
        }
Esempio n. 6
0
        /// <summary>
        /// Evaluate the model against dataset sored in the dataset file, and exports the result in csv format for further analysis
        /// </summary>
        /// <param name="mlF"> ml factory object contains members needed to evaluation process</param>
        /// <param name="mbs"> Minibatch source which provides helpers members needed for for evaluation</param>
        /// <param name="strDataSetPath"> file of dataset</param>
        /// <param name="modelPath"> models which will be evaluate</param>
        /// <param name="resultExportPath"> result file in which the result will be exported</param>
        /// <param name="device"> device for computation</param>
        public static void EvaluateModel(string mlconfigPath, string bestTrainedModelPath, DeviceDescriptor device)
        {
            //Load ML model configuration file
            var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath);

            //add full path of model folder since model file doesn't contains any absolute path
            dicMParameters.Add("root", MLFactory.GetMLConfigFolder(mlconfigPath));

            //get model daa paths
            var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]);

            //parse feature variables
            var projectValues            = dicMParameters["training"].Split(MLFactory.m_cntkSpearator, StringSplitOptions.RemoveEmptyEntries);
            var trainedModelRelativePath = MLFactory.GetParameterValue(projectValues, "TrainedModel");


            //Minibatch type
            var           mbTypestr = MLFactory.GetParameterValue(projectValues, "Type");
            MinibatchType mbType    = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true);
            //prepare MLFactory
            var f = MLFactory.CreateMLFactory(dicMParameters);

            //prepare data paths for mini-batch source
            var strTrainPath = $"{dicMParameters["root"]}\\{dicPath["Training"]}";
            var strValidPath = $"{dicMParameters["root"]}\\{dicPath["Validation"]}";
            var strResult    = $"{dicMParameters["root"]}\\{dicPath["Result"]}";

            var bestModelFullPath = $"{dicMParameters["root"]}\\{bestTrainedModelPath}";
            //decide what data to evaluate
            var dataPath = strValidPath;

            //load model
            var model = Function.Load(bestModelFullPath, device);

            //get data for evaluation by calling GetFullBatch
            var minibatchData = MinibatchSourceEx.GetFullBatch(mbType, dataPath, f.StreamConfigurations.ToArray(), device);
            //input map creation for model evaluation
            var inputMap = new Dictionary <Variable, Value>();

            foreach (var v in minibatchData)
            {
                var vv         = model.Arguments.Where(x => x.Name == v.Key.m_name).FirstOrDefault();
                var streamInfo = v.Key;
                if (vv != null)
                {
                    inputMap.Add(vv, minibatchData[streamInfo].data);
                }
            }

            //output map
            var predictedDataMap = new Dictionary <Variable, Value>();

            foreach (var outp in model.Outputs)
            {
                predictedDataMap.Add(outp, null);
            }

            //model evaluation
            model.Evaluate(inputMap, predictedDataMap, device);

            //retrieve actual and predicted values from model
            List <List <float> > actual  = new List <List <float> >();
            List <List <float> > predict = new List <List <float> >();

            foreach (var output in model.Outputs)
            {
                //label stream
                var labelStream = minibatchData.Keys.Where(x => x.m_name == output.Name).First();

                //actual values
                List <List <float> > av = MLValue.GetValues(output, minibatchData[labelStream].data);
                //predicted values
                List <List <float> > pv = MLValue.GetValues(output, predictedDataMap[output]);

                for (int i = 0; i < av.Count; i++)
                {
                    //actual
                    var act = av[i];
                    if (actual.Count <= i)
                    {
                        actual.Add(new List <float>());
                    }
                    actual[i].AddRange(act);
                    //prediction
                    var prd = pv[i];
                    if (predict.Count <= i)
                    {
                        predict.Add(new List <float>());
                    }
                    predict[i].AddRange(prd);
                }
            }


            //export result
            MLValue.ValueToFile(actual, predict, strResult);

            //
            Console.WriteLine(Environment.NewLine);
            Console.WriteLine($"*******************Model Evaluation**************");
            Console.WriteLine(Environment.NewLine);
            Console.WriteLine($"Model Evaluation successfully exported result into file {strResult}!");
            Console.WriteLine(Environment.NewLine);
        }
Esempio n. 7
0
        public static EvaluationResult EvaluateModel(string mlconfigPath, DataSetType dsType, EvaluationType evType, ProcessDevice pdevice)
        {
            var er = new EvaluationResult();

            er.Header = new List <string>();
            //device definition
            DeviceDescriptor device = MLFactory.GetDevice(pdevice);
            //Load ML model configuration file
            var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath);

            //add full path of model folder since model file doesn't contains any absolute path
            dicMParameters.Add("root", Project.GetMLConfigFolder(mlconfigPath));

            // get model data paths
            var dicPath     = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]);
            var modelName   = Project.GetParameterValue(dicMParameters["training"], "TrainedModel");
            var nnModelPath = Path.Combine(dicMParameters["root"], modelName);

            //check if model exists
            if (!MLFactory.IsFileExist(nnModelPath))
            {
                return(er);
            }


            //check if dataset files exist
            var dataPath = GetDataPath(dicMParameters, dsType);

            if (!MLFactory.IsFileExist(dataPath))
            {
                //in case validation dataset is not defiend just export traininign dataset
                if (dsType == DataSetType.Validation)
                {
                    dataPath = GetDataPath(dicMParameters, DataSetType.Training);
                }
                if (!MLFactory.IsFileExist(dataPath))
                {
                    return(er);
                }
            }

            //get output classes in case the ml problem is classification
            var strCls = dicMParameters.ContainsKey("metadata") ? dicMParameters["metadata"] : "";

            er.OutputClasses = DataDescriptor.GetOutputClasses(strCls);

            //Minibatch type
            var           mbTypestr = Project.GetParameterValue(dicMParameters["training"], "Type");
            MinibatchType mbType    = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true);
            var           mbSizetr  = Project.GetParameterValue(dicMParameters["training"], "BatchSize");

            var mf = MLFactory.CreateMLFactory(dicMParameters);
            //perform evaluation
            var evParams = new EvaluationParameters()
            {
                MinibatchSize = uint.Parse(mbSizetr),
                MBSource      = new MinibatchSourceEx(mbType, mf.StreamConfigurations.ToArray(), dataPath, null, MinibatchSource.FullDataSweep, false),
                Input         = mf.InputVariables,
                Ouptut        = mf.OutputVariables,
            };

            //evaluate model
            if (evType == EvaluationType.FeaturesOnly)
            {
                if (!dicMParameters.ContainsKey("metadata"))
                {
                    throw new Exception("The result cannot be exported to Excel, since no metadata is stored in mlconfig file.");
                }
                var desc = ParseRawDataSet(dicMParameters["metadata"]);
                er.Header  = generateHeader(desc);
                er.DataSet = FeatureAndLabels(nnModelPath, dataPath, evParams, device);

                return(er);
            }
            else if (evType == EvaluationType.Results)
            {
                //define header
                er.Header.Add(evParams.Ouptut.First().Name + "_actual");
                er.Header.Add(evParams.Ouptut.First().Name + "_predicted");

                var result = EvaluateFunction(nnModelPath, dataPath, evParams, device);
                er.Actual    = result.actual.ToList();
                er.Predicted = result.predicted.ToList();
                return(er);
            }
            else if (evType == EvaluationType.ResultyExtended)
            {
                //define header
                er.Header.Add(evParams.Ouptut.First().Name + "_actual");
                er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
                er.Actual      = new List <float>();
                er.Predicted   = new List <float>();
                er.ActualEx    = new List <List <float> >();
                er.PredictedEx = new List <List <float> >();
                //
                var resultEx = EvaluateFunctionEx(nnModelPath, dataPath, evParams, device);
                for (int i = 0; i < resultEx.actual.Count(); i++)
                {
                    var res1 = MLValue.GetResult(resultEx.actual[i]);
                    er.Actual.Add(res1);
                    var res2 = MLValue.GetResult(resultEx.predicted[i]);
                    er.Predicted.Add(res2);
                }
                er.ActualEx    = resultEx.actual;
                er.PredictedEx = resultEx.predicted;

                return(er);
            }
            else
            {
                throw new Exception("Unknown evaluation type!");
            }
        }