Ejemplo n.º 1
0
        /// <summary>
        /// Evaluates the trained model for quality assurance against a
        /// second independent test data set.
        ///
        /// Loads the test dataset.
        /// Creates the binary evaluator.
        /// Evaluates the model and create metrics.
        ///
        /// Displays the metrics.
        /// </summary>
        /// <param name="model"></param>
        internal static void Evaluate(
            PredictionModel <ClassificationData, ClassPrediction> model,
            string testDatafile)
        {
            // loads the new test dataset with the same schema.
            // You can evaluate the model using this dataset as a quality check.
            var testData = new TextLoader(testDatafile).CreateFrom <ClassificationData>();

            // Computes the quality metrics for the PredictionModel using the specified dataset.
            var evaluator = new ClassificationEvaluator();

            // The BinaryClassificationMetrics contains the overall metrics computed by binary
            // classification evaluators. To display these to determine the quality of the model,
            // you need to get the metrics first.
            ClassificationMetrics metrics = evaluator.Evaluate(model, testData);

            // Displaying the metrics for model validation
            Console.WriteLine();
            Console.WriteLine("PredictionModel quality metrics evaluation");
            Console.WriteLine("------------------------------------------");
            Console.WriteLine("  Accuracy Macro: {0:P2}", metrics.AccuracyMacro);
            Console.WriteLine("  Accuracy Micro: {0:P2}", metrics.AccuracyMicro);
            Console.WriteLine("   Top KAccuracy: {0:P2}", metrics.TopKAccuracy);

            Console.WriteLine("         LogLoss: {0:P2}", metrics.LogLoss);

            Console.WriteLine("");
            Console.WriteLine(" PerClassLogLoss:");
            for (int i = 0; i < metrics.PerClassLogLoss.Length; i++)
            {
                Console.WriteLine("       Class: {0} - {1:P2}", i, metrics.PerClassLogLoss[i]);
            }
        }
Ejemplo n.º 2
0
        /// <summary>
        /// Ussing passed testing data and model, it calculates model's accuracy.
        /// </summary>
        /// <returns>Accuracy of the model.</returns>
        public double Evaluate(PredictionModel <IrisFlower, IrisPredict> model, string testDataLocation)
        {
            var testData = new TextLoader(testDataLocation).CreateFrom <IrisFlower>(useHeader: true, separator: ',');
            var metrics  = new ClassificationEvaluator().Evaluate(model, testData);

            return(metrics.AccuracyMacro);
        }
Ejemplo n.º 3
0
        /// Evaluates the trained model for quality assurance against a second independent test data set.
        internal static void Evaluate(
            PredictionModel <ClassificationData, ClassPrediction> model,
            string testDatafile)
        {
            //Loading test data
            var testData = new TextLoader(testDatafile).CreateFrom <ClassificationData>();

            //Deciding which evaluator will be used
            var evaluator = new ClassificationEvaluator();

            //Start evaluation
            ClassificationMetrics metrics = evaluator.Evaluate(model, testData);

            Console.WriteLine();
            Console.WriteLine("------------------------------------------");
            Console.WriteLine("  Accuracy Macro: {0:P2}", metrics.AccuracyMacro);
            Console.WriteLine("  Accuracy Micro: {0:P2}", metrics.AccuracyMicro);
            Console.WriteLine("         LogLoss: {0:P2}", metrics.LogLoss);
            Console.WriteLine();
            Console.WriteLine(" PerClassLogLoss:");
            for (int i = 0; i < metrics.PerClassLogLoss.Length; i++)
            {
                Console.WriteLine("       Class: {0} - {1:P2}", i, metrics.PerClassLogLoss[i]);
            }
        }
        public void TrainOneVersusAll()
        {
            string dataPath = GetDataPath("iris.txt");

            var pipeline = new Legacy.LearningPipeline(seed: 1, conc: 1);

            pipeline.Add(new TextLoader(dataPath).CreateFrom <IrisData>(useHeader: false));
            pipeline.Add(new ColumnConcatenator(outputColumn: "Features",
                                                "SepalLength", "SepalWidth", "PetalLength", "PetalWidth"));

            pipeline.Add(OneVersusAll.With(new StochasticDualCoordinateAscentBinaryClassifier()));

            var model = pipeline.Train <IrisData, IrisPrediction>();

            var testData  = new TextLoader(dataPath).CreateFrom <IrisData>(useHeader: false);
            var evaluator = new ClassificationEvaluator();
            ClassificationMetrics metrics = evaluator.Evaluate(model, testData);

            CheckMetrics(metrics);

            var trainTest = new TrainTestEvaluator()
            {
                Kind = MacroUtilsTrainerKinds.SignatureMultiClassClassifierTrainer
            }.TrainTestEvaluate <IrisData, IrisPrediction>(pipeline, testData);

            CheckMetrics(trainTest.ClassificationMetrics);
        }
 protected void ShowMetrics(string testLocation, PredictionModel<ImageNetData, ImageNetPrediction> model)
 {
     var evaluator = new ClassificationEvaluator();
     var testDataSource = new TextLoader(testLocation).CreateFrom<ImageData.ImageNetData>();
     ClassificationMetrics metrics = evaluator.Evaluate(model, testDataSource);
     PrintMetrics(metrics);
 }
Ejemplo n.º 6
0
        public async Task EvaluateAsync(string modelPath, string csvPath)
        {
            var model = await PredictionModel.ReadAsync <Product, ProductCategoryPrediction>(modelPath).ConfigureAwait(false);

            // To evaluate how good the model predicts values, the model is ran against new set
            // of data (test data) that was not involved in training.
            var testData = new TextLoader(csvPath).CreateFrom <Product>(useHeader: false, allowQuotedStrings: false, supportSparse: false);

            // ClassificationEvaluator performs evaluation for Multiclass Classification type of ML problems.
            var evaluator = new ClassificationEvaluator {
                OutputTopKAcc = 3
            };

            Console.WriteLine("=============== Evaluating model ===============");

            var metrics = evaluator.Evaluate(model, testData);

            Console.WriteLine("Metrics:");
            Console.WriteLine($"    AccuracyMacro = {metrics.AccuracyMacro:0.####}, a value between 0 and 1, the closer to 1, the better");
            Console.WriteLine($"    AccuracyMicro = {metrics.AccuracyMicro:0.####}, a value between 0 and 1, the closer to 1, the better");
            Console.WriteLine($"    LogLoss = {metrics.LogLoss:0.####}, the closer to 0, the better");
            Console.WriteLine($"    LogLoss for class 1 = {metrics.PerClassLogLoss[0]:0.####}, the closer to 0, the better");
            Console.WriteLine($"    LogLoss for class 2 = {metrics.PerClassLogLoss[1]:0.####}, the closer to 0, the better");
            Console.WriteLine($"    LogLoss for class 3 = {metrics.PerClassLogLoss[2]:0.####}, the closer to 0, the better");

            Console.WriteLine("=============== End evaluating ===============");
            Console.WriteLine();
        }
Ejemplo n.º 7
0
        /// <summary>
        /// Evaluates the trained model for quality assurance against a second data set.
        ///
        /// Loads the test dataset.
        /// Creates the binary evaluator.
        /// Evaluates the model and create metrics.
        ///
        /// Displays the metrics.
        /// </summary>
        /// <param name="model"></param>
        internal static void Evaluate(
            PredictionModel <ClassificationData, SentimentPrediction> model,
            InputData input)
        {
            // loads the new test dataset with the same schema.
            // You can evaluate the model using this dataset as a quality check.

            //var testData = new TextLoader(_testDataPath).CreateFrom<SentimentData>();
            var testData = new TextLoader(input.TestData).CreateFrom <ClassificationData>();

            // Computes the quality metrics for the PredictionModel using the specified dataset.
            var evaluator = new ClassificationEvaluator();

            // The BinaryClassificationMetrics contains the overall metrics computed by binary
            // classification evaluators. To display these to determine the quality of the model,
            // you need to get the metrics first.
            ClassificationMetrics metrics = evaluator.Evaluate(model, testData);

            // Displaying the metrics for model validation
            Console.WriteLine();
            Console.WriteLine("PredictionModel quality metrics evaluation");
            Console.WriteLine("------------------------------------------");
            Console.WriteLine($"Accuracy Macro: {metrics.AccuracyMacro:P2}");
            Console.WriteLine($"Accuracy Micro: {metrics.AccuracyMicro:P2}");
            Console.WriteLine($" Top KAccuracy: {metrics.TopKAccuracy:P2}");
        }
        public ClassificationMetrics Evaluate(string testDataPath)
        {
            var testData = new TextLoader(testDataPath).CreateFrom <MulticlassClassificationData>();

            var evaluator = new ClassificationEvaluator();

            return(evaluator.Evaluate(Model, testData));
        }
Ejemplo n.º 9
0
        private void Evaluate()
        {
            var loader     = new TextLoader("./mnist_test.csv").CreateFrom <MinstDigit>(separator: ',', useHeader: true);
            var evaluator  = new ClassificationEvaluator();
            var evaluation = evaluator.Evaluate(_model, loader);

            Console.WriteLine(JsonConvert.SerializeObject(evaluation, Formatting.Indented));
        }
Ejemplo n.º 10
0
        public static double EvaluateModel <T, G>(PredictionModel <T, G> prediction, string path)
            where T : class
            where G : class, new()
        {
            var dataPrediction = new TextLoader(path).CreateFrom <IrisData>(separator: ',');
            var metrics        = new ClassificationEvaluator().Evaluate(prediction, dataPrediction);

            return(metrics.AccuracyMacro);
        }
Ejemplo n.º 11
0
        static void Main(string[] args)
        {
            var trainData = GeneratePData(2000);

            Print(trainData.Take(20));
            var testData = GeneratePData(50, test: true);

            // ML görevi için obje oluşturur
            var learningPipe    = new LearningPipeline();
            var trainCollection = CollectionDataSource.Create(trainData);

            learningPipe.Add(trainCollection);
            // Verilerin kolon isimleri olan labelları numeric indexe çevirir.
            learningPipe.Add(new Dictionarizer("Label"));

            learningPipe.Add(
                new ColumnConcatenator("Features", "UnitA", "UnitS", "Volume"));

            // Algoritma sınıflandırması
            learningPipe.Add(new StochasticDualCoordinateAscentClassifier());

            // Tahmin edilen kolon değerini çevir
            learningPipe.Add(new PredictedLabelColumnOriginalValueConverter()
            {
                PredictedLabelColumn = "PredictedLabel"
            });

            // Modeli eğitme
            var model = learningPipe.Train <ProcessData, ProcessPrediction>();

            // Model değerlendirilmesi ve kesin kontrol
            var evaluator = new ClassificationEvaluator();
            var metrics   = evaluator.Evaluate(model, trainCollection);

            Console.WriteLine("AccuracyMicro: " + metrics.AccuracyMicro);
            Console.WriteLine("LogLoss: " + metrics.LogLoss);

            // Test datayı tahmin et
            var predicted = model.Predict(testData);

            // Testdata ve tahmin edilen labelı string içinde topla
            var results = testData.Zip(predicted, (t, p) => new ProcessData
            {
                UnitA  = t.UnitA,
                UnitS  = t.UnitS,
                Volume = t.Volume,
                Label  = p.PredictedLabels
            }).ToList();

            // Sonucu yazdır
            Print(results);

            Console.ReadLine();
        }
Ejemplo n.º 12
0
        protected void EvaluateModel(string testLocation, PredictionModel <ImageNetData, ImageNetPrediction> model)
        {
            ConsoleWriteHeader("Metrics for Image Classification");
            var evaluator                 = new ClassificationEvaluator();
            var testDataSource            = new TextLoader(testLocation).CreateFrom <ImageData.ImageNetData>();
            ClassificationMetrics metrics = evaluator.Evaluate(model, testDataSource);

            // Log loss is nearly zero. The lower the better, so in this case can't be better
            // This is an "ideal" situation, usually you get higher values
            Console.WriteLine($"Log Loss: {metrics.LogLoss}");
        }
Ejemplo n.º 13
0
        public static void Evaluate(PredictionModel <IntentData, IntentPrediction> model)
        {
            var testData  = new TextLoader <IntentData>(testPath, useHeader: false, separator: "tab");
            var evaluator = new ClassificationEvaluator();
            var metrics   = evaluator.Evaluate(model, testData);

            Console.WriteLine();
            Console.WriteLine("PredictionModel quality metrics evaluation");
            Console.WriteLine("------------------------------------------");
            Console.WriteLine($"Accuracy Micro: {metrics.AccuracyMicro * 100}%");
            Console.WriteLine($"Accuracy Macro: {metrics.AccuracyMacro * 100}%");
        }
        public Task <ClassificationMetrics> Evaluate(string testDataPath)
        {
            return(Task.Run(() =>
            {
                var testData = new TextLoader(testDataPath).CreateFrom <MulticlassClassificationData>();

                // Computes the quality metrics for the PredictionModel using the specified dataset.
                var evaluator = new ClassificationEvaluator();
                var metrics = evaluator.Evaluate(Model, testData);
                return metrics;
            }));
        }
Ejemplo n.º 15
0
        public static void Evaluate(PredictionModel <PokerHandData, PokerHandPrediction> model, IEnumerable <PokerHandData> data)
        {
            var evaluator  = new ClassificationEvaluator();
            var collection = CollectionDataSource.Create(data);
            var metrics    = evaluator.Evaluate(model, collection);

            Console.WriteLine();
            Console.WriteLine("PredictionModel quality metrics evaluation");
            Console.WriteLine("------------------------------------------");
            Console.WriteLine($"LogLossReduction: {metrics.LogLossReduction }");
            Console.WriteLine($"LogLoss: {metrics.LogLoss }");
        }
Ejemplo n.º 16
0
        public static void Evaluate(PredictionModel <NewsData, NewsPrediction> model)
        {
            var testData  = new TextLoader <NewsData>(_trainingSet, useHeader: false, separator: "tab");
            var evaluator = new ClassificationEvaluator();
            var metrics   = evaluator.Evaluate(model, testData);

            Console.WriteLine();
            Console.WriteLine("PredictionModel quality metrics evaluation");
            Console.WriteLine("------------------------------------------");
            Console.WriteLine($"AccuracyMacro: {metrics.AccuracyMacro:P2}");
            Console.WriteLine($"AccuracyMicro: {metrics.AccuracyMicro:P2}");
            Console.WriteLine($"LogLoss: {metrics.LogLoss:P2}");
        }
Ejemplo n.º 17
0
        public async Task <ClassificationMetrics> EvaluateAsync(string testPath)
        {
            var testData  = new TextLoader(testPath).CreateFrom <BankStatementLineItem>(separator: ',', useHeader: true);
            var evaluator = new ClassificationEvaluator();

            if (_model == null)
            {
                _model = await PredictionModel.ReadAsync <BankStatementLineItem, PredictedLabel>(PredictionModelWrapper
                                                                                                 .Model1Path);
            }

            var metrics = evaluator.Evaluate(_model, testData);

            return(metrics);
        }
Ejemplo n.º 18
0
        static void EvaluateNews(PredictionModel <NewsData, NewsPrediction> model)
        {
            const string trainingSet = @"news-train.txt";

            var testData  = new TextLoader(trainingSet).CreateFrom <NewsData>();
            var evaluator = new ClassificationEvaluator();
            var metrics   = evaluator.Evaluate(model, testData);

            Console.WriteLine();
            Console.WriteLine("PredictionModel quality metrics evaluation");
            Console.WriteLine("------------------------------------------");
            Console.WriteLine($"AccuracyMacro: {metrics.AccuracyMacro:P2}");
            Console.WriteLine($"AccuracyMicro: {metrics.AccuracyMicro:P2}");
            Console.WriteLine($"LogLoss: {metrics.LogLoss:P2}");
        }
Ejemplo n.º 19
0
        private static void Evaluate(PredictionModel <CarEval, CarEvalPredictor> model)
        {
            // To evaluate how good the model predicts values, it is run against new set
            // of data (test data) that was not involved in training.
            var testData = new TextLoader(TestDataPath).CreateFrom <CarEval>(separator: ',');

            // RegressionEvaluator calculates the differences (in various metrics) between predicted and actual
            // values in the test dataset.
            var evaluator = new ClassificationEvaluator();

            Console.WriteLine("=============== Evaluating model ===============");

            var metrics = evaluator.Evaluate(model, testData);

            Console.WriteLine($"Micro-Accuray is {metrics.AccuracyMicro}");
        }
Ejemplo n.º 20
0
        private static void Evaluate(PredictionModel <Master, MasterPrediction> model)
        {
            var testData  = new TextLoader(TestDataPath).CreateFrom <Master>();
            var evaluator = new ClassificationEvaluator {
                OutputTopKAcc = 3
            };

            var metrics = evaluator.Evaluate(model, testData);

            for (var i = 0; i < metrics.ConfusionMatrix.Order; i++)
            {
                for (var j = 0; j < metrics.ConfusionMatrix.ClassNames.Count; j++)
                {
                    Console.Write("\t" + metrics.ConfusionMatrix[i, j]);
                }
            }
        }
Ejemplo n.º 21
0
        /// <summary>
        /// Evaluates trained machine learning model for iris flower species determination.
        /// </summary>
        /// <param name="trainedModel">Trained machine learning model for iris flower species determination.</param>
        /// <returns>Overall metrics of trained machine learning model for iris flower species determination.</returns>
        public ClassificationMetrics Evaluate(PredictionModel <DataModel, PredictionModel> trainedModel)
        {
            // 1) Load test data.
            var testModels = new TextLoader(Constants.TestDataFileLocation)
                             .CreateFrom <DataModel>(useHeader: true, separator: ',');

            // 2) Evaluate trained model.
            var modelMetrics = new ClassificationEvaluator().Evaluate(trainedModel, testModels);

            Console.WriteLine($"*************************************************");
            Console.WriteLine("Prediction model quality metrics after evaluation");
            Console.WriteLine("------------------------------------------");
            Console.WriteLine($"*      Accuracy of prediction model: {modelMetrics.AccuracyMacro * 100}%       *");
            Console.WriteLine($"*************************************************");

            return(modelMetrics);
        }
Ejemplo n.º 22
0
        private static async Task TrainAsync()
        {
            Console.WriteLine("============ 準備訓練資料 ==============");

            var pipeline = new LearningPipeline();

            pipeline.Add(new TextLoader(DataPath).CreateFrom <GitHubIssue>(useHeader: true));

            // 把兩個用來訓練的欄位變成數字的vector
            pipeline.Add(new TextFeaturizer("Title", "Title"));
            pipeline.Add(new TextFeaturizer("Description", "Description"));

            // 把Area轉換成為Dictionary數字
            pipeline.Add(new Dictionarizer(("Area", "Label")));

            // Title 和 Description合并變成訓練的欄位
            pipeline.Add(new ColumnConcatenator("Features", "Title", "Description"));

            // 使用StochasticDualCoordinateAscent演算法
            pipeline.Add(new StochasticDualCoordinateAscentClassifier());

            // 把判斷出來的數字轉回文字版本
            pipeline.Add(new PredictedLabelColumnOriginalValueConverter()
            {
                PredictedLabelColumn = "PredictedLabel"
            });


            Console.WriteLine("=============== 訓練模型 ===============");

            var model = pipeline.Train <GitHubIssue, GithubIssueLabelPrediction>();

            var testData = new TextLoader(TestPath).CreateFrom <GitHubIssue>(useHeader: true);

            var evaluator = new ClassificationEvaluator();

            ClassificationMetrics metrics = evaluator.Evaluate(model, testData);

            Console.WriteLine();
            Console.WriteLine("Micro-Accuracy: {0}", metrics.AccuracyMicro);

            await model.WriteAsync(ModelPath);

            Console.WriteLine("=============== 訓練完成 ===============");
            Console.WriteLine("Model路徑: {0}", ModelPath);
        }
Ejemplo n.º 23
0
        public static void Ex()
        {
            LearningPipeline pipeline = new LearningPipeline();

            pipeline.Add(new TextLoader(@"C:\Users\Дарья\Desktop\AdverbNoun.csv").CreateFrom <Model>(useHeader: true, separator: ','));
            pipeline.Add(new Dictionarizer(("Category", "Label")));
            pipeline.Add(new TextFeaturizer("Features", "Word"));
            pipeline.Add(new StochasticDualCoordinateAscentClassifier());
            //Vec < R4, 472 >
            pipeline.Add(new PredictedLabelColumnOriginalValueConverter()
            {
                PredictedLabelColumn = "PredictedLabel"
            });
            var model = pipeline.Train <Model, CategoryPrediction>();

            model.WriteAsync(@"C:\Users\Дарья\Desktop\NewModel.zip");

            model = PredictionModel.ReadAsync <Model, CategoryPrediction>(@"C:\Users\Дарья\Desktop\NewModel.zip").Result;


            var testData  = new TextLoader(@"C:\Users\Дарья\Desktop\TestAdverbNoun.csv").CreateFrom <Model>(useHeader: true, separator: ',');
            var evaluator = new ClassificationEvaluator();
            var metrics   = evaluator.Evaluate(model, testData);

            Console.WriteLine($"AccuracyMicro: {metrics.AccuracyMicro}\nAccuracyMacro: {metrics.AccuracyMacro}");

            var          data   = ExcelDataProvider.GetData(@"C:\Users\Дарья\Desktop\TestAdverbNoun.xlsx", 0);
            List <Model> models = new List <Model>();

            data.ForEach(x => models.Add(new Model()
            {
                Category = x.Category,
                Word     = x.Word
            }));

            foreach (var sample in models)
            {
                var prediction = model.Predict(sample);

                Console.WriteLine($"Word: {sample.Word}");
                Console.WriteLine($"\nActual Category: {sample.Category} \nPredicted Category: {prediction.Category}\nPredicted Category scores: [{String.Join(",", prediction.Score)}]\n\n");
            }

            Console.ReadLine();
        }
Ejemplo n.º 24
0
        static void EvaluateOpen311(PredictionModel <Open311Data, Open311DataPrediction> model, string testDataPath)
        {
            var testData  = CollectionDataSource.Create(OpenFile(testDataPath, 3, 0, 1, 2));
            var evaluator = new ClassificationEvaluator();
            var metrics   = evaluator.Evaluate(model, testData);

            Console.WriteLine();
            Console.WriteLine("PredictionModel quality metrics evaluation");
            Console.WriteLine("------------------------------------------");
            Console.WriteLine($"Accuracy Macro: {metrics.AccuracyMacro:P2}");
            Console.WriteLine($"Accuracy Micro: {metrics.AccuracyMicro:P2}");
            Console.WriteLine($"Top KAccuracy: {metrics.TopKAccuracy:P2}");
            Console.WriteLine($"LogLoss: {metrics.LogLoss:P2}");
            for (var classIndex = 0; classIndex < metrics.PerClassLogLoss.Length; classIndex++)
            {
                Console.WriteLine($"Class: {classIndex} - {metrics.PerClassLogLoss[classIndex]:P2}");
            }
        }
Ejemplo n.º 25
0
        public static void Evaluate(PredictionModel <MNISTData, MNISTPrediction> model)
        {
            //Creation d'un analyseur de donnée pour le fichier test
            var testData = new TextLoader(_testDataPath)
            {
                Arguments = new TextLoaderArguments
                {
                    Separator = new[] { ',' },
                    HasHeader = true,
                    Column    = new[]
                    {
                        new TextLoaderColumn()
                        {
                            Name   = "Label",
                            Source = new [] { new TextLoaderRange(0) },
                            Type   = DataKind.Num
                        },

                        new TextLoaderColumn()
                        {
                            Name   = "Features",
                            Source = new [] { new TextLoaderRange(1, 784) },
                            Type   = DataKind.Num
                        }
                    }
                }
            };

            var evaluator = new ClassificationEvaluator();

            //On évalue ici notre modèle selon les données test, les résultats statistiques sont contenus dans metrics
            ClassificationMetrics metrics = evaluator.Evaluate(model, testData);

            Console.WriteLine();
            Console.WriteLine("PredictionModel quality metrics evaluation");
            Console.WriteLine("------------------------------------------");

            //deux exemples de statistques utiles : la précision macroscopique et la matrice de confusion
            Console.WriteLine("Macro Acc : {0}", metrics.AccuracyMacro);
            Console.WriteLine("----------------------------------");
            Console.WriteLine("Confusion Matrix");
            PrintMatrix(metrics.ConfusionMatrix);
        }
Ejemplo n.º 26
0
        private static void EvaluateLanguage(PredictionModel <LanguageModel, LanguagePrediction> model)
        {
            // To evaluate how good the model predicts values, the model is ran against new set
            // of data (test data) that was not involved in training.
            var testData = new TextLoader(DataPath.TestDataPath).CreateFrom <LanguageModel>();

            // ClassificationEvaluator .
            var evaluator = new ClassificationEvaluator();

            Console.WriteLine("=============== Evaluating model ===============");

            var metrics = evaluator.Evaluate(model, testData);

            Console.WriteLine("Metrics:");
            Console.WriteLine($"    AccuracyMacro = {metrics:0.####}, a value between 0 and 1, the closer to 1, the better");
            Console.WriteLine($"    AccuracyMicro = {metrics.AccuracyMicro:0.####}, a value between 0 and 1, the closer to 1, the better");

            Console.WriteLine("=============== End evaluating ===============");
            Console.WriteLine();
        }
        public static async Task Main(List <User> patient)
        {
            //call Train method
            PredictionModel <FoodCustomClass, ClassPrediction> model = Train();
            await model.WriteAsync(_modelPath);

            //Call FoodClassificationTest class for prediction
            var prediction = model.Predict(FoodClassificationTest.FoodClassification());

            //Store each food in a list
            List <FoodCustomClass> foods = FoodClassificationTest.FoodClassification();

            var testData  = new TextLoader(_testdataPath).CreateFrom <FoodCustomClass>(separator: '*');
            var evaluator = new ClassificationEvaluator();
            ClassificationMetrics metrics = evaluator.Evaluate(model, testData);

            // Displaying the metrics for model validation
            System.Diagnostics.Debug.WriteLine("PredictionModel quality metrics evaluation");
            System.Diagnostics.Debug.WriteLine("------------------------------------------");
            System.Diagnostics.Debug.WriteLine($"*       MicroAccuracy:    {metrics.AccuracyMicro:0.###}");
            System.Diagnostics.Debug.WriteLine($"*       MacroAccuracy:    {metrics.AccuracyMacro:0.###}");
            System.Diagnostics.Debug.WriteLine($"*       LogLoss:          {metrics.LogLoss:#.###}");
            System.Diagnostics.Debug.WriteLine($"*       LogLossReduction: {metrics.LogLossReduction:#.###}");


            //Combine List foods with their respective results after classification

            /*
             * foodsAndClassification contain each food from FoodDB database classified in terms
             * of the quantity of each macronutrient present
             */
            var foodsAndClassification = foods.Zip(prediction, (n, w) => new { Food = n, Category = w });

            //call ID3Tree.cs and send foodsAndClassification as parameter to ID3Tree constructor
            var function = new ID3Tree().ID3(foodsAndClassification, patient);

            //foreach (var nw in foodsAndClassification)
            //{
            //    System.Diagnostics.Debug.WriteLine($"{nw.Food.Food}: {nw.Category.Predicted}" +"\n" + $"{float.Parse(nw.Category.Score.First().ToString(), CultureInfo.InvariantCulture.NumberFormat)}");
            //}
        }
Ejemplo n.º 28
0
        public void SetupPredictBenchmarks()
        {
            _trainedModel = Train(_dataPath);
            _consumer.Consume(_trainedModel.Predict(_example));

            var testData  = new Legacy.Data.TextLoader(_dataPath).CreateFrom <IrisData>(useHeader: true);
            var evaluator = new ClassificationEvaluator();

            _metrics = evaluator.Evaluate(_trainedModel, testData);

            _batches = new IrisData[_batchSizes.Length][];
            for (int i = 0; i < _batches.Length; i++)
            {
                var batch = new IrisData[_batchSizes[i]];
                _batches[i] = batch;
                for (int bi = 0; bi < batch.Length; bi++)
                {
                    batch[bi] = _example;
                }
            }
        }
Ejemplo n.º 29
0
        //test the model
        public static void Evaluate(PredictionModel <SentimentData, SentimentPrediction> model)
        {
            //load the test data
            var testData = new TextLoader(_testDataPath).CreateFrom <SentimentData>(useHeader: true);

            //computes the quality metrics for the PredictionModel
            var evaluator = new ClassificationEvaluator();

            //to get metrices computed by binary classification evaluator
            ClassificationMetrics metrics = evaluator.Evaluate(model, testData);

            Console.WriteLine();
            Console.WriteLine("PredictionModel quality metrics evaluation");
            Console.WriteLine("------------------------------------------");
            Console.WriteLine($"LogLoss: { metrics.LogLoss:P2}");
            //Console.WriteLine($"ConfusionMatrix: { metrics.ConfusionMatrix:P2}");
            Console.WriteLine($"AccuracyMicro: { metrics.AccuracyMicro:P2}");
            Console.WriteLine($"Accuracy: {metrics.AccuracyMacro:P2}");
            //Console.WriteLine($"Auc: {metrics.Auc:P2}");
            //Console.WriteLine($"F1Score: {metrics.F1Score:P2}");
        }
        public void Setup()
        {
            s_dataPath     = Program.GetDataPath("iris.txt");
            s_trainedModel = TrainCore();
            IrisPrediction prediction = s_trainedModel.Predict(s_example);

            var testData  = new TextLoader(s_dataPath).CreateFrom <IrisData>(useHeader: true);
            var evaluator = new ClassificationEvaluator();

            s_metrics = evaluator.Evaluate(s_trainedModel, testData);

            s_batches = new IrisData[s_batchSizes.Length][];
            for (int i = 0; i < s_batches.Length; i++)
            {
                var batch = new IrisData[s_batchSizes[i]];
                s_batches[i] = batch;
                for (int bi = 0; bi < batch.Length; bi++)
                {
                    batch[bi] = s_example;
                }
            }
        }