public static void Evaluate(MLContext mlContext, ITransformer model, IDataView splitTestSet) { Console.WriteLine("=============== Evaluating Model accuracy with Test data==============="); IDataView predictions = model.Transform(splitTestSet); CalibratedBinaryClassificationMetrics metrics = mlContext.BinaryClassification.Evaluate(predictions, "Label"); Console.WriteLine(); Console.WriteLine("Model quality metrics evaluation"); Console.WriteLine("--------------------------------"); Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}"); Console.WriteLine($"Auc: {metrics.Auc:P2}"); Console.WriteLine($"F1Score: {metrics.F1Score:P2}"); Console.WriteLine("=============== End of model evaluation ==============="); SaveModelAsFile(mlContext, model); }
public void Test(string testDL, string modelLocation) { Console.WriteLine("Testing..."); Load(modelLocation); testDataView = mlContext.Data.LoadFromTextFile <SentimentIssue>(testDL, hasHeader: true); var predictions = trainedModel.Transform(testDataView); metrics = mlContext.BinaryClassification.Evaluate(data: predictions, labelColumnName: "Label", scoreColumnName: "Score"); Console.WriteLine("Accuracy: {0}\n" + "TestDataFile: {1}\n" + "TrainDataFile: {2}\n" + "TrainedModelLocation: {3}", metrics.Accuracy, testDataLocation, trainDataLocation, modelLocation); Console.WriteLine("Done Testing, Press Any Key To Exit..."); Console.ReadKey(); }
public static void PrintBinaryClassificationMetrics(string name, CalibratedBinaryClassificationMetrics metrics) { Console.WriteLine($"************************************************************"); Console.WriteLine($"* Metrics for {name} binary classification model "); Console.WriteLine($"*-----------------------------------------------------------"); Console.WriteLine($"* Accuracy: {metrics.Accuracy:P2}"); Console.WriteLine($"* F1Score: {metrics.F1Score:P2}"); Console.WriteLine($"* LogLoss: {metrics.LogLoss:#.##}"); Console.WriteLine($"* LogLossReduction: {metrics.LogLossReduction:#.##}"); Console.WriteLine($"* PositivePrecision: {metrics.PositivePrecision:#.##}"); Console.WriteLine($"* PositiveRecall: {metrics.PositiveRecall:#.##}"); Console.WriteLine($"* NegativePrecision: {metrics.NegativePrecision:#.##}"); Console.WriteLine($"* NegativeRecall: {metrics.NegativeRecall:P2}"); Console.WriteLine($"************************************************************"); }
public static void Evaluate(MLContext mlContext, ITransformer model, IDataView splitTestSet) { Console.WriteLine("=============== Evaluating Model accuracy with Test data==============="); IDataView predictions = model.Transform(splitTestSet); CalibratedBinaryClassificationMetrics metrics = mlContext.BinaryClassification.Evaluate(predictions, "Label"); Console.WriteLine(); Console.WriteLine("Model quality metrics evaluation"); Console.WriteLine("--------------------------------"); Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}"); Console.WriteLine($"Auc: {metrics.Auc:P2}"); Console.WriteLine($"F1Score: {metrics.F1Score:P2}"); Console.WriteLine("=============== End of model evaluation ==============="); using (var fs = new FileStream(_modelPath, FileMode.Create, FileAccess.Write, FileShare.Write)) mlContext.Model.Save(model, fs); }
public static void PrintBinaryClassificationMetrics(string name, CalibratedBinaryClassificationMetrics metrics) { s_log.Debug($"************************************************************"); s_log.Debug($"* Metrics for {name} binary classification model "); s_log.Debug($"*-----------------------------------------------------------"); s_log.Debug($"* Accuracy: {metrics.Accuracy:P2}"); s_log.Debug($"* Area Under Curve: {metrics.AreaUnderRocCurve:P2}"); s_log.Debug($"* Area under Precision recall Curve: {metrics.AreaUnderPrecisionRecallCurve:P2}"); s_log.Debug($"* F1Score: {metrics.F1Score:P2}"); s_log.Debug($"* LogLoss: {metrics.LogLoss:#.##}"); s_log.Debug($"* LogLossReduction: {metrics.LogLossReduction:#.##}"); s_log.Debug($"* PositivePrecision: {metrics.PositivePrecision:#.##}"); s_log.Debug($"* PositiveRecall: {metrics.PositiveRecall:#.##}"); s_log.Debug($"* NegativePrecision: {metrics.NegativePrecision:#.##}"); s_log.Debug($"* NegativeRecall: {metrics.NegativeRecall:P2}"); s_log.Debug($"************************************************************"); }
public static void PrintBinaryClassificationMetrics(string name, CalibratedBinaryClassificationMetrics metrics) { Console.WriteLine($"************************************************************"); Console.WriteLine($"* 二进制分类模型的指标 {name} "); Console.WriteLine($"*-----------------------------------------------------------"); Console.WriteLine($"* Accuracy: {metrics.Accuracy:P2}"); Console.WriteLine($"* Area Under Curve: {metrics.AreaUnderRocCurve:P2}"); Console.WriteLine($"* Area under Precision recall Curve: {metrics.AreaUnderPrecisionRecallCurve:P2}"); Console.WriteLine($"* F1Score: {metrics.F1Score:P2}"); Console.WriteLine($"* LogLoss: {metrics.LogLoss:#.##}"); Console.WriteLine($"* LogLossReduction: {metrics.LogLossReduction:#.##}"); Console.WriteLine($"* PositivePrecision: {metrics.PositivePrecision:#.##}"); Console.WriteLine($"* PositiveRecall: {metrics.PositiveRecall:#.##}"); Console.WriteLine($"* NegativePrecision: {metrics.NegativePrecision:#.##}"); Console.WriteLine($"* NegativeRecall: {metrics.NegativeRecall:P2}"); Console.WriteLine($"************************************************************"); }
public static void Evaluate(MLContext mlContext, ITransformer model, IDataView splitTestSet) { Console.WriteLine("========== Evaluating Model accuracy with Test data =========="); // Transform the splitTestSet data by adding the following code to Evaluate(): IDataView predictions = model.Transform(splitTestSet); // Evaluate the model by adding the following as the next line of code in the Evaluate() method: // Once you have the prediction set, Evaluate() assesses the model, which compares the predicted values with the actual Labels in the test dataset // It returns a CalibratedBinaryClassificationMetrics object on how the model is performing. CalibratedBinaryClassificationMetrics metrics = mlContext.BinaryClassification.Evaluate(predictions, "Label"); Console.WriteLine("Model quality metrics evaluation:"); Console.WriteLine($"- Accuracy: {metrics.Accuracy:P2}"); Console.WriteLine($"- Auc: {metrics.AreaUnderRocCurve:P2}"); Console.WriteLine($"- F1Score: {metrics.F1Score:P2}"); Console.WriteLine("========== End of model evaluation =========="); }
public static void Evaluate(MLContext mlContext, ITransformer model, IDataView splitTestSet) { // Evaluate the model and show accuracy stats //Take the data in, make transformations, output the data. // <SnippetTransformData> Console.WriteLine("=============== Evaluating Model accuracy with Test data==============="); IDataView predictions = model.Transform(splitTestSet); // </SnippetTransformData> // BinaryClassificationContext.Evaluate returns a BinaryClassificationEvaluator.CalibratedResult // that contains the computed overall metrics. // <SnippetEvaluate> CalibratedBinaryClassificationMetrics metrics = mlContext.BinaryClassification.Evaluate(predictions, "Label"); // </SnippetEvaluate> // The Accuracy metric gets the accuracy of a classifier, which is the proportion // of correct predictions in the test set. // The Auc metric gets the area under the ROC curve. // The area under the ROC curve is equal to the probability that the classifier ranks // a randomly chosen positive instance higher than a randomly chosen negative one // (assuming 'positive' ranks higher than 'negative'). // The F1Score metric gets the classifier's F1 score. // The F1 score is the harmonic mean of precision and recall: // 2 * precision * recall / (precision + recall). // <SnippetDisplayMetrics> Console.WriteLine(); Console.WriteLine("Model quality metrics evaluation"); Console.WriteLine("--------------------------------"); Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}"); Console.WriteLine($"Auc: {metrics.Auc:P2}"); Console.WriteLine($"F1Score: {metrics.F1Score:P2}"); Console.WriteLine("=============== End of model evaluation ==============="); //</SnippetDisplayMetrics> // Save the new model to .ZIP file // <SnippetCallSaveModel> SaveModelAsFile(mlContext, model); // </SnippetCallSaveModel> }
public static void Evaluate(ITransformer model, IDataView testSet) { Console.WriteLine("=============== Evaluating Model accuracy with Test data==============="); CalibratedBinaryClassificationMetrics metrics = GetMetrics(model, testSet); Console.WriteLine("Sensitivity: {0}", metrics.PositiveRecall); Console.WriteLine("Specificity: {0}", metrics.NegativeRecall); Console.WriteLine(); Console.WriteLine("Model quality metrics evaluation"); Console.WriteLine("--------------------------------"); Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}"); Console.WriteLine($"Auc: {metrics.AreaUnderRocCurve:P2}"); Console.WriteLine($"F1Score: {metrics.F1Score:P2}"); Output("AUC" + metrics.AreaUnderRocCurve); Output("Accuracy" + metrics.Accuracy); Output("Specificity" + metrics.NegativeRecall); Console.WriteLine("=============== End of model evaluation ==============="); }
// Function used to show evaluation metrics such as accuracy of predictions. private static void PrintMetrics( CalibratedBinaryClassificationMetrics metrics) { Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); Console.WriteLine($"Negative Precision: " + $"{metrics.NegativePrecision:F2}"); Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); Console.WriteLine($"Positive Precision: " + $"{metrics.PositivePrecision:F2}"); Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}"); Console.WriteLine($"Log Loss: {metrics.LogLoss:F2}"); Console.WriteLine($"Log Loss Reduction: {metrics.LogLossReduction:F2}"); Console.WriteLine($"Entropy: {metrics.Entropy:F2}"); }
public static void Evaluate(MLContext mlContext, ITransformer model, IDataView splitTestSet) { Console.WriteLine("=============== Evaluating Model accuracy with Test data==============="); IDataView predictions = model.Transform(splitTestSet); // Transform() method to make predictions for multiple provided input rows of a test dataset. /* * Evaluate() method assesses the model, which compares the predicted values with the actual Labels in the test dataset * and returns a CalibratedBinaryClassificationMetrics object on how the model is performing. */ CalibratedBinaryClassificationMetrics metrics = mlContext.BinaryClassification.Evaluate(predictions, "Label"); Console.WriteLine(); Console.WriteLine("Model quality metrics evaluation"); Console.WriteLine("--------------------------------"); Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}"); //accuracy of a model, which is the proportion of correct predictions in the test set. Console.WriteLine($"Auc: {metrics.AreaUnderRocCurve:P2}"); //how confident the model is correctly classifying the positive and negative classes. You want the AreaUnderRocCurve to be as close to one as possible. Console.WriteLine($"F1Score: {metrics.F1Score:P2}"); //measure of balance between precision and recall. You want the F1Score to be as close to one as possible. Console.WriteLine("=============== End of model evaluation ==============="); }
/// <summary> /// 加载测试数据集。 /// 创建 BinaryClassification 计算器。 /// 评估模型并创建指标。 /// 显示指标。 /// </summary> /// <param name="mlContext">环境</param> /// <param name="model">训练好的模型</param> /// <param name="splitTestSet">训练集</param> public static void Evaluate(MLContext mlContext, ITransformer model, IDataView splitTestSet) { // Evaluate the model and show accuracy stats //Take the data in, make transformations, output the data. // 用测试集对模型进行评估/只是把测试集加载进去(懒加载的形式) 预测集 // <SnippetTransformData> Console.WriteLine("=============== Evaluating Model accuracy with Test data==============="); IDataView predictions = model.Transform(splitTestSet); // </SnippetTransformData> // BinaryClassificationContext.Evaluate returns a BinaryClassificationEvaluator.CalibratedResult // that contains the computed overall metrics. // 把预测集 扔进去 指定测试集中的lable 对进度进行评估 返回评估的打分 // <SnippetEvaluate> CalibratedBinaryClassificationMetrics metrics = mlContext.BinaryClassification.Evaluate(predictions, "Label"); // </SnippetEvaluate> // The Accuracy metric gets the accuracy of a model, which is the proportion // of correct predictions in the test set. // The AreaUnderROCCurve metric is equal to the probability that the algorithm ranks // a randomly chosen positive instance higher than a randomly chosen negative one // (assuming 'positive' ranks higher than 'negative'). // The F1Score metric gets the model's F1 score. // The F1 score is the harmonic mean of precision and recall: // 2 * precision * recall / (precision + recall). // <SnippetDisplayMetrics> Console.WriteLine(); Console.WriteLine("Model quality metrics evaluation/ 模型质量量度评估"); Console.WriteLine("--------------------------------"); //Accuracy 指标可获取模型的准确性,即测试集中正确预测所占的比例。 Console.WriteLine($"Accuracy/正确比例: {metrics.Accuracy:P2}"); //AreaUnderRocCurve 指标指示模型对正面类和负面类进行正确分类的置信度。 应该使 AreaUnderRocCurve 尽可能接近 1。 Console.WriteLine($"Auc/todo鉴别: {metrics.AreaUnderRocCurve:P2}"); //F1Score 指标可获取模型的 F1 分数,该分数是查准率和查全率之间的平衡关系的度量值。 应该使 F1Score 尽可能接近 1。 Console.WriteLine($"F1Score/模型的F1得分: {metrics.F1Score:P2}"); Console.WriteLine("=============== End of model evaluation ==============="); //</SnippetDisplayMetrics> }
public static void Evaluate(MLContext mlContext, ITransformer model, IDataView splitTestSet) { // Evaluate the model and show accuracy stats Console.WriteLine("=== Evaluating model accuracy with test data ==="); //Take the data in, make transformations, output the data. IDataView predictions = model.Transform(splitTestSet); // BinaryClassificationContext.Evaluate returns computed overall metrics. CalibratedBinaryClassificationMetrics metrics = mlContext.BinaryClassification.Evaluate(predictions, "Label"); Console.WriteLine(); Console.WriteLine("Model quality metrics evaluation"); Console.WriteLine("--------------------------------"); Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}"); Console.WriteLine($"Auc: {metrics.Auc:P2}"); Console.WriteLine($"F1Score: {metrics.F1Score:P2}"); Console.WriteLine("=== End of model evaluation ==="); // Save the new model to .ZIP file SaveModelAsFile(mlContext, model); }
/// <summary> /// At the time when the ~10% of the data gets chosen for training, another 10% gets chosen for evaluation. Then after training, /// the effectiveness of the model gets evaluated on the test set. The results of that evaluation are converted to text values called /// BinarySearchTreeMetrics and this gets written to the results.tsv /// </summary> /// <param name="name"></param> /// <param name="metrics"></param> /// <returns></returns> public static string PrintBinaryClassificationMetrics(string name, CalibratedBinaryClassificationMetrics metrics) { StringBuilder s = new StringBuilder(); s.AppendLine("************************************************************"); s.AppendLine("* Metrics for Determination of PEP Using Binary Classification "); s.AppendLine("*-----------------------------------------------------------"); s.AppendLine("* Accuracy: " + metrics.Accuracy.ToString()); s.AppendLine("* Area Under Curve: " + metrics.AreaUnderRocCurve.ToString()); s.AppendLine("* Area under Precision recall Curve: " + metrics.AreaUnderPrecisionRecallCurve.ToString()); s.AppendLine("* F1Score: " + metrics.F1Score.ToString()); s.AppendLine("* LogLoss: " + metrics.LogLoss.ToString()); s.AppendLine("* LogLossReduction: " + metrics.LogLossReduction.ToString()); s.AppendLine("* PositivePrecision: " + metrics.PositivePrecision.ToString()); s.AppendLine("* PositiveRecall: " + metrics.PositiveRecall.ToString()); s.AppendLine("* NegativePrecision: " + metrics.NegativePrecision.ToString()); s.AppendLine("* NegativeRecall: " + metrics.NegativeRecall.ToString()); s.AppendLine("************************************************************"); return(s.ToString()); }
public void Evaluate() { if (initalized) { Console.WriteLine("=============== Evaluating Model accuracy with Test data==============="); IDataView predictions = model.Transform(splitDataView.TestSet); CalibratedBinaryClassificationMetrics metrics = mlContext.BinaryClassification.Evaluate(predictions, "Label"); Console.WriteLine(); Console.WriteLine("Model quality metrics evaluation"); Console.WriteLine("--------------------------------"); Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}"); Console.WriteLine($"Auc: {metrics.AreaUnderRocCurve:P2}"); Console.WriteLine($"F1Score: {metrics.F1Score:P2}"); Console.WriteLine("=============== End of model evaluation ==============="); } else { Console.WriteLine("Error! Sentiment Analyser not initalized."); } }
public static void Evaluate(MLContext mlContext, ITransformer model, IDataView testSet) { // Transform the test set in order to make predictions Console.WriteLine("========== Evaluating Model Accuracy with Test Data =========="); IDataView predictions = model.Transform(testSet); // Compare predicted values with actual labels to assess model's performance CalibratedBinaryClassificationMetrics metrics = mlContext.BinaryClassification.Evaluate(predictions, "Label"); // Display metrics // Accuracy proportion of correct predictions in the test set // AUC is model confidence in classifying positive/negative classes. The closer to 1, the better // F1 Score is measure of balance between precision and recall. The closer to 1, the better Console.WriteLine(); Console.WriteLine("Model Quality Metrics Evaluation"); Console.WriteLine("--------------------------------"); Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}"); Console.WriteLine($"Area Under Roc Curve: {metrics.AreaUnderRocCurve:P2}"); Console.WriteLine($"F1Score: {metrics.F1Score:P2}"); Console.WriteLine("========== End of Model Evaluation =========="); }
/// <summary> /// Evaluate value /// </summary> /// <param name="mlContext"></param> /// <param name="model"></param> /// <param name="splitTestSet"></param> public static void Evaluate(MLContext mlContext, ITransformer model, IDataView splitTestSet) { IDataView predictions = null; try { predictions = model.Transform(splitTestSet); CalibratedBinaryClassificationMetrics metrics = mlContext.BinaryClassification.Evaluate(predictions, "Label"); //Console.WriteLine(); //Console.WriteLine("Model quality metrics evaluation"); //Console.WriteLine("--------------------------------"); //Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}"); //Console.WriteLine($"Auc: {metrics.AreaUnderRocCurve:P2}"); //Console.WriteLine($"F1Score: {metrics.F1Score:P2}"); //Console.WriteLine("=============== End of model evaluation ==============="); } catch (Exception ex) { Console.WriteLine(ex.StackTrace); } }
public void Evaluate(MLContext mlContext, ITransformer model, IDataView splitTestSet, string labelColumn) { // This code was taken from the linear regession demo on the ML.NET github page // Evaluate the model and show accuracy stats //Take the data in, make transformations, output the data. // <SnippetTransformData> Console.WriteLine("=============== Evaluating Model accuracy with Test data==============="); IDataView predictions = model.Transform(splitTestSet); // </SnippetTransformData> // BinaryClassificationContext.Evaluate returns a BinaryClassificationEvaluator.CalibratedResult // that contains the computed overall metrics. // <SnippetEvaluate> CalibratedBinaryClassificationMetrics metrics = mlContext.BinaryClassification.Evaluate(predictions, labelColumn); // </SnippetEvaluate> // The Accuracy metric gets the accuracy of a model, which is the proportion // of correct predictions in the test set. // The AreaUnderROCCurve metric is equal to the probability that the algorithm ranks // a randomly chosen positive instance higher than a randomly chosen negative one // (assuming 'positive' ranks higher than 'negative'). // The F1Score metric gets the model's F1 score. // The F1 score is the harmonic mean of precision and recall: // 2 * precision * recall / (precision + recall). // <SnippetDisplayMetrics> Console.WriteLine(); Console.WriteLine("Model quality metrics evaluation"); Console.WriteLine("--------------------------------"); Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}"); Console.WriteLine($"Auc: {metrics.AreaUnderRocCurve:P2}"); Console.WriteLine($"F1Score: {metrics.F1Score:P2}"); Console.WriteLine("\n=============== End of model evaluation ==============="); //</SnippetDisplayMetrics> }
SetThreshold(BinaryPredictionTransformer <Microsoft.ML.Calibrators.CalibratedModelParametersBase <LinearBinaryModelParameters, Microsoft.ML.Calibrators.PlattCalibrator> > lrModel, IDataView testSet) { float threshold = 1.0F; double currentSpecificity = 1.0; do { threshold -= tick; CalibratedBinaryClassificationMetrics metrics = GetMetrics((ITransformer)lrModel, testSet); currentSpecificity = metrics.NegativeRecall; double AUC = metrics.AreaUnderRocCurve; Console.WriteLine("Threshold: {0:0.00}; Specificity: {1:0.00}; AUC: {2:0.00}", threshold, currentSpecificity, AUC); lrModel = mlContext.BinaryClassification.ChangeModelThreshold(lrModel, threshold); Thread.Sleep(5); } while (currentSpecificity > minimumSpecificity); return(lrModel); }
public static void Evaluate(MLContext mlContext, ITransformer model, IDataView splitTestSet) { // Evaluate the model and show accuracy stats //Take the data in, make transformations, output the data. // <SnippetTransformData> Console.WriteLine("=============== Evaluating Model accuracy with Test data==============="); IDataView predictions = model.Transform(splitTestSet); // </SnippetTransformData> // BinaryClassificationContext.Evaluate returns a BinaryClassificationEvaluator.CalibratedResult // that contains the computed overall metrics. // <SnippetEvaluate> CalibratedBinaryClassificationMetrics metrics = mlContext.BinaryClassification.Evaluate(predictions, "Label"); // </SnippetEvaluate> // The Accuracy metric gets the accuracy of a model, which is the proportion // of correct predictions in the test set. // The AreaUnderRocCurve metric is an indicator of how confident the model is // correctly classifying the positive and negative classes as such. // The F1Score metric gets the model's F1 score. // F1 is a measure of tradeoff between precision and recall. // 2 * precision * recall / (precision + recall). // <SnippetDisplayMetrics> Console.WriteLine(); Console.WriteLine("Model quality metrics evaluation"); Console.WriteLine("--------------------------------"); Console.WriteLine($" Accuracy: {metrics.Accuracy:P2}"); Console.WriteLine($"Area Under Roc Curve: {metrics.AreaUnderRocCurve:P2}"); Console.WriteLine($" F1Score: {metrics.F1Score:P2}"); Console.WriteLine("=============== End of model evaluation ==============="); //</SnippetDisplayMetrics> }
public StringBuilder Evaluate(MLContext mlContext, ITransformer model, IDataView splitTestSet) { StringBuilder builder = new StringBuilder(); builder.AppendLine("=============== Evaluating Model accuracy with Test data==============="); IDataView predictions = model.Transform(splitTestSet); //Transform() method makes predictions for multiple provided input rows of a test dataset. CalibratedBinaryClassificationMetrics metrics = mlContext.BinaryClassification.Evaluate(predictions, "Label"); #region Comments#2 //Once you have the prediction set(predictions), the Evaluate() method assesses the model, //which compares the predicted values with the actual Labels in the test dataset and //returns a CalibratedBinaryClassificationMetrics object on how the model is performing. #endregion builder.AppendLine("Model quality metrics evaluation"); builder.AppendLine("--------------------------------"); builder.AppendLine($"Accuracy: {metrics.Accuracy:P2}"); builder.AppendLine($"Auc: {metrics.AreaUnderRocCurve:P2}"); builder.AppendLine($"F1Score: {metrics.F1Score:P2}"); builder.AppendLine("=============== End of model evaluation ==============="); #region Comments#3 //The Accuracy metric gets the accuracy of a model, which is the proportion of correct //predictions in the test set. //The AreaUnderRocCurve metric indicates how confident the model is correctly //classifying the positive and negative classes. You want the AreaUnderRocCurve to //be as close to one as possible. //The F1Score metric gets the model's F1 score, which is a measure of balance //between precision and recall. You want the F1Score to be as close to one as possible. #endregion return(builder); }
static void Main(string[] args) { MLContext mlContext = new MLContext(); // Load Data IDataView dataView = mlContext.Data.LoadFromTextFile <SentimentData>(_dataPath, hasHeader: false); TrainTestData splitDataView = mlContext.Data.TrainTestSplit(dataView, testFraction: 0.2); // BuildAndTrainModel var estimator = mlContext.Transforms.Text .FeaturizeText(outputColumnName: "Features", inputColumnName: nameof(SentimentData.SentimentText)) .Append(mlContext.BinaryClassification.Trainers.SdcaLogisticRegression(labelColumnName: "Label", featureColumnName: "Features", maximumNumberOfIterations: 100)); Console.WriteLine("=============== Create and Train the Model ==============="); var model = estimator.Fit(splitDataView.TrainSet); Console.WriteLine("=============== End of training ==============="); Console.WriteLine(); // Evaluate Console.WriteLine("=============== Evaluating Model accuracy with Test data==============="); IDataView predictions = model.Transform(splitDataView.TestSet); CalibratedBinaryClassificationMetrics metrics = mlContext.BinaryClassification.Evaluate(predictions, "Label"); Console.WriteLine(); Console.WriteLine("Model quality metrics evaluation"); Console.WriteLine("--------------------------------"); Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}"); Console.WriteLine($"Auc: {metrics.AreaUnderRocCurve:P2}"); Console.WriteLine($"F1Score: {metrics.F1Score:P2}"); Console.WriteLine("=============== End of model evaluation ==============="); // UseModelWithSingleItem PredictionEngine <SentimentData, SentimentPrediction> predictionFunction = mlContext.Model.CreatePredictionEngine <SentimentData, SentimentPrediction>(model); SentimentData sampleStatement = new SentimentData { SentimentText = "이 영화 정말 재미없어요" }; var resultPrediction = predictionFunction.Predict(sampleStatement); Console.WriteLine(); Console.WriteLine("=============== Prediction Test of model with a single sample and test dataset==============="); Console.WriteLine(); Console.WriteLine($"Sentiment: {resultPrediction.SentimentText} | Prediction: {(Convert.ToBoolean(resultPrediction.Prediction) ? "Positive" : "Negative")} | Probability: {resultPrediction.Probability} "); Console.WriteLine("=============== End of Predictions ==============="); Console.WriteLine(); // UseModelWithBatchItems IEnumerable <SentimentData> sentiments = new[] { new SentimentData { SentimentText = "지루한 영화에요" }, new SentimentData { SentimentText = "이거 정말 최고에요!" }, new SentimentData { SentimentText = "올해의 영화로 손꼽고 싶군요" } }; IDataView batchComments = mlContext.Data.LoadFromEnumerable(sentiments); predictions = model.Transform(batchComments); // Use model to predict whether comment data is Positive (1) or Negative (0). IEnumerable <SentimentPrediction> predictedResults = mlContext.Data.CreateEnumerable <SentimentPrediction>(predictions, reuseRowObject: false); Console.WriteLine(); Console.WriteLine("=============== Prediction Test of loaded model with multiple samples ==============="); foreach (SentimentPrediction prediction in predictedResults) { Console.WriteLine($"Sentiment: {prediction.SentimentText} | Prediction:{ (Convert.ToBoolean(prediction.Prediction) ? "Positive" : "Negative")} | Probability:{ prediction.Probability}"); } Console.WriteLine("=============== End of predictions ==============="); }
private void Evaluate(IDataView splitTestSet) { IDataView predictions = model.Transform(splitTestSet); metrics = mlContext.BinaryClassification.Evaluate(predictions, "Label"); }
static void Main(string[] args) { MLContext ml = new MLContext(seed: 1); var trainData = ml.Data.LoadFromTextFile <MlInput>("learningdata.csv", ',', hasHeader: true); //var select = ml.Transforms.SelectColumns("Airlines", "Architecture", "Art & Museums", "Automotive", "Banking & Financial", "Cannabis", "Casinos & Lottery"); //var selectTransform = select.Fit(trainData).Transform(trainData); var inputcolumns = new List <string> { }; double f1sum = 0; int count = 0; var techlist = new List <string> { "360 Photo/Video", "3D Environments", "3D Modelling", "AR", "AR Authoring", "AR Cloud", "AR Headsets", "AR Kit", "AR: Hololens", "AR: Magic Leap", "ARCore", "Avatar Creation", "Distribution & Device Management", "Eye Tracking", "Haptics & Peripherals", "Motion Capture", "Motion Simulators", "Networking", "Photogrammetry", "Spatial Audio", "Unity", "Unreal", "VR", "VR Authoring", "VR Headsets", "VR: Mobile", "VR: PC", "Volumetric Capture", "Web XR" }; foreach (var technology in techlist.ToList()) { var removedlist = techlist.ToList(); removedlist.Remove(technology); string[] removedarray = removedlist.ToArray(); var prepedData = ml.Transforms.DropColumns(removedarray); var prepedDataTransform = prepedData.Fit(trainData).Transform(trainData); TrainTestData trainTestData = ml.Data.TrainTestSplit(prepedDataTransform, testFraction: 0.2); IEstimator <ITransformer> dataPipe = ml.Transforms.Concatenate("Features", new[] { "Airlines", "Architecture", "Art & Museums", "Automotive", "Banking & Financial", "Cannabis", "Casinos & Lottery", "Charities", "Education & Training", "Education (K-12)", "Emergency Response", "Health & Medical", "Industrial", "Legal & Insurance", "Media & News & Entertainment", "Military", "Music", "Real Estate", "Restaurant & Food", "Retail", "Sales & Marketing", "Sports", "Telecommunications", "Travel & Tourism", "Collaboration & Social", "Communications", "Data Analytics", "Design", "Gaming", "General Training", "Health & Safety", "LBE", "Leadership", "Marketing", "Rehabilitation", "Simulation", "Tools", "Training: Hands On", "Training: Hard Skills", "Training: Soft Skills" }); var options = new LbfgsLogisticRegressionBinaryTrainer.Options() { LabelColumnName = technology, FeatureColumnName = "Features", MaximumNumberOfIterations = 100, OptimizationTolerance = 1e-8f }; var lbfsgLogistic = ml.BinaryClassification.Trainers.LbfgsLogisticRegression(options); var trainPipe = dataPipe.Append(lbfsgLogistic); Display(trainTestData.TrainSet); var model = trainPipe.Fit(trainTestData.TrainSet); IDataView predictionView = model.Transform(trainTestData.TestSet); CalibratedBinaryClassificationMetrics metrics = ml.BinaryClassification.Evaluate(predictionView, labelColumnName: technology); //Console.WriteLine($"F1 {metrics.ConfusionMatrix.GetFormattedConfusionTable().ToString()} "); Console.WriteLine($"F1 {metrics.F1Score} "); if (metrics.F1Score > 0) { f1sum += metrics.F1Score; count++; } } Console.WriteLine($"Average: {f1sum/count}"); }
public static void Evaluate(MLContext mlContext, ITransformer model, IDataView splitTestSet) { IDataView predictions = model.Transform(splitTestSet); CalibratedBinaryClassificationMetrics metrics = mlContext.BinaryClassification.Evaluate(predictions, "Label"); }
public async Task <IActionResult> OnPostAsync() { if (await _context.AccountDatasets.FirstOrDefaultAsync(x => x.DatasetId == DatasetId && x.AccountId == HttpContext.Session.GetInt32("id")) == null) { return(RedirectToPage("Error")); } Models.Dataset dataset = await _context.Datasets.FirstOrDefaultAsync(x => x.Id == DatasetId); List <Models.Article> articles = await _context.Articles.Where(x => x.DatasetId == dataset.Id).ToListAsync(); if (articles.Count(x => x.Classification) < 5 || articles.Count(x => !x.Classification) < 5) { ModelState.AddModelError("DatasetId", "Dataset must have at least 5 articles of each classification."); } if (!ModelState.IsValid) { Datasets = await _context.Datasets.Where(x => x.AccountDatasets.Any(y => y.DatasetId == x.Id && y.AccountId == HttpContext.Session.GetInt32("id"))).ToListAsync(); return(Page()); } List <ArticleData> data = articles.Select(x => new ArticleData() { Text = x.Title + " " + x.Abstract, Classification = x.Classification }).ToList(); MLContext mlContext = new MLContext(); IDataView dataView = mlContext.Data.LoadFromEnumerable(data); TrainTestData splitDataView = mlContext.Data.TrainTestSplit(dataView, 0.2); var estimator = mlContext.Transforms.Text.FeaturizeText(outputColumnName: "Features", inputColumnName: nameof(ArticleData.Text)) .Append(mlContext.BinaryClassification.Trainers.SdcaLogisticRegression(labelColumnName: "Label", featureColumnName: "Features")); ITransformer model = estimator.Fit(splitDataView.TrainSet); IDataView predictions = model.Transform(splitDataView.TestSet); CalibratedBinaryClassificationMetrics metrics = mlContext.BinaryClassification.Evaluate(predictions, "Label"); var guid = Guid.NewGuid().ToString(); var path = Path.Combine(_environment.WebRootPath, "classifiers", $"{guid}.zip"); mlContext.Model.Save(model, dataView.Schema, path); Models.Classifier classifier = new Models.Classifier() { Name = Name, Accuracy = metrics.Accuracy, Precision = metrics.PositivePrecision, Recall = metrics.PositiveRecall, Date = DateTime.Now, Model = guid }; Models.Account account = await _context.Accounts.FirstOrDefaultAsync(x => x.Id == HttpContext.Session.GetInt32("id")); _context.Classifiers.Add(classifier); _context.AccountClassifiers.Add(new Models.AccountClassifier() { Account = account, Classifier = classifier }); await _context.SaveChangesAsync(); return(RedirectToPage("Classifiers")); }