/// <summary>
/// Evaluates the trained model for quality assurance against a second data set.
///
/// Loads the test dataset.
/// Creates the binary evaluator.
/// Evaluates the model and create metrics.
///
/// Displays the metrics.
/// </summary>
/// <param name="model"></param>
internal static void Evaluate(
PredictionModel <ClassificationData, SentimentPrediction> model,
InputData input)
{
// loads the new test dataset with the same schema.
// You can evaluate the model using this dataset as a quality check.
//var testData = new TextLoader(_testDataPath).CreateFrom<SentimentData>();
var testData = new TextLoader(input.TestData).CreateFrom <ClassificationData>();
// Computes the quality metrics for the PredictionModel using the specified dataset.
var evaluator = new ClassificationEvaluator();
// The BinaryClassificationMetrics contains the overall metrics computed by binary
// classification evaluators. To display these to determine the quality of the model,
// you need to get the metrics first.
ClassificationMetrics metrics = evaluator.Evaluate(model, testData);
// Displaying the metrics for model validation
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
Console.WriteLine($"Accuracy Macro: {metrics.AccuracyMacro:P2}");
Console.WriteLine($"Accuracy Micro: {metrics.AccuracyMicro:P2}");
Console.WriteLine($" Top KAccuracy: {metrics.TopKAccuracy:P2}");
}
public void TrainOneVersusAll()
{
string dataPath = GetDataPath("iris.txt");
var pipeline = new Legacy.LearningPipeline(seed: 1, conc: 1);
pipeline.Add(new TextLoader(dataPath).CreateFrom <IrisData>(useHeader: false));
pipeline.Add(new ColumnConcatenator(outputColumn: "Features",
"SepalLength", "SepalWidth", "PetalLength", "PetalWidth"));
pipeline.Add(OneVersusAll.With(new StochasticDualCoordinateAscentBinaryClassifier()));
var model = pipeline.Train <IrisData, IrisPrediction>();
var testData = new TextLoader(dataPath).CreateFrom <IrisData>(useHeader: false);
var evaluator = new ClassificationEvaluator();
ClassificationMetrics metrics = evaluator.Evaluate(model, testData);
CheckMetrics(metrics);
var trainTest = new TrainTestEvaluator()
{
Kind = MacroUtilsTrainerKinds.SignatureMultiClassClassifierTrainer
}.TrainTestEvaluate <IrisData, IrisPrediction>(pipeline, testData);
CheckMetrics(trainTest.ClassificationMetrics);
}
/// <summary>
/// Evaluates the trained model for quality assurance against a
/// second independent test data set.
///
/// Loads the test dataset.
/// Creates the binary evaluator.
/// Evaluates the model and create metrics.
///
/// Displays the metrics.
/// </summary>
/// <param name="model"></param>
internal static void Evaluate(
PredictionModel <ClassificationData, ClassPrediction> model,
string testDatafile)
{
// loads the new test dataset with the same schema.
// You can evaluate the model using this dataset as a quality check.
var testData = new TextLoader(testDatafile).CreateFrom <ClassificationData>();
// Computes the quality metrics for the PredictionModel using the specified dataset.
var evaluator = new ClassificationEvaluator();
// The BinaryClassificationMetrics contains the overall metrics computed by binary
// classification evaluators. To display these to determine the quality of the model,
// you need to get the metrics first.
ClassificationMetrics metrics = evaluator.Evaluate(model, testData);
// Displaying the metrics for model validation
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
Console.WriteLine(" Accuracy Macro: {0:P2}", metrics.AccuracyMacro);
Console.WriteLine(" Accuracy Micro: {0:P2}", metrics.AccuracyMicro);
Console.WriteLine(" Top KAccuracy: {0:P2}", metrics.TopKAccuracy);
Console.WriteLine(" LogLoss: {0:P2}", metrics.LogLoss);
Console.WriteLine("");
Console.WriteLine(" PerClassLogLoss:");
for (int i = 0; i < metrics.PerClassLogLoss.Length; i++)
{
Console.WriteLine(" Class: {0} - {1:P2}", i, metrics.PerClassLogLoss[i]);
}
}
protected void ShowMetrics(string testLocation, PredictionModel<ImageNetData, ImageNetPrediction> model)
{
var evaluator = new ClassificationEvaluator();
var testDataSource = new TextLoader(testLocation).CreateFrom<ImageData.ImageNetData>();
ClassificationMetrics metrics = evaluator.Evaluate(model, testDataSource);
PrintMetrics(metrics);
}
/// Evaluates the trained model for quality assurance against a second independent test data set.
internal static void Evaluate(
PredictionModel <ClassificationData, ClassPrediction> model,
string testDatafile)
{
//Loading test data
var testData = new TextLoader(testDatafile).CreateFrom <ClassificationData>();
//Deciding which evaluator will be used
var evaluator = new ClassificationEvaluator();
//Start evaluation
ClassificationMetrics metrics = evaluator.Evaluate(model, testData);
Console.WriteLine();
Console.WriteLine("------------------------------------------");
Console.WriteLine(" Accuracy Macro: {0:P2}", metrics.AccuracyMacro);
Console.WriteLine(" Accuracy Micro: {0:P2}", metrics.AccuracyMicro);
Console.WriteLine(" LogLoss: {0:P2}", metrics.LogLoss);
Console.WriteLine();
Console.WriteLine(" PerClassLogLoss:");
for (int i = 0; i < metrics.PerClassLogLoss.Length; i++)
{
Console.WriteLine(" Class: {0} - {1:P2}", i, metrics.PerClassLogLoss[i]);
}
}
public async Task EvaluateAsync(string modelPath, string csvPath)
{
var model = await PredictionModel.ReadAsync <Product, ProductCategoryPrediction>(modelPath).ConfigureAwait(false);
// To evaluate how good the model predicts values, the model is ran against new set
// of data (test data) that was not involved in training.
var testData = new TextLoader(csvPath).CreateFrom <Product>(useHeader: false, allowQuotedStrings: false, supportSparse: false);
// ClassificationEvaluator performs evaluation for Multiclass Classification type of ML problems.
var evaluator = new ClassificationEvaluator {
OutputTopKAcc = 3
};
Console.WriteLine("=============== Evaluating model ===============");
var metrics = evaluator.Evaluate(model, testData);
Console.WriteLine("Metrics:");
Console.WriteLine($" AccuracyMacro = {metrics.AccuracyMacro:0.####}, a value between 0 and 1, the closer to 1, the better");
Console.WriteLine($" AccuracyMicro = {metrics.AccuracyMicro:0.####}, a value between 0 and 1, the closer to 1, the better");
Console.WriteLine($" LogLoss = {metrics.LogLoss:0.####}, the closer to 0, the better");
Console.WriteLine($" LogLoss for class 1 = {metrics.PerClassLogLoss[0]:0.####}, the closer to 0, the better");
Console.WriteLine($" LogLoss for class 2 = {metrics.PerClassLogLoss[1]:0.####}, the closer to 0, the better");
Console.WriteLine($" LogLoss for class 3 = {metrics.PerClassLogLoss[2]:0.####}, the closer to 0, the better");
Console.WriteLine("=============== End evaluating ===============");
Console.WriteLine();
}
public ClassificationMetrics Evaluate(string testDataPath)
{
var testData = new TextLoader(testDataPath).CreateFrom <MulticlassClassificationData>();
var evaluator = new ClassificationEvaluator();
return(evaluator.Evaluate(Model, testData));
}
private void Evaluate()
{
var loader = new TextLoader("./mnist_test.csv").CreateFrom <MinstDigit>(separator: ',', useHeader: true);
var evaluator = new ClassificationEvaluator();
var evaluation = evaluator.Evaluate(_model, loader);
Console.WriteLine(JsonConvert.SerializeObject(evaluation, Formatting.Indented));
}
static void Main(string[] args)
{
var trainData = GeneratePData(2000);
Print(trainData.Take(20));
var testData = GeneratePData(50, test: true);
// ML görevi için obje oluşturur
var learningPipe = new LearningPipeline();
var trainCollection = CollectionDataSource.Create(trainData);
learningPipe.Add(trainCollection);
// Verilerin kolon isimleri olan labelları numeric indexe çevirir.
learningPipe.Add(new Dictionarizer("Label"));
learningPipe.Add(
new ColumnConcatenator("Features", "UnitA", "UnitS", "Volume"));
// Algoritma sınıflandırması
learningPipe.Add(new StochasticDualCoordinateAscentClassifier());
// Tahmin edilen kolon değerini çevir
learningPipe.Add(new PredictedLabelColumnOriginalValueConverter()
{
PredictedLabelColumn = "PredictedLabel"
});
// Modeli eğitme
var model = learningPipe.Train <ProcessData, ProcessPrediction>();
// Model değerlendirilmesi ve kesin kontrol
var evaluator = new ClassificationEvaluator();
var metrics = evaluator.Evaluate(model, trainCollection);
Console.WriteLine("AccuracyMicro: " + metrics.AccuracyMicro);
Console.WriteLine("LogLoss: " + metrics.LogLoss);
// Test datayı tahmin et
var predicted = model.Predict(testData);
// Testdata ve tahmin edilen labelı string içinde topla
var results = testData.Zip(predicted, (t, p) => new ProcessData
{
UnitA = t.UnitA,
UnitS = t.UnitS,
Volume = t.Volume,
Label = p.PredictedLabels
}).ToList();
// Sonucu yazdır
Print(results);
Console.ReadLine();
}
protected void EvaluateModel(string testLocation, PredictionModel <ImageNetData, ImageNetPrediction> model)
{
ConsoleWriteHeader("Metrics for Image Classification");
var evaluator = new ClassificationEvaluator();
var testDataSource = new TextLoader(testLocation).CreateFrom <ImageData.ImageNetData>();
ClassificationMetrics metrics = evaluator.Evaluate(model, testDataSource);
// Log loss is nearly zero. The lower the better, so in this case can't be better
// This is an "ideal" situation, usually you get higher values
Console.WriteLine($"Log Loss: {metrics.LogLoss}");
}
public Task <ClassificationMetrics> Evaluate(string testDataPath)
{
return(Task.Run(() =>
{
var testData = new TextLoader(testDataPath).CreateFrom <MulticlassClassificationData>();
// Computes the quality metrics for the PredictionModel using the specified dataset.
var evaluator = new ClassificationEvaluator();
var metrics = evaluator.Evaluate(Model, testData);
return metrics;
}));
}
public static void Evaluate(PredictionModel <IntentData, IntentPrediction> model)
{
var testData = new TextLoader <IntentData>(testPath, useHeader: false, separator: "tab");
var evaluator = new ClassificationEvaluator();
var metrics = evaluator.Evaluate(model, testData);
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
Console.WriteLine($"Accuracy Micro: {metrics.AccuracyMicro * 100}%");
Console.WriteLine($"Accuracy Macro: {metrics.AccuracyMacro * 100}%");
}
public static void Evaluate(PredictionModel <PokerHandData, PokerHandPrediction> model, IEnumerable <PokerHandData> data)
{
var evaluator = new ClassificationEvaluator();
var collection = CollectionDataSource.Create(data);
var metrics = evaluator.Evaluate(model, collection);
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
Console.WriteLine($"LogLossReduction: {metrics.LogLossReduction }");
Console.WriteLine($"LogLoss: {metrics.LogLoss }");
}
public static void Evaluate(PredictionModel <NewsData, NewsPrediction> model)
{
var testData = new TextLoader <NewsData>(_trainingSet, useHeader: false, separator: "tab");
var evaluator = new ClassificationEvaluator();
var metrics = evaluator.Evaluate(model, testData);
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
Console.WriteLine($"AccuracyMacro: {metrics.AccuracyMacro:P2}");
Console.WriteLine($"AccuracyMicro: {metrics.AccuracyMicro:P2}");
Console.WriteLine($"LogLoss: {metrics.LogLoss:P2}");
}
public async Task <ClassificationMetrics> EvaluateAsync(string testPath)
{
var testData = new TextLoader(testPath).CreateFrom <BankStatementLineItem>(separator: ',', useHeader: true);
var evaluator = new ClassificationEvaluator();
if (_model == null)
{
_model = await PredictionModel.ReadAsync <BankStatementLineItem, PredictedLabel>(PredictionModelWrapper
.Model1Path);
}
var metrics = evaluator.Evaluate(_model, testData);
return(metrics);
}
static void EvaluateNews(PredictionModel <NewsData, NewsPrediction> model)
{
const string trainingSet = @"news-train.txt";
var testData = new TextLoader(trainingSet).CreateFrom <NewsData>();
var evaluator = new ClassificationEvaluator();
var metrics = evaluator.Evaluate(model, testData);
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
Console.WriteLine($"AccuracyMacro: {metrics.AccuracyMacro:P2}");
Console.WriteLine($"AccuracyMicro: {metrics.AccuracyMicro:P2}");
Console.WriteLine($"LogLoss: {metrics.LogLoss:P2}");
}
private static void Evaluate(PredictionModel <CarEval, CarEvalPredictor> model)
{
// To evaluate how good the model predicts values, it is run against new set
// of data (test data) that was not involved in training.
var testData = new TextLoader(TestDataPath).CreateFrom <CarEval>(separator: ',');
// RegressionEvaluator calculates the differences (in various metrics) between predicted and actual
// values in the test dataset.
var evaluator = new ClassificationEvaluator();
Console.WriteLine("=============== Evaluating model ===============");
var metrics = evaluator.Evaluate(model, testData);
Console.WriteLine($"Micro-Accuray is {metrics.AccuracyMicro}");
}
private static void Evaluate(PredictionModel <Master, MasterPrediction> model)
{
var testData = new TextLoader(TestDataPath).CreateFrom <Master>();
var evaluator = new ClassificationEvaluator {
OutputTopKAcc = 3
};
var metrics = evaluator.Evaluate(model, testData);
for (var i = 0; i < metrics.ConfusionMatrix.Order; i++)
{
for (var j = 0; j < metrics.ConfusionMatrix.ClassNames.Count; j++)
{
Console.Write("\t" + metrics.ConfusionMatrix[i, j]);
}
}
}
private static async Task TrainAsync()
{
Console.WriteLine("============ 準備訓練資料 ==============");
var pipeline = new LearningPipeline();
pipeline.Add(new TextLoader(DataPath).CreateFrom <GitHubIssue>(useHeader: true));
// 把兩個用來訓練的欄位變成數字的vector
pipeline.Add(new TextFeaturizer("Title", "Title"));
pipeline.Add(new TextFeaturizer("Description", "Description"));
// 把Area轉換成為Dictionary數字
pipeline.Add(new Dictionarizer(("Area", "Label")));
// Title 和 Description合并變成訓練的欄位
pipeline.Add(new ColumnConcatenator("Features", "Title", "Description"));
// 使用StochasticDualCoordinateAscent演算法
pipeline.Add(new StochasticDualCoordinateAscentClassifier());
// 把判斷出來的數字轉回文字版本
pipeline.Add(new PredictedLabelColumnOriginalValueConverter()
{
PredictedLabelColumn = "PredictedLabel"
});
Console.WriteLine("=============== 訓練模型 ===============");
var model = pipeline.Train <GitHubIssue, GithubIssueLabelPrediction>();
var testData = new TextLoader(TestPath).CreateFrom <GitHubIssue>(useHeader: true);
var evaluator = new ClassificationEvaluator();
ClassificationMetrics metrics = evaluator.Evaluate(model, testData);
Console.WriteLine();
Console.WriteLine("Micro-Accuracy: {0}", metrics.AccuracyMicro);
await model.WriteAsync(ModelPath);
Console.WriteLine("=============== 訓練完成 ===============");
Console.WriteLine("Model路徑: {0}", ModelPath);
}
public static void Ex()
{
LearningPipeline pipeline = new LearningPipeline();
pipeline.Add(new TextLoader(@"C:\Users\Дарья\Desktop\AdverbNoun.csv").CreateFrom <Model>(useHeader: true, separator: ','));
pipeline.Add(new Dictionarizer(("Category", "Label")));
pipeline.Add(new TextFeaturizer("Features", "Word"));
pipeline.Add(new StochasticDualCoordinateAscentClassifier());
//Vec < R4, 472 >
pipeline.Add(new PredictedLabelColumnOriginalValueConverter()
{
PredictedLabelColumn = "PredictedLabel"
});
var model = pipeline.Train <Model, CategoryPrediction>();
model.WriteAsync(@"C:\Users\Дарья\Desktop\NewModel.zip");
model = PredictionModel.ReadAsync <Model, CategoryPrediction>(@"C:\Users\Дарья\Desktop\NewModel.zip").Result;
var testData = new TextLoader(@"C:\Users\Дарья\Desktop\TestAdverbNoun.csv").CreateFrom <Model>(useHeader: true, separator: ',');
var evaluator = new ClassificationEvaluator();
var metrics = evaluator.Evaluate(model, testData);
Console.WriteLine($"AccuracyMicro: {metrics.AccuracyMicro}\nAccuracyMacro: {metrics.AccuracyMacro}");
var data = ExcelDataProvider.GetData(@"C:\Users\Дарья\Desktop\TestAdverbNoun.xlsx", 0);
List <Model> models = new List <Model>();
data.ForEach(x => models.Add(new Model()
{
Category = x.Category,
Word = x.Word
}));
foreach (var sample in models)
{
var prediction = model.Predict(sample);
Console.WriteLine($"Word: {sample.Word}");
Console.WriteLine($"\nActual Category: {sample.Category} \nPredicted Category: {prediction.Category}\nPredicted Category scores: [{String.Join(",", prediction.Score)}]\n\n");
}
Console.ReadLine();
}
static void EvaluateOpen311(PredictionModel <Open311Data, Open311DataPrediction> model, string testDataPath)
{
var testData = CollectionDataSource.Create(OpenFile(testDataPath, 3, 0, 1, 2));
var evaluator = new ClassificationEvaluator();
var metrics = evaluator.Evaluate(model, testData);
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
Console.WriteLine($"Accuracy Macro: {metrics.AccuracyMacro:P2}");
Console.WriteLine($"Accuracy Micro: {metrics.AccuracyMicro:P2}");
Console.WriteLine($"Top KAccuracy: {metrics.TopKAccuracy:P2}");
Console.WriteLine($"LogLoss: {metrics.LogLoss:P2}");
for (var classIndex = 0; classIndex < metrics.PerClassLogLoss.Length; classIndex++)
{
Console.WriteLine($"Class: {classIndex} - {metrics.PerClassLogLoss[classIndex]:P2}");
}
}
public static void Evaluate(PredictionModel <MNISTData, MNISTPrediction> model)
{
//Creation d'un analyseur de donnée pour le fichier test
var testData = new TextLoader(_testDataPath)
{
Arguments = new TextLoaderArguments
{
Separator = new[] { ',' },
HasHeader = true,
Column = new[]
{
new TextLoaderColumn()
{
Name = "Label",
Source = new [] { new TextLoaderRange(0) },
Type = DataKind.Num
},
new TextLoaderColumn()
{
Name = "Features",
Source = new [] { new TextLoaderRange(1, 784) },
Type = DataKind.Num
}
}
}
};
var evaluator = new ClassificationEvaluator();
//On évalue ici notre modèle selon les données test, les résultats statistiques sont contenus dans metrics
ClassificationMetrics metrics = evaluator.Evaluate(model, testData);
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
//deux exemples de statistques utiles : la précision macroscopique et la matrice de confusion
Console.WriteLine("Macro Acc : {0}", metrics.AccuracyMacro);
Console.WriteLine("----------------------------------");
Console.WriteLine("Confusion Matrix");
PrintMatrix(metrics.ConfusionMatrix);
}
private static void EvaluateLanguage(PredictionModel <LanguageModel, LanguagePrediction> model)
{
// To evaluate how good the model predicts values, the model is ran against new set
// of data (test data) that was not involved in training.
var testData = new TextLoader(DataPath.TestDataPath).CreateFrom <LanguageModel>();
// ClassificationEvaluator .
var evaluator = new ClassificationEvaluator();
Console.WriteLine("=============== Evaluating model ===============");
var metrics = evaluator.Evaluate(model, testData);
Console.WriteLine("Metrics:");
Console.WriteLine($" AccuracyMacro = {metrics:0.####}, a value between 0 and 1, the closer to 1, the better");
Console.WriteLine($" AccuracyMicro = {metrics.AccuracyMicro:0.####}, a value between 0 and 1, the closer to 1, the better");
Console.WriteLine("=============== End evaluating ===============");
Console.WriteLine();
}
public static async Task Main(List <User> patient)
{
//call Train method
PredictionModel <FoodCustomClass, ClassPrediction> model = Train();
await model.WriteAsync(_modelPath);
//Call FoodClassificationTest class for prediction
var prediction = model.Predict(FoodClassificationTest.FoodClassification());
//Store each food in a list
List <FoodCustomClass> foods = FoodClassificationTest.FoodClassification();
var testData = new TextLoader(_testdataPath).CreateFrom <FoodCustomClass>(separator: '*');
var evaluator = new ClassificationEvaluator();
ClassificationMetrics metrics = evaluator.Evaluate(model, testData);
// Displaying the metrics for model validation
System.Diagnostics.Debug.WriteLine("PredictionModel quality metrics evaluation");
System.Diagnostics.Debug.WriteLine("------------------------------------------");
System.Diagnostics.Debug.WriteLine($"* MicroAccuracy: {metrics.AccuracyMicro:0.###}");
System.Diagnostics.Debug.WriteLine($"* MacroAccuracy: {metrics.AccuracyMacro:0.###}");
System.Diagnostics.Debug.WriteLine($"* LogLoss: {metrics.LogLoss:#.###}");
System.Diagnostics.Debug.WriteLine($"* LogLossReduction: {metrics.LogLossReduction:#.###}");
//Combine List foods with their respective results after classification
/*
* foodsAndClassification contain each food from FoodDB database classified in terms
* of the quantity of each macronutrient present
*/
var foodsAndClassification = foods.Zip(prediction, (n, w) => new { Food = n, Category = w });
//call ID3Tree.cs and send foodsAndClassification as parameter to ID3Tree constructor
var function = new ID3Tree().ID3(foodsAndClassification, patient);
//foreach (var nw in foodsAndClassification)
//{
// System.Diagnostics.Debug.WriteLine($"{nw.Food.Food}: {nw.Category.Predicted}" +"\n" + $"{float.Parse(nw.Category.Score.First().ToString(), CultureInfo.InvariantCulture.NumberFormat)}");
//}
}
public void SetupPredictBenchmarks()
{
_trainedModel = Train(_dataPath);
_consumer.Consume(_trainedModel.Predict(_example));
var testData = new Legacy.Data.TextLoader(_dataPath).CreateFrom <IrisData>(useHeader: true);
var evaluator = new ClassificationEvaluator();
_metrics = evaluator.Evaluate(_trainedModel, testData);
_batches = new IrisData[_batchSizes.Length][];
for (int i = 0; i < _batches.Length; i++)
{
var batch = new IrisData[_batchSizes[i]];
_batches[i] = batch;
for (int bi = 0; bi < batch.Length; bi++)
{
batch[bi] = _example;
}
}
}
//test the model
public static void Evaluate(PredictionModel <SentimentData, SentimentPrediction> model)
{
//load the test data
var testData = new TextLoader(_testDataPath).CreateFrom <SentimentData>(useHeader: true);
//computes the quality metrics for the PredictionModel
var evaluator = new ClassificationEvaluator();
//to get metrices computed by binary classification evaluator
ClassificationMetrics metrics = evaluator.Evaluate(model, testData);
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
Console.WriteLine($"LogLoss: { metrics.LogLoss:P2}");
//Console.WriteLine($"ConfusionMatrix: { metrics.ConfusionMatrix:P2}");
Console.WriteLine($"AccuracyMicro: { metrics.AccuracyMicro:P2}");
Console.WriteLine($"Accuracy: {metrics.AccuracyMacro:P2}");
//Console.WriteLine($"Auc: {metrics.Auc:P2}");
//Console.WriteLine($"F1Score: {metrics.F1Score:P2}");
}
public void Setup()
{
s_dataPath = Program.GetDataPath("iris.txt");
s_trainedModel = TrainCore();
IrisPrediction prediction = s_trainedModel.Predict(s_example);
var testData = new TextLoader(s_dataPath).CreateFrom <IrisData>(useHeader: true);
var evaluator = new ClassificationEvaluator();
s_metrics = evaluator.Evaluate(s_trainedModel, testData);
s_batches = new IrisData[s_batchSizes.Length][];
for (int i = 0; i < s_batches.Length; i++)
{
var batch = new IrisData[s_batchSizes[i]];
s_batches[i] = batch;
for (int bi = 0; bi < batch.Length; bi++)
{
batch[bi] = s_example;
}
}
}
private static void Evaluate(PredictionModel <IrisData, IrisPrediction> model)
{
// To evaluate how good the model predicts values, the model is ran against new set
// of data (test data) that was not involved in training.
var testData = new TextLoader(TestDataPath).CreateFrom <IrisData>();
// ClassificationEvaluator performs evaluation for Multiclass Classification type of ML problems.
var evaluator = new ClassificationEvaluator {
OutputTopKAcc = 3
};
Console.WriteLine("=============== Evaluating model ===============");
var metrics = evaluator.Evaluate(model, testData);
Console.WriteLine("Metrics:");
Console.WriteLine($" AccuracyMacro = {metrics.AccuracyMacro:0.####}, a value between 0 and 1, the closer to 1, the better");
Console.WriteLine($" AccuracyMicro = {metrics.AccuracyMicro:0.####}, a value between 0 and 1, the closer to 1, the better");
Console.WriteLine($" LogLoss = {metrics.LogLoss:0.####}, the closer to 0, the better");
Console.WriteLine($" LogLoss for class 1 = {metrics.PerClassLogLoss[0]:0.####}, the closer to 0, the better");
Console.WriteLine($" LogLoss for class 2 = {metrics.PerClassLogLoss[1]:0.####}, the closer to 0, the better");
Console.WriteLine($" LogLoss for class 3 = {metrics.PerClassLogLoss[2]:0.####}, the closer to 0, the better");
Console.WriteLine();
Console.WriteLine($" ConfusionMatrix:");
// Print confusion matrix
for (var i = 0; i < metrics.ConfusionMatrix.Order; i++)
{
for (var j = 0; j < metrics.ConfusionMatrix.ClassNames.Count; j++)
{
Console.Write("\t" + metrics.ConfusionMatrix[i, j]);
}
Console.WriteLine();
}
Console.WriteLine("=============== End evaluating ===============");
Console.WriteLine();
}
void Metacomponents()
{
var dataPath = GetDataPath(IrisDataPath);
var pipeline = new LearningPipeline(seed: 1, conc: 1);
pipeline.Add(new TextLoader(dataPath).CreateFrom <IrisData>(useHeader: false));
pipeline.Add(new Dictionarizer(new[] { "Label" }));
pipeline.Add(new ColumnConcatenator(outputColumn: "Features",
"SepalLength", "SepalWidth", "PetalLength", "PetalWidth"));
// This will throw exception during training time if you specify any other than binary classifier.
pipeline.Add(OneVersusAll.With(new StochasticDualCoordinateAscentBinaryClassifier()));
var model = pipeline.Train <IrisData, IrisPrediction>();
var testData = new TextLoader(dataPath).CreateFrom <IrisData>(useHeader: false);
var evaluator = new ClassificationEvaluator();
ClassificationMetrics metrics = evaluator.Evaluate(model, testData);
var prediction = model.Predict(new IrisData {
PetalLength = 1, PetalWidth = 2, SepalLength = 1.4f, SepalWidth = 1.6f
});
}
public static void Training()
{
// STEP 2: 建立執行預測運算的 Pipeline
var pipeline = new LearningPipeline
{
// A. 設定訓練預測的數據集的來源
// 若使用 Visual Studio 開發，請確認該檔案有設定"複製到輸出目錄"屬性成"一律複製"
// 資料集來源：https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data
// 從數據集中讀取資料，每一行為一筆資料，使用 ',' 當作分隔字元
new TextLoader(DataPath).CreateFrom <MachineStatusData>(separator: ',', useHeader: false),
// B. 轉換資料，將資料型別為文字的標籤屬性，建立字典並轉透過數字來表示，因為在訓練模型時，只能包含數字型別的值
new Dictionarizer("Label"),
// C. 設定要作為學習的特徵放入向量中
new ColumnConcatenator("Features", "MachineTemperature", "MachinePressure"),
// D. 設定學習器，將要作為學習方法的演算法加入 pipeline 中
// 這是一個分類的場景，使用隨機雙座標上升分類器 (Stochastic Dual Coordinate Ascent Classifier) 作為分類方案，預測該鳶尾花是哪種類別
new StochasticDualCoordinateAscentClassifier(),
// E. 轉換資料，將標籤轉換回原始文字（前述步驟曾將他轉換成數字）
new PredictedLabelColumnOriginalValueConverter()
{
PredictedLabelColumn = "PredictedLabel"
}
};
// STEP 3: 根據所提供的數據集來訓練模型
var model = pipeline.Train <MachineStatusData, MachineStatusPrediction>();
// STEP 4: 輸出評估結果
var testData = new TextLoader(TestPath).CreateFrom <MachineStatusData>(separator: ',', useHeader: false);
var evaluator = new ClassificationEvaluator();
var metrics = evaluator.Evaluate(model, testData);
Console.WriteLine($"微精度: {metrics.AccuracyMicro}");
// STEP 5: 儲存訓練後的預測模型
model.WriteAsync(ModelPath).ConfigureAwait(false);
}