private async void TrainProxyWrapper_MulticlassMetricsUpdated(Microsoft.ML.Data.MulticlassClassificationMetrics metrics) { await _trainLogHub.Clients.All.SendAsync("Log", string.Format("Log-loss: {0}", metrics.LogLoss)); await _trainLogHub.Clients.All.SendAsync("Log", string.Format("Log-loss measures the performance of a classifier with respect to how much the predicted probabilities diverge from the true class label. Lower log-loss indicates a better model. A perfect model, which predicts a probability of 1 for the true class, will have a log-loss of 0.")); await _trainLogHub.Clients.All.SendAsync("Log", string.Format("Log-loss Reduction: {0}", metrics.LogLossReduction)); await _trainLogHub.Clients.All.SendAsync("Log", string.Format("It gives a measure of how much a model improves on a model that gives random predictions. Log-loss reduction closer to 1 indicates a better model.")); await _trainLogHub.Clients.All.SendAsync("Log", string.Format("Macro Accuracy: {0}", metrics.MacroAccuracy)); await _trainLogHub.Clients.All.SendAsync("Log", string.Format("The accuracy for each class is computed and the macro-accuracy is the average of these accuracies. The macro-average metric gives the same weight to each class, no matter how many instances from that class the dataset contains.")); await _trainLogHub.Clients.All.SendAsync("Log", string.Format("Micro Accuracy: {0}", metrics.MicroAccuracy)); await _trainLogHub.Clients.All.SendAsync("Log", string.Format("The micro-average is the fraction of instances predicted correctly across all classes. Micro-average can be a more useful metric than macro-average if class imbalance is suspected.")); await _trainLogHub.Clients.All.SendAsync("Log", string.Format("This is the relative number of examples where the true label one of the top K predicted labels by the predictor.")); await _trainLogHub.Clients.All.SendAsync("Log", string.Format("Top K Prediction Count: {0}", metrics.TopKPredictionCount)); await _trainLogHub.Clients.All.SendAsync("Log", string.Format("Top K Accuracy: {0}", metrics.TopKAccuracy)); await _trainLogHub.Clients.All.SendAsync("Log", string.Format("If positive, this indicates the K in Top K Accuracy and Top K Accuracy for all K.")); if (metrics.TopKAccuracyForAllK?.Count > 0) { await _trainLogHub.Clients.All.SendAsync("Log", string.Format("Top K Accuracy for all K: ({0})", string.Join(", ", metrics.TopKAccuracyForAllK))); } if (metrics.PerClassLogLoss?.Count > 0) { await _trainLogHub.Clients.All.SendAsync("Log", string.Format("Per Class Log-loss: ({0})", string.Join(", ", metrics.PerClassLogLoss))); } }
private void PrintMultiClassClassificationMetrics(string name, Microsoft.ML.Data.MulticlassClassificationMetrics metrics) { Console.WriteLine($"************************************************************"); Console.WriteLine($"* Metrics for {name} multi-class classification model "); Console.WriteLine($"*-----------------------------------------------------------"); Console.WriteLine($" AccuracyMacro = {metrics.MacroAccuracy:0.####}, a value between 0 and 1, the closer to 1, the better"); Console.WriteLine($" AccuracyMicro = {metrics.MicroAccuracy:0.####}, a value between 0 and 1, the closer to 1, the better"); Console.WriteLine($" LogLoss = {metrics.LogLoss:0.####}, the closer to 0, the better"); int i = 0; foreach (var classLogLoss in metrics.PerClassLogLoss) { i++; Console.WriteLine($" LogLoss for class {i} = {classLogLoss:0.####}, the closer to 0, the better"); } Console.WriteLine($"************************************************************"); }
public async Task <Microsoft.ML.Data.MulticlassClassificationMetrics> Start(Action <TrainerOptions> action) { _ = action ?? throw new ArgumentNullException(); var options = new TrainerOptions(); action.Invoke(options); Microsoft.ML.Data.MulticlassClassificationMetrics metrics = null; try { HasStarted = true; DataPreprocessor.OnPreprocessorChanged = OnPreprocessorChanged ?? DataPreprocessor.OnPreprocessorChanged; DataPreprocessor.OnPreprocessorStop = OnPreprocessorStop ?? DataPreprocessor.OnPreprocessorStop; Trainer.OnStageChanged = (tracker) => { if (tracker.TrainingStage == TrainingStage.Evaluating) { OnPhaseChangedEvent?.Invoke(TrainerPhases.Testing); } }; OnPhaseChangedEvent?.Invoke(TrainerPhases.DataPreprocessing); await DataPreprocessor.Start(o => { o.DataFileName = options.DataFileName; o.PDataFileName = options.ProcessedDataFileName; }); OnPhaseChangedEvent?.Invoke(TrainerPhases.TestDataPreprocessing); await DataPreprocessor.Start(o => { o.DataFileName = options.TestDataFileName; o.PDataFileName = "Processed" + options.TestDataFileName; }); OnPhaseChangedEvent?.Invoke(TrainerPhases.Training); metrics = await Trainer.Start(o => { o.ProcessedDataFileName = options.ProcessedDataFileName; o.TestPDataFileName = "Processed" + options.TestDataFileName; o.MLModelFileName = options.MLModelFileName; }); await MetricsDataStorage.CreateMetricsFileAsync(options.MLModelFileName, metrics); OnPhaseChangedEvent?.Invoke(TrainerPhases.Completed); } catch (UnauthorizedAccessException ex) { throw new UnauthorizedAccessException("User may not have permission to access the required file"); } catch (System.IO.FileNotFoundException ex) { throw new System.IO.FileNotFoundException("File does not found"); } catch (Exception ex) { throw new Exception("Unexpected error occured"); } finally { HasStarted = false; } return(metrics); }