private async void TrainProxyWrapper_MulticlassMetricsUpdated(Microsoft.ML.Data.MulticlassClassificationMetrics metrics)
{
await _trainLogHub.Clients.All.SendAsync("Log", string.Format("Log-loss: {0}", metrics.LogLoss));
await _trainLogHub.Clients.All.SendAsync("Log", string.Format("Log-loss measures the performance of a classifier with respect to how much the predicted probabilities diverge from the true class label. Lower log-loss indicates a better model. A perfect model, which predicts a probability of 1 for the true class, will have a log-loss of 0."));
await _trainLogHub.Clients.All.SendAsync("Log", string.Format("Log-loss Reduction: {0}", metrics.LogLossReduction));
await _trainLogHub.Clients.All.SendAsync("Log", string.Format("It gives a measure of how much a model improves on a model that gives random predictions. Log-loss reduction closer to 1 indicates a better model."));
await _trainLogHub.Clients.All.SendAsync("Log", string.Format("Macro Accuracy: {0}", metrics.MacroAccuracy));
await _trainLogHub.Clients.All.SendAsync("Log", string.Format("The accuracy for each class is computed and the macro-accuracy is the average of these accuracies. The macro-average metric gives the same weight to each class, no matter how many instances from that class the dataset contains."));
await _trainLogHub.Clients.All.SendAsync("Log", string.Format("Micro Accuracy: {0}", metrics.MicroAccuracy));
await _trainLogHub.Clients.All.SendAsync("Log", string.Format("The micro-average is the fraction of instances predicted correctly across all classes. Micro-average can be a more useful metric than macro-average if class imbalance is suspected."));
await _trainLogHub.Clients.All.SendAsync("Log", string.Format("This is the relative number of examples where the true label one of the top K predicted labels by the predictor."));
await _trainLogHub.Clients.All.SendAsync("Log", string.Format("Top K Prediction Count: {0}", metrics.TopKPredictionCount));
await _trainLogHub.Clients.All.SendAsync("Log", string.Format("Top K Accuracy: {0}", metrics.TopKAccuracy));
await _trainLogHub.Clients.All.SendAsync("Log", string.Format("If positive, this indicates the K in Top K Accuracy and Top K Accuracy for all K."));
if (metrics.TopKAccuracyForAllK?.Count > 0)
{
await _trainLogHub.Clients.All.SendAsync("Log", string.Format("Top K Accuracy for all K: ({0})", string.Join(", ", metrics.TopKAccuracyForAllK)));
}
if (metrics.PerClassLogLoss?.Count > 0)
{
await _trainLogHub.Clients.All.SendAsync("Log", string.Format("Per Class Log-loss: ({0})", string.Join(", ", metrics.PerClassLogLoss)));
}
}
private void PrintMultiClassClassificationMetrics(string name, Microsoft.ML.Data.MulticlassClassificationMetrics metrics)
{
Console.WriteLine($"************************************************************");
Console.WriteLine($"* Metrics for {name} multi-class classification model ");
Console.WriteLine($"*-----------------------------------------------------------");
Console.WriteLine($" AccuracyMacro = {metrics.MacroAccuracy:0.####}, a value between 0 and 1, the closer to 1, the better");
Console.WriteLine($" AccuracyMicro = {metrics.MicroAccuracy:0.####}, a value between 0 and 1, the closer to 1, the better");
Console.WriteLine($" LogLoss = {metrics.LogLoss:0.####}, the closer to 0, the better");
int i = 0;
foreach (var classLogLoss in metrics.PerClassLogLoss)
{
i++;
Console.WriteLine($" LogLoss for class {i} = {classLogLoss:0.####}, the closer to 0, the better");
}
Console.WriteLine($"************************************************************");
}
public async Task <Microsoft.ML.Data.MulticlassClassificationMetrics> Start(Action <TrainerOptions> action)
{
_ = action ?? throw new ArgumentNullException();
var options = new TrainerOptions();
action.Invoke(options);
Microsoft.ML.Data.MulticlassClassificationMetrics metrics = null;
try
{
HasStarted = true;
DataPreprocessor.OnPreprocessorChanged = OnPreprocessorChanged ?? DataPreprocessor.OnPreprocessorChanged;
DataPreprocessor.OnPreprocessorStop = OnPreprocessorStop ?? DataPreprocessor.OnPreprocessorStop;
Trainer.OnStageChanged = (tracker) =>
{
if (tracker.TrainingStage == TrainingStage.Evaluating)
{
OnPhaseChangedEvent?.Invoke(TrainerPhases.Testing);
}
};
OnPhaseChangedEvent?.Invoke(TrainerPhases.DataPreprocessing);
await DataPreprocessor.Start(o =>
{
o.DataFileName = options.DataFileName;
o.PDataFileName = options.ProcessedDataFileName;
});
OnPhaseChangedEvent?.Invoke(TrainerPhases.TestDataPreprocessing);
await DataPreprocessor.Start(o =>
{
o.DataFileName = options.TestDataFileName;
o.PDataFileName = "Processed" + options.TestDataFileName;
});
OnPhaseChangedEvent?.Invoke(TrainerPhases.Training);
metrics = await Trainer.Start(o =>
{
o.ProcessedDataFileName = options.ProcessedDataFileName;
o.TestPDataFileName = "Processed" + options.TestDataFileName;
o.MLModelFileName = options.MLModelFileName;
});
await MetricsDataStorage.CreateMetricsFileAsync(options.MLModelFileName, metrics);
OnPhaseChangedEvent?.Invoke(TrainerPhases.Completed);
}
catch (UnauthorizedAccessException ex)
{
throw new UnauthorizedAccessException("User may not have permission to access the required file");
}
catch (System.IO.FileNotFoundException ex)
{
throw new System.IO.FileNotFoundException("File does not found");
}
catch (Exception ex)
{
throw new Exception("Unexpected error occured");
}
finally
{
HasStarted = false;
}
return(metrics);
}
