public virtual void CalculateMetrics(FeatureSubsetModel <IPredictorProducing <TOutput> > model,
ISubsetSelector subsetSelector, Subset subset, Batch batch, bool needMetrics)
{
if (!needMetrics || model == null || model.Metrics != null)
{
return;
}
using (var ch = Host.Start("Calculate metrics"))
{
RoleMappedData testData = subsetSelector.GetTestData(subset, batch);
// Because the training and test datasets are drawn from the same base dataset, the test data role mappings
// are the same as for the train data.
IDataScorerTransform scorePipe = ScoreUtils.GetScorer(model.Predictor, testData, Host, testData.Schema);
// REVIEW: Should we somehow allow the user to customize the evaluator?
// By what mechanism should we allow that?
var evalComp = GetEvaluatorSubComponent();
RoleMappedData scoredTestData = RoleMappedData.Create(scorePipe,
GetColumnRoles(testData.Schema, scorePipe.Schema));
IEvaluator evaluator = evalComp.CreateInstance(Host);
// REVIEW: with the new evaluators, metrics of individual models are no longer
// printed to the Console. Consider adding an option on the combiner to print them.
// REVIEW: Consider adding an option to the combiner to save a data view
// containing all the results of the individual models.
var metricsDict = evaluator.Evaluate(scoredTestData);
if (!metricsDict.TryGetValue(MetricKinds.OverallMetrics, out IDataView metricsView))
{
throw Host.Except("Evaluator did not produce any overall metrics");
}
// REVIEW: We're assuming that the metrics of interest are always doubles here.
var metrics = EvaluateUtils.GetMetrics(metricsView, getVectorMetrics: false);
model.Metrics = metrics.ToArray();
ch.Done();
}
}
