/// <summary>
/// Evaluates scored multiclass classification data.
/// </summary>
/// <param name="data">The scored data.</param>
/// <param name="label">The name of the label column in <paramref name="data"/>.</param>
/// <param name="score">The name of the score column in <paramref name="data"/>.</param>
/// <param name="predictedLabel">The name of the predicted label column in <paramref name="data"/>.</param>
/// <param name="topK">If given a positive value, the <see cref="MulticlassClassificationMetrics.TopKAccuracy"/> will be filled with
/// the top-K accuracy, that is, the accuracy assuming we consider an example with the correct class within
/// the top-K values as being stored "correctly."</param>
/// <returns>The evaluation results for these calibrated outputs.</returns>
public MulticlassClassificationMetrics Evaluate(IDataView data, string label = DefaultColumnNames.Label, string score = DefaultColumnNames.Score,
string predictedLabel = DefaultColumnNames.PredictedLabel, int topK = 0)
{
Environment.CheckValue(data, nameof(data));
Environment.CheckNonEmpty(label, nameof(label));
Environment.CheckNonEmpty(score, nameof(score));
Environment.CheckNonEmpty(predictedLabel, nameof(predictedLabel));
var args = new MulticlassClassificationEvaluator.Arguments()
{
};
if (topK > 0)
{
args.OutputTopKAcc = topK;
}
var eval = new MulticlassClassificationEvaluator(Environment, args);
return(eval.Evaluate(data, label, score, predictedLabel));
}
/// <summary>
/// Evaluates scored multiclass classification data.
/// </summary>
/// <param name="data">The scored data.</param>
/// <param name="labelColumnName">The name of the label column in <paramref name="data"/>.</param>
/// <param name="scoreColumnName">The name of the score column in <paramref name="data"/>.</param>
/// <param name="predictedLabelColumnName">The name of the predicted label column in <paramref name="data"/>.</param>
/// <param name="topKPredictionCount">If given a positive value, the <see cref="MulticlassClassificationMetrics.TopKAccuracy"/> will be filled with
/// the top-K accuracy, that is, the accuracy assuming we consider an example with the correct class within
/// the top-K values as being stored "correctly."</param>
/// <returns>The evaluation results for these calibrated outputs.</returns>
public MulticlassClassificationMetrics Evaluate(IDataView data, string labelColumnName = DefaultColumnNames.Label, string scoreColumnName = DefaultColumnNames.Score,
string predictedLabelColumnName = DefaultColumnNames.PredictedLabel, int topKPredictionCount = 0)
{
Environment.CheckValue(data, nameof(data));
Environment.CheckNonEmpty(labelColumnName, nameof(labelColumnName));
Environment.CheckNonEmpty(scoreColumnName, nameof(scoreColumnName));
Environment.CheckNonEmpty(predictedLabelColumnName, nameof(predictedLabelColumnName));
Environment.CheckUserArg(topKPredictionCount >= 0, nameof(topKPredictionCount), "Must be non-negative");
var args = new MulticlassClassificationEvaluator.Arguments()
{
};
if (topKPredictionCount > 0)
{
args.OutputTopKAcc = topKPredictionCount;
}
var eval = new MulticlassClassificationEvaluator(Environment, args);
return(eval.Evaluate(data, labelColumnName, scoreColumnName, predictedLabelColumnName));
}
public void SetupPredictBenchmarks()
{
_trainedModel = Train(_dataPath);
_predictionEngine = _mlContext.Model.CreatePredictionEngine <IrisData, IrisPrediction>(_trainedModel);
_consumer.Consume(_predictionEngine.Predict(_example));
// Create text loader.
var options = new TextLoader.Options()
{
Columns = new[]
{
new TextLoader.Column("Label", DataKind.Single, 0),
new TextLoader.Column("SepalLength", DataKind.Single, 1),
new TextLoader.Column("SepalWidth", DataKind.Single, 2),
new TextLoader.Column("PetalLength", DataKind.Single, 3),
new TextLoader.Column("PetalWidth", DataKind.Single, 4),
},
HasHeader = true,
};
var loader = new TextLoader(_mlContext, options: options);
IDataView testData = loader.Load(_dataPath);
_scoredIrisTestData = _trainedModel.Transform(testData);
_evaluator = new MulticlassClassificationEvaluator(_mlContext, new MulticlassClassificationEvaluator.Arguments());
_metrics = _evaluator.Evaluate(_scoredIrisTestData, DefaultColumnNames.Label, DefaultColumnNames.Score, DefaultColumnNames.PredictedLabel);
_batches = new IrisData[_batchSizes.Length][];
for (int i = 0; i < _batches.Length; i++)
{
var batch = new IrisData[_batchSizes[i]];
for (int bi = 0; bi < batch.Length; bi++)
{
batch[bi] = _example;
}
_batches[i] = batch;
}
}
public void EvaluateMetrics() => _evaluator.Evaluate(_scoredIrisTestData, DefaultColumnNames.Label, DefaultColumnNames.Score, DefaultColumnNames.PredictedLabel);