public void SetupPredictBenchmarks()
{
_trainedModel = Train(_dataPath);
_predictionEngine = _trainedModel.CreatePredictionEngine <IrisData, IrisPrediction>(_env);
_consumer.Consume(_predictionEngine.Predict(_example));
var reader = new TextLoader(_env,
columns: new[]
{
new TextLoader.Column("Label", DataKind.R4, 0),
new TextLoader.Column("SepalLength", DataKind.R4, 1),
new TextLoader.Column("SepalWidth", DataKind.R4, 2),
new TextLoader.Column("PetalLength", DataKind.R4, 3),
new TextLoader.Column("PetalWidth", DataKind.R4, 4),
},
hasHeader: true
);
IDataView testData = reader.Read(_dataPath);
IDataView scoredTestData = _trainedModel.Transform(testData);
var evaluator = new MultiClassClassifierEvaluator(_env, new MultiClassClassifierEvaluator.Arguments());
_metrics = evaluator.Evaluate(scoredTestData, DefaultColumnNames.Label, DefaultColumnNames.Score, DefaultColumnNames.PredictedLabel);
_batches = new IrisData[_batchSizes.Length][];
for (int i = 0; i < _batches.Length; i++)
{
var batch = new IrisData[_batchSizes[i]];
for (int bi = 0; bi < batch.Length; bi++)
{
batch[bi] = _example;
}
_batches[i] = batch;
}
}
protected void EvaluateModel(string testLocation, PredictionFunction <ImageNetData, ImageNetPrediction> model)
{
ConsoleWriteHeader("Metrics for Image Classification");
var evaluator = new MultiClassClassifierEvaluator(env, new MultiClassClassifierEvaluator.Arguments());
var data = TextLoader.ReadFile(env,
new TextLoader.Arguments
{
Separator = "tab",
HasHeader = false,
Column = new []
{
new TextLoader.Column("ImagePath", DataKind.Text, 0),
new TextLoader.Column("Label", DataKind.Text, 1)
}
},
new MultiFileSource(testLocation));
var evalRoles = new RoleMappedData(data, label: "Label", feature: "ImagePath");
var metrics = evaluator.Evaluate(evalRoles);
//var logLoss = metrics[MultiClassClassifierEvaluator.LogLoss];
foreach (var item in metrics)
{
Console.WriteLine($"{item.Key}: {item.Value}");
}
}
/// <summary>
/// Evaluates scored multiclass classification data.
/// </summary>
/// <param name="data">The scored data.</param>
/// <param name="label">The name of the label column in <paramref name="data"/>.</param>
/// <param name="score">The name of the score column in <paramref name="data"/>.</param>
/// <param name="predictedLabel">The name of the predicted label column in <paramref name="data"/>.</param>
/// <param name="topK">If given a positive value, the <see cref="MultiClassClassifierMetrics.TopKAccuracy"/> will be filled with
/// the top-K accuracy, that is, the accuracy assuming we consider an example with the correct class within
/// the top-K values as being stored "correctly."</param>
/// <returns>The evaluation results for these calibrated outputs.</returns>
public MultiClassClassifierMetrics Evaluate(IDataView data, string label = DefaultColumnNames.Label, string score = DefaultColumnNames.Score,
string predictedLabel = DefaultColumnNames.PredictedLabel, int topK = 0)
{
Environment.CheckValue(data, nameof(data));
Environment.CheckNonEmpty(label, nameof(label));
Environment.CheckNonEmpty(score, nameof(score));
Environment.CheckNonEmpty(predictedLabel, nameof(predictedLabel));
var args = new MultiClassClassifierEvaluator.Arguments()
{
};
if (topK > 0)
{
args.OutputTopKAcc = topK;
}
var eval = new MultiClassClassifierEvaluator(Environment, args);
return(eval.Evaluate(data, label, score, predictedLabel));
}
