/// <summary>
/// Executes the test for a given recommender under a specified name.
/// </summary>
public void Execute()
{
// Report that the run has been started
if (this.Started != null)
{
this.Started(this, EventArgs.Empty);
}
try
{
Rand.Restart(1984); // Run should produce the same results every time
TimeSpan totalTrainingTime = TimeSpan.Zero;
TimeSpan totalPredictionTime = TimeSpan.Zero;
TimeSpan totalEvaluationTime = TimeSpan.Zero;
Stopwatch totalTimer = Stopwatch.StartNew();
MetricValueDistributionCollection metrics = null;
for (int i = 0; i < this.FoldCount; ++i)
{
// Start timer measuring total time spent on this fold
Stopwatch totalFoldTimer = Stopwatch.StartNew();
SplittingMapping splittingMapping = this.SplittingMappingFactory();
Recommender recommender = this.RecommenderFactory(splittingMapping);
Evaluator evaluator = new Evaluator(new EvaluatorMapping(splittingMapping));
// Train the recommender
Stopwatch foldTrainingTimer = Stopwatch.StartNew();
recommender.Train(SplitInstanceSource.Training(this.RecommenderDataset));
TimeSpan foldTrainingTime = foldTrainingTimer.Elapsed;
// Run each test on the trained recommender
var foldMetrics = new MetricValueDistributionCollection();
TimeSpan foldPredictionTime = TimeSpan.Zero;
TimeSpan foldEvaluationTime = TimeSpan.Zero;
foreach (RecommenderTest test in this.Tests)
{
// Perform the test
TimeSpan testPredictionTime, testEvaluationTime;
MetricValueDistributionCollection testMetrics;
test.Execute(
recommender,
evaluator,
SplitInstanceSource.Test(this.RecommenderDataset),
out testPredictionTime,
out testEvaluationTime,
out testMetrics);
// Merge the timings and the metrics
foldPredictionTime += testPredictionTime;
foldEvaluationTime += testEvaluationTime;
foldMetrics.SetToUnionWith(testMetrics);
}
// Stop timer measuring total time spent on this fold
TimeSpan totalFoldTime = totalFoldTimer.Elapsed;
// Report that the fold has been processed
if (this.FoldProcessed != null)
{
this.FoldProcessed(
this,
new RecommenderRunFoldProcessedEventArgs(i, totalFoldTime, foldTrainingTime, foldPredictionTime, foldEvaluationTime, foldMetrics));
}
// Merge the timings
totalTrainingTime += foldTrainingTime;
totalPredictionTime += foldPredictionTime;
totalEvaluationTime += foldEvaluationTime;
// Merge the metrics
if (metrics == null)
{
metrics = foldMetrics;
}
else
{
metrics.MergeWith(foldMetrics);
}
}
// Report that the run has been completed
TimeSpan totalTime = totalTimer.Elapsed;
if (this.Completed != null)
{
this.Completed(
this,
new RecommenderRunCompletedEventArgs(totalTime, totalTrainingTime, totalPredictionTime, totalEvaluationTime, metrics));
}
}
catch (Exception e)
{
if (this.Interrupted != null)
{
this.Interrupted(this, new RecommenderRunInterruptedEventArgs(e));
}
}
}
