/// <summary>
/// Executes the test for a given recommender and a test dataset.
/// </summary>
/// <param name="recommender">The recommender to evaluate.</param>
/// <param name="evaluator">The evaluator.</param>
/// <param name="testDataset">The test dataset.</param>
/// <param name="predictionTime">When the method returns, this parameter contains the total prediction time.</param>
/// <param name="evaluationTime">When the method returns, this parameter contains the total evaluation time.</param>
/// <param name="metrics">When the method returns, this parameter contains names and values of the computed metrics.</param>
public override void Execute(
Recommender recommender,
Evaluator evaluator,
SplitInstanceSource <RecommenderDataset> testDataset,
out TimeSpan predictionTime,
out TimeSpan evaluationTime,
out MetricValueDistributionCollection metrics)
{
Stopwatch predictionTimer = Stopwatch.StartNew();
IDictionary <User, IDictionary <Item, int> > predictions = recommender.Predict(testDataset);
IDictionary <User, IDictionary <Item, RatingDistribution> > uncertainPredictions =
this.computeUncertainPredictionMetrics ? recommender.PredictDistribution(testDataset) : null;
predictionTime = predictionTimer.Elapsed;
var evaluationTimer = Stopwatch.StartNew();
metrics = new MetricValueDistributionCollection();
metrics.Add("MAE", new MetricValueDistribution(evaluator.ModelDomainRatingPredictionMetric(testDataset, predictions, Metrics.AbsoluteError)));
metrics.Add("Per-user MAE", new MetricValueDistribution(evaluator.ModelDomainRatingPredictionMetric(testDataset, predictions, Metrics.AbsoluteError, RecommenderMetricAggregationMethod.PerUserFirst)));
metrics.Add("RMSE", new MetricValueDistribution(Math.Sqrt(evaluator.ModelDomainRatingPredictionMetric(testDataset, predictions, Metrics.SquaredError))));
metrics.Add("Per-user RMSE", new MetricValueDistribution(Math.Sqrt(evaluator.ModelDomainRatingPredictionMetric(testDataset, predictions, Metrics.SquaredError, RecommenderMetricAggregationMethod.PerUserFirst))));
if (this.computeUncertainPredictionMetrics)
{
metrics.Add("Expected MAE", new MetricValueDistribution(evaluator.ModelDomainRatingPredictionMetricExpectation(testDataset, uncertainPredictions, Metrics.AbsoluteError)));
}
evaluationTime = evaluationTimer.Elapsed;
}
/// <summary>
/// Executes the test for a given recommender and a test dataset.
/// </summary>
/// <param name="recommender">The recommender to evaluate.</param>
/// <param name="evaluator">The evaluator.</param>
/// <param name="testDataset">The test dataset.</param>
/// <param name="predictionTime">When the method returns, this parameter contains the total prediction time.</param>
/// <param name="evaluationTime">When the method returns, this parameter contains the total evaluation time.</param>
/// <param name="metrics">When the method returns, this parameter contains names and values of the computed metrics.</param>
public override void Execute(
Recommender recommender,
Evaluator evaluator,
SplitInstanceSource <RecommenderDataset> testDataset,
out TimeSpan predictionTime,
out TimeSpan evaluationTime,
out MetricValueDistributionCollection metrics)
{
Stopwatch predictionTimer = Stopwatch.StartNew();
IDictionary <User, IEnumerable <User> > predictions = evaluator.FindRelatedUsersWhoRatedSameItems(
recommender, testDataset, this.maxRelatedUserCount, this.minCommonRatingCount, this.minRelatedUserPoolSize);
predictionTime = predictionTimer.Elapsed;
Stopwatch evaluationTimer = Stopwatch.StartNew();
metrics = new MetricValueDistributionCollection();
metrics.Add(
"Related users L1 Sim NDCG",
new MetricValueDistribution(evaluator.RelatedUsersMetric(testDataset, predictions, this.minCommonRatingCount, Metrics.Ndcg, Metrics.NormalizedManhattanSimilarity)));
metrics.Add(
"Related users L2 Sim NDCG",
new MetricValueDistribution(evaluator.RelatedUsersMetric(testDataset, predictions, this.minCommonRatingCount, Metrics.Ndcg, Metrics.NormalizedEuclideanSimilarity)));
evaluationTime = evaluationTimer.Elapsed;
}
/// <summary>
/// This function should be implemented in the derived classes to execute the test for a given recommender and a test dataset.
/// </summary>
/// <param name="recommender">The recommender to evaluate.</param>
/// <param name="evaluator">The evaluator.</param>
/// <param name="testDataset">The test dataset.</param>
/// <param name="predictionTime">When the method returns, this parameter contains the total prediction time.</param>
/// <param name="evaluationTime">When the method returns, this parameter contains the total evaluation time.</param>
/// <param name="metrics">When the method returns, this parameter contains names and values of the computed metrics.</param>
public abstract void Execute(
Recommender recommender,
Evaluator evaluator,
SplitInstanceSource <RecommenderDataset> testDataset,
out TimeSpan predictionTime,
out TimeSpan evaluationTime,
out MetricValueDistributionCollection metrics);
/// <summary>
/// Adds metric value distributions from the given collection to this collection.
/// Collections must have disjoint sets of metrics.
/// </summary>
/// <param name="other">The collection to compute union with.</param>
public void SetToUnionWith(MetricValueDistributionCollection other)
{
Debug.Assert(other != null, "A valid collection should be provided.");
foreach (KeyValuePair <string, MetricValueDistribution> metricValueDistribution in other.metricNameToValueDistribution)
{
this.Add(metricValueDistribution.Key, metricValueDistribution.Value);
}
}
/// <summary>
/// Merges metric value distributions in this collection with the distributions in a given collection.
/// Both collections must have the same set of metrics.
/// </summary>
/// <param name="other">The collection to merge with.</param>
public void MergeWith(MetricValueDistributionCollection other)
{
Debug.Assert(other != null, "A valid collection should be provided.");
Debug.Assert(
this.metricNameToValueDistribution.Count == other.metricNameToValueDistribution.Count,
"Both collections should have the same set of metrics.");
foreach (KeyValuePair <string, MetricValueDistribution> metricValueDistribution in other.metricNameToValueDistribution)
{
Debug.Assert(
this.metricNameToValueDistribution.ContainsKey(metricValueDistribution.Key),
"Both collections should have the same set of metrics.");
this.metricNameToValueDistribution[metricValueDistribution.Key].MergeWith(metricValueDistribution.Value);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="RecommenderRunCompletedEventArgs"/> class.
/// </summary>
/// <param name="totalTime">The total time of the run.</param>
/// <param name="trainingTime">The total training time.</param>
/// <param name="predictionTime">The total prediction time.</param>
/// <param name="evaluationTime">The total evaluation time.</param>
/// <param name="metrics">The collection of metric names and value distributions.</param>
public RecommenderRunCompletedEventArgs(
TimeSpan totalTime,
TimeSpan trainingTime,
TimeSpan predictionTime,
TimeSpan evaluationTime,
MetricValueDistributionCollection metrics)
{
Debug.Assert(totalTime >= trainingTime + predictionTime + evaluationTime, "Given time spans are inconsistent.");
Debug.Assert(metrics != null, "Valid metrics should be provided.");
this.TotalTime = totalTime;
this.TrainingTime = trainingTime;
this.PredictionTime = predictionTime;
this.EvaluationTime = evaluationTime;
this.Metrics = metrics;
}
/// <summary>
/// Initializes a new instance of the <see cref="RecommenderRunFoldProcessedEventArgs"/> class.
/// </summary>
/// <param name="foldNumber">The number of the fold that has been processed.</param>
/// <param name="totalTime">The total time spent while processing the fold.</param>
/// <param name="trainingTime">The total training time for the fold.</param>
/// <param name="predictionTime">The total prediction time for the fold.</param>
/// <param name="evaluationTime">The total evaluation time for the fold.</param>
/// <param name="metrics">The collection of metric names and value distributions.</param>
public RecommenderRunFoldProcessedEventArgs(
int foldNumber,
TimeSpan totalTime,
TimeSpan trainingTime,
TimeSpan predictionTime,
TimeSpan evaluationTime,
MetricValueDistributionCollection metrics)
{
Debug.Assert(foldNumber >= 0, "A valid fold number should be provided.");
Debug.Assert(totalTime >= trainingTime + predictionTime + evaluationTime, "Given time spans are inconsistent.");
Debug.Assert(metrics != null, "Valid metrics should be provided.");
this.FoldNumber = foldNumber;
this.TotalTime = totalTime;
this.TrainingTime = trainingTime;
this.PredictionTime = predictionTime;
this.EvaluationTime = evaluationTime;
this.Metrics = metrics;
}
/// <summary>
/// Executes the test for a given recommender and a test dataset.
/// </summary>
/// <param name="recommender">The recommender to evaluate.</param>
/// <param name="evaluator">The evaluator.</param>
/// <param name="testDataset">The test dataset.</param>
/// <param name="predictionTime">When the method returns, this parameter contains the total prediction time.</param>
/// <param name="evaluationTime">When the method returns, this parameter contains the total evaluation time.</param>
/// <param name="metrics">When the method returns, this parameter contains names and values of the computed metrics.</param>
public override void Execute(
Recommender recommender,
Evaluator evaluator,
SplitInstanceSource <RecommenderDataset> testDataset,
out TimeSpan predictionTime,
out TimeSpan evaluationTime,
out MetricValueDistributionCollection metrics)
{
var predictionTimer = Stopwatch.StartNew();
IDictionary <User, IEnumerable <Item> > recommendations = evaluator.RecommendRatedItems(
recommender, testDataset, this.maxRecommendedItemCount, this.minRecommendationPoolSize);
predictionTime = predictionTimer.Elapsed;
var evaluationTimer = Stopwatch.StartNew();
metrics = new MetricValueDistributionCollection();
Func <int, double> ratingToGainConverter = r => r - testDataset.InstanceSource.StarRatingInfo.MinStarRating + 1; // Prevent non-positive gains
metrics.Add("Recommendation NDCG", new MetricValueDistribution(evaluator.ItemRecommendationMetric(testDataset, recommendations, Metrics.Ndcg, ratingToGainConverter)));
metrics.Add("Recommendation GAP", new MetricValueDistribution(evaluator.ItemRecommendationMetric(testDataset, recommendations, Metrics.GradedAveragePrecision, ratingToGainConverter)));
evaluationTime = evaluationTimer.Elapsed;
}
/// <summary>
/// Executes the test for a given recommender under a specified name.
/// </summary>
public void Execute()
{
// Report that the run has been started
if (this.Started != null)
{
this.Started(this, EventArgs.Empty);
}
try
{
Rand.Restart(1984); // Run should produce the same results every time
TimeSpan totalTrainingTime = TimeSpan.Zero;
TimeSpan totalPredictionTime = TimeSpan.Zero;
TimeSpan totalEvaluationTime = TimeSpan.Zero;
Stopwatch totalTimer = Stopwatch.StartNew();
MetricValueDistributionCollection metrics = null;
for (int i = 0; i < this.FoldCount; ++i)
{
// Start timer measuring total time spent on this fold
Stopwatch totalFoldTimer = Stopwatch.StartNew();
SplittingMapping splittingMapping = this.SplittingMappingFactory();
Recommender recommender = this.RecommenderFactory(splittingMapping);
Evaluator evaluator = new Evaluator(new EvaluatorMapping(splittingMapping));
// Train the recommender
Stopwatch foldTrainingTimer = Stopwatch.StartNew();
recommender.Train(SplitInstanceSource.Training(this.RecommenderDataset));
TimeSpan foldTrainingTime = foldTrainingTimer.Elapsed;
// Run each test on the trained recommender
var foldMetrics = new MetricValueDistributionCollection();
TimeSpan foldPredictionTime = TimeSpan.Zero;
TimeSpan foldEvaluationTime = TimeSpan.Zero;
foreach (RecommenderTest test in this.Tests)
{
// Perform the test
TimeSpan testPredictionTime, testEvaluationTime;
MetricValueDistributionCollection testMetrics;
test.Execute(
recommender,
evaluator,
SplitInstanceSource.Test(this.RecommenderDataset),
out testPredictionTime,
out testEvaluationTime,
out testMetrics);
// Merge the timings and the metrics
foldPredictionTime += testPredictionTime;
foldEvaluationTime += testEvaluationTime;
foldMetrics.SetToUnionWith(testMetrics);
}
// Stop timer measuring total time spent on this fold
TimeSpan totalFoldTime = totalFoldTimer.Elapsed;
// Report that the fold has been processed
if (this.FoldProcessed != null)
{
this.FoldProcessed(
this,
new RecommenderRunFoldProcessedEventArgs(i, totalFoldTime, foldTrainingTime, foldPredictionTime, foldEvaluationTime, foldMetrics));
}
// Merge the timings
totalTrainingTime += foldTrainingTime;
totalPredictionTime += foldPredictionTime;
totalEvaluationTime += foldEvaluationTime;
// Merge the metrics
if (metrics == null)
{
metrics = foldMetrics;
}
else
{
metrics.MergeWith(foldMetrics);
}
}
// Report that the run has been completed
TimeSpan totalTime = totalTimer.Elapsed;
if (this.Completed != null)
{
this.Completed(
this,
new RecommenderRunCompletedEventArgs(totalTime, totalTrainingTime, totalPredictionTime, totalEvaluationTime, metrics));
}
}
catch (Exception e)
{
if (this.Interrupted != null)
{
this.Interrupted(this, new RecommenderRunInterruptedEventArgs(e));
}
}
}
