/// <summary> /// Executes the test for a given recommender and a test dataset. /// </summary> /// <param name="recommender">The recommender to evaluate.</param> /// <param name="evaluator">The evaluator.</param> /// <param name="testDataset">The test dataset.</param> /// <param name="predictionTime">When the method returns, this parameter contains the total prediction time.</param> /// <param name="evaluationTime">When the method returns, this parameter contains the total evaluation time.</param> /// <param name="metrics">When the method returns, this parameter contains names and values of the computed metrics.</param> public override void Execute( Recommender recommender, Evaluator evaluator, SplitInstanceSource <RecommenderDataset> testDataset, out TimeSpan predictionTime, out TimeSpan evaluationTime, out MetricValueDistributionCollection metrics) { Stopwatch predictionTimer = Stopwatch.StartNew(); IDictionary <User, IDictionary <Item, int> > predictions = recommender.Predict(testDataset); IDictionary <User, IDictionary <Item, RatingDistribution> > uncertainPredictions = this.computeUncertainPredictionMetrics ? recommender.PredictDistribution(testDataset) : null; predictionTime = predictionTimer.Elapsed; var evaluationTimer = Stopwatch.StartNew(); metrics = new MetricValueDistributionCollection(); metrics.Add("MAE", new MetricValueDistribution(evaluator.ModelDomainRatingPredictionMetric(testDataset, predictions, Metrics.AbsoluteError))); metrics.Add("Per-user MAE", new MetricValueDistribution(evaluator.ModelDomainRatingPredictionMetric(testDataset, predictions, Metrics.AbsoluteError, RecommenderMetricAggregationMethod.PerUserFirst))); metrics.Add("RMSE", new MetricValueDistribution(Math.Sqrt(evaluator.ModelDomainRatingPredictionMetric(testDataset, predictions, Metrics.SquaredError)))); metrics.Add("Per-user RMSE", new MetricValueDistribution(Math.Sqrt(evaluator.ModelDomainRatingPredictionMetric(testDataset, predictions, Metrics.SquaredError, RecommenderMetricAggregationMethod.PerUserFirst)))); if (this.computeUncertainPredictionMetrics) { metrics.Add("Expected MAE", new MetricValueDistribution(evaluator.ModelDomainRatingPredictionMetricExpectation(testDataset, uncertainPredictions, Metrics.AbsoluteError))); } evaluationTime = evaluationTimer.Elapsed; }
/// <summary> /// Executes the test for a given recommender and a test dataset. /// </summary> /// <param name="recommender">The recommender to evaluate.</param> /// <param name="evaluator">The evaluator.</param> /// <param name="testDataset">The test dataset.</param> /// <param name="predictionTime">When the method returns, this parameter contains the total prediction time.</param> /// <param name="evaluationTime">When the method returns, this parameter contains the total evaluation time.</param> /// <param name="metrics">When the method returns, this parameter contains names and values of the computed metrics.</param> public override void Execute( Recommender recommender, Evaluator evaluator, SplitInstanceSource <RecommenderDataset> testDataset, out TimeSpan predictionTime, out TimeSpan evaluationTime, out MetricValueDistributionCollection metrics) { Stopwatch predictionTimer = Stopwatch.StartNew(); IDictionary <User, IEnumerable <User> > predictions = evaluator.FindRelatedUsersWhoRatedSameItems( recommender, testDataset, this.maxRelatedUserCount, this.minCommonRatingCount, this.minRelatedUserPoolSize); predictionTime = predictionTimer.Elapsed; Stopwatch evaluationTimer = Stopwatch.StartNew(); metrics = new MetricValueDistributionCollection(); metrics.Add( "Related users L1 Sim NDCG", new MetricValueDistribution(evaluator.RelatedUsersMetric(testDataset, predictions, this.minCommonRatingCount, Metrics.Ndcg, Metrics.NormalizedManhattanSimilarity))); metrics.Add( "Related users L2 Sim NDCG", new MetricValueDistribution(evaluator.RelatedUsersMetric(testDataset, predictions, this.minCommonRatingCount, Metrics.Ndcg, Metrics.NormalizedEuclideanSimilarity))); evaluationTime = evaluationTimer.Elapsed; }
/// <summary> /// This function should be implemented in the derived classes to execute the test for a given recommender and a test dataset. /// </summary> /// <param name="recommender">The recommender to evaluate.</param> /// <param name="evaluator">The evaluator.</param> /// <param name="testDataset">The test dataset.</param> /// <param name="predictionTime">When the method returns, this parameter contains the total prediction time.</param> /// <param name="evaluationTime">When the method returns, this parameter contains the total evaluation time.</param> /// <param name="metrics">When the method returns, this parameter contains names and values of the computed metrics.</param> public abstract void Execute( Recommender recommender, Evaluator evaluator, SplitInstanceSource <RecommenderDataset> testDataset, out TimeSpan predictionTime, out TimeSpan evaluationTime, out MetricValueDistributionCollection metrics);
/// <summary> /// Adds metric value distributions from the given collection to this collection. /// Collections must have disjoint sets of metrics. /// </summary> /// <param name="other">The collection to compute union with.</param> public void SetToUnionWith(MetricValueDistributionCollection other) { Debug.Assert(other != null, "A valid collection should be provided."); foreach (KeyValuePair <string, MetricValueDistribution> metricValueDistribution in other.metricNameToValueDistribution) { this.Add(metricValueDistribution.Key, metricValueDistribution.Value); } }
/// <summary> /// Merges metric value distributions in this collection with the distributions in a given collection. /// Both collections must have the same set of metrics. /// </summary> /// <param name="other">The collection to merge with.</param> public void MergeWith(MetricValueDistributionCollection other) { Debug.Assert(other != null, "A valid collection should be provided."); Debug.Assert( this.metricNameToValueDistribution.Count == other.metricNameToValueDistribution.Count, "Both collections should have the same set of metrics."); foreach (KeyValuePair <string, MetricValueDistribution> metricValueDistribution in other.metricNameToValueDistribution) { Debug.Assert( this.metricNameToValueDistribution.ContainsKey(metricValueDistribution.Key), "Both collections should have the same set of metrics."); this.metricNameToValueDistribution[metricValueDistribution.Key].MergeWith(metricValueDistribution.Value); } }
/// <summary> /// Initializes a new instance of the <see cref="RecommenderRunCompletedEventArgs"/> class. /// </summary> /// <param name="totalTime">The total time of the run.</param> /// <param name="trainingTime">The total training time.</param> /// <param name="predictionTime">The total prediction time.</param> /// <param name="evaluationTime">The total evaluation time.</param> /// <param name="metrics">The collection of metric names and value distributions.</param> public RecommenderRunCompletedEventArgs( TimeSpan totalTime, TimeSpan trainingTime, TimeSpan predictionTime, TimeSpan evaluationTime, MetricValueDistributionCollection metrics) { Debug.Assert(totalTime >= trainingTime + predictionTime + evaluationTime, "Given time spans are inconsistent."); Debug.Assert(metrics != null, "Valid metrics should be provided."); this.TotalTime = totalTime; this.TrainingTime = trainingTime; this.PredictionTime = predictionTime; this.EvaluationTime = evaluationTime; this.Metrics = metrics; }
/// <summary> /// Initializes a new instance of the <see cref="RecommenderRunFoldProcessedEventArgs"/> class. /// </summary> /// <param name="foldNumber">The number of the fold that has been processed.</param> /// <param name="totalTime">The total time spent while processing the fold.</param> /// <param name="trainingTime">The total training time for the fold.</param> /// <param name="predictionTime">The total prediction time for the fold.</param> /// <param name="evaluationTime">The total evaluation time for the fold.</param> /// <param name="metrics">The collection of metric names and value distributions.</param> public RecommenderRunFoldProcessedEventArgs( int foldNumber, TimeSpan totalTime, TimeSpan trainingTime, TimeSpan predictionTime, TimeSpan evaluationTime, MetricValueDistributionCollection metrics) { Debug.Assert(foldNumber >= 0, "A valid fold number should be provided."); Debug.Assert(totalTime >= trainingTime + predictionTime + evaluationTime, "Given time spans are inconsistent."); Debug.Assert(metrics != null, "Valid metrics should be provided."); this.FoldNumber = foldNumber; this.TotalTime = totalTime; this.TrainingTime = trainingTime; this.PredictionTime = predictionTime; this.EvaluationTime = evaluationTime; this.Metrics = metrics; }
/// <summary> /// Executes the test for a given recommender and a test dataset. /// </summary> /// <param name="recommender">The recommender to evaluate.</param> /// <param name="evaluator">The evaluator.</param> /// <param name="testDataset">The test dataset.</param> /// <param name="predictionTime">When the method returns, this parameter contains the total prediction time.</param> /// <param name="evaluationTime">When the method returns, this parameter contains the total evaluation time.</param> /// <param name="metrics">When the method returns, this parameter contains names and values of the computed metrics.</param> public override void Execute( Recommender recommender, Evaluator evaluator, SplitInstanceSource <RecommenderDataset> testDataset, out TimeSpan predictionTime, out TimeSpan evaluationTime, out MetricValueDistributionCollection metrics) { var predictionTimer = Stopwatch.StartNew(); IDictionary <User, IEnumerable <Item> > recommendations = evaluator.RecommendRatedItems( recommender, testDataset, this.maxRecommendedItemCount, this.minRecommendationPoolSize); predictionTime = predictionTimer.Elapsed; var evaluationTimer = Stopwatch.StartNew(); metrics = new MetricValueDistributionCollection(); Func <int, double> ratingToGainConverter = r => r - testDataset.InstanceSource.StarRatingInfo.MinStarRating + 1; // Prevent non-positive gains metrics.Add("Recommendation NDCG", new MetricValueDistribution(evaluator.ItemRecommendationMetric(testDataset, recommendations, Metrics.Ndcg, ratingToGainConverter))); metrics.Add("Recommendation GAP", new MetricValueDistribution(evaluator.ItemRecommendationMetric(testDataset, recommendations, Metrics.GradedAveragePrecision, ratingToGainConverter))); evaluationTime = evaluationTimer.Elapsed; }
/// <summary> /// Executes the test for a given recommender under a specified name. /// </summary> public void Execute() { // Report that the run has been started if (this.Started != null) { this.Started(this, EventArgs.Empty); } try { Rand.Restart(1984); // Run should produce the same results every time TimeSpan totalTrainingTime = TimeSpan.Zero; TimeSpan totalPredictionTime = TimeSpan.Zero; TimeSpan totalEvaluationTime = TimeSpan.Zero; Stopwatch totalTimer = Stopwatch.StartNew(); MetricValueDistributionCollection metrics = null; for (int i = 0; i < this.FoldCount; ++i) { // Start timer measuring total time spent on this fold Stopwatch totalFoldTimer = Stopwatch.StartNew(); SplittingMapping splittingMapping = this.SplittingMappingFactory(); Recommender recommender = this.RecommenderFactory(splittingMapping); Evaluator evaluator = new Evaluator(new EvaluatorMapping(splittingMapping)); // Train the recommender Stopwatch foldTrainingTimer = Stopwatch.StartNew(); recommender.Train(SplitInstanceSource.Training(this.RecommenderDataset)); TimeSpan foldTrainingTime = foldTrainingTimer.Elapsed; // Run each test on the trained recommender var foldMetrics = new MetricValueDistributionCollection(); TimeSpan foldPredictionTime = TimeSpan.Zero; TimeSpan foldEvaluationTime = TimeSpan.Zero; foreach (RecommenderTest test in this.Tests) { // Perform the test TimeSpan testPredictionTime, testEvaluationTime; MetricValueDistributionCollection testMetrics; test.Execute( recommender, evaluator, SplitInstanceSource.Test(this.RecommenderDataset), out testPredictionTime, out testEvaluationTime, out testMetrics); // Merge the timings and the metrics foldPredictionTime += testPredictionTime; foldEvaluationTime += testEvaluationTime; foldMetrics.SetToUnionWith(testMetrics); } // Stop timer measuring total time spent on this fold TimeSpan totalFoldTime = totalFoldTimer.Elapsed; // Report that the fold has been processed if (this.FoldProcessed != null) { this.FoldProcessed( this, new RecommenderRunFoldProcessedEventArgs(i, totalFoldTime, foldTrainingTime, foldPredictionTime, foldEvaluationTime, foldMetrics)); } // Merge the timings totalTrainingTime += foldTrainingTime; totalPredictionTime += foldPredictionTime; totalEvaluationTime += foldEvaluationTime; // Merge the metrics if (metrics == null) { metrics = foldMetrics; } else { metrics.MergeWith(foldMetrics); } } // Report that the run has been completed TimeSpan totalTime = totalTimer.Elapsed; if (this.Completed != null) { this.Completed( this, new RecommenderRunCompletedEventArgs(totalTime, totalTrainingTime, totalPredictionTime, totalEvaluationTime, metrics)); } } catch (Exception e) { if (this.Interrupted != null) { this.Interrupted(this, new RecommenderRunInterruptedEventArgs(e)); } } }