/// <summary> /// Calculates absolute error between prediction and ground truth /// </summary> /// <param name="testSource">A source of ratings made by users</param> /// <param name="mapping">A mapping to convert source ratings into experiment specific ones</param> /// <param name="predictions">Ratings which are predicted by the recommender</param> /// <returns></returns> private static Dictionary <Movie, List <double> > PredictionError( SplitInstanceSource <string> testSource, IStarRatingRecommenderMapping <SplitInstanceSource <string>, RatingTriple, string, Movie, int, NoFeatureSource, Vector> mapping, IDictionary <string, IDictionary <Movie, IDictionary <int, double> > > predictions) { var evMapping = mapping.ForEvaluation(); var allErrors = new Dictionary <Movie, List <double> >(); foreach (var userWithPredictionList in predictions) { foreach (var itemPrediction in userWithPredictionList.Value) { var prediction = itemPrediction.Value; var groundTruth = evMapping.GetRating(testSource, userWithPredictionList.Key, itemPrediction.Key); var predictedRating = prediction.Aggregate((l, r) => l.Value > r.Value ? l : r).Key; var error = Math.Abs(groundTruth - predictedRating); if (allErrors.TryGetValue(itemPrediction.Key, out var values)) { values.Add(error); } else { allErrors.Add(itemPrediction.Key, new List <double>() { error }); } } } return(allErrors); }
/// <summary> /// Calculates MAE of items placed to different buckets according to Number ofrating given to them. /// </summary> /// <param name="trainSource">An instance source</param> /// <param name="mapping">Mapping which converts input data to RatingTriples</param> /// <param name="predictionErrors">Absolute errors of predictions</param> /// <returns></returns> private static Dictionary <string, double> CreateItemPopularityPredictions( SplitInstanceSource <string> trainSource, IRecommenderMapping <SplitInstanceSource <string>, RatingTriple, string, Movie, NoFeatureSource, Vector> mapping, Dictionary <Movie, List <double> > predictionErrors) { Rand.Restart(RandomSeed); const int BucketCount = 4; var trainingSetCounts = mapping.GetInstances(trainSource) .GroupBy(i => i.Movie) .ToDictionary(x => x.Key, x => x.Count()); var orderedItems = predictionErrors .Keys .Select(x => new { Item = x, Count = GetItemPopularityCount(x, trainingSetCounts) }) .OrderBy(x => x.Count) .Select(x => x.Item) .ToArray(); var result = new Dictionary <string, double>(); for (var bucket = 0; bucket < BucketCount; ++bucket) { var items = GetFixedItems(bucket, orderedItems, trainingSetCounts); var bucketName = GetBucketName(items, trainingSetCounts); var mae = ComputeMae(items, predictionErrors) / 2.0; result.Add(bucketName, mae); } return(result); }
/// <summary> /// Executes the test for a given recommender and a test dataset. /// </summary> /// <param name="recommender">The recommender to evaluate.</param> /// <param name="evaluator">The evaluator.</param> /// <param name="testDataset">The test dataset.</param> /// <param name="predictionTime">When the method returns, this parameter contains the total prediction time.</param> /// <param name="evaluationTime">When the method returns, this parameter contains the total evaluation time.</param> /// <param name="metrics">When the method returns, this parameter contains names and values of the computed metrics.</param> public override void Execute( Recommender recommender, Evaluator evaluator, SplitInstanceSource <RecommenderDataset> testDataset, out TimeSpan predictionTime, out TimeSpan evaluationTime, out MetricValueDistributionCollection metrics) { Stopwatch predictionTimer = Stopwatch.StartNew(); IDictionary <User, IDictionary <Item, int> > predictions = recommender.Predict(testDataset); IDictionary <User, IDictionary <Item, RatingDistribution> > uncertainPredictions = this.computeUncertainPredictionMetrics ? recommender.PredictDistribution(testDataset) : null; predictionTime = predictionTimer.Elapsed; var evaluationTimer = Stopwatch.StartNew(); metrics = new MetricValueDistributionCollection(); metrics.Add("MAE", new MetricValueDistribution(evaluator.ModelDomainRatingPredictionMetric(testDataset, predictions, Metrics.AbsoluteError))); metrics.Add("Per-user MAE", new MetricValueDistribution(evaluator.ModelDomainRatingPredictionMetric(testDataset, predictions, Metrics.AbsoluteError, RecommenderMetricAggregationMethod.PerUserFirst))); metrics.Add("RMSE", new MetricValueDistribution(Math.Sqrt(evaluator.ModelDomainRatingPredictionMetric(testDataset, predictions, Metrics.SquaredError)))); metrics.Add("Per-user RMSE", new MetricValueDistribution(Math.Sqrt(evaluator.ModelDomainRatingPredictionMetric(testDataset, predictions, Metrics.SquaredError, RecommenderMetricAggregationMethod.PerUserFirst)))); if (this.computeUncertainPredictionMetrics) { metrics.Add("Expected MAE", new MetricValueDistribution(evaluator.ModelDomainRatingPredictionMetricExpectation(testDataset, uncertainPredictions, Metrics.AbsoluteError))); } evaluationTime = evaluationTimer.Elapsed; }
/// <summary> /// Calculates MAE on 10-star rating input data with feature info /// </summary> /// <returns> MAE of movies grouped by number of ratings given for them </returns> public Dictionary <string, double> GetRatingsToMaeOnFeaturePredictions() { var starRatingTrainTestSplittingMapping = RecommenderMappingFactory.GetStarsMapping(false); var trainSource = SplitInstanceSource.Training(RatingsPath); var testSource = SplitInstanceSource.Test(RatingsPath); Console.WriteLine($"Calculation of mean absolute error for movies with different numbers of ratings in the training set for data with feature info."); Rand.Restart(RandomSeed); var recommender = GetRecommender(starRatingTrainTestSplittingMapping, 16); recommender.Settings.Training.UseItemFeatures = true; recommender.Settings.Training.UseSharedUserThresholds = true; recommender.Settings.Training.Advanced.UserThresholdPriorVariance = 10; recommender.Train(trainSource); var distribution = recommender.PredictDistribution(testSource); var predictionError = PredictionError(testSource, starRatingTrainTestSplittingMapping, distribution); var ratingsNumToMae = CreateItemPopularityPredictions(trainSource, starRatingTrainTestSplittingMapping, predictionError); return(ratingsNumToMae); }
/// <summary> /// Executes the test for a given recommender and a test dataset. /// </summary> /// <param name="recommender">The recommender to evaluate.</param> /// <param name="evaluator">The evaluator.</param> /// <param name="testDataset">The test dataset.</param> /// <param name="predictionTime">When the method returns, this parameter contains the total prediction time.</param> /// <param name="evaluationTime">When the method returns, this parameter contains the total evaluation time.</param> /// <param name="metrics">When the method returns, this parameter contains names and values of the computed metrics.</param> public override void Execute( Recommender recommender, Evaluator evaluator, SplitInstanceSource <RecommenderDataset> testDataset, out TimeSpan predictionTime, out TimeSpan evaluationTime, out MetricValueDistributionCollection metrics) { Stopwatch predictionTimer = Stopwatch.StartNew(); IDictionary <User, IEnumerable <User> > predictions = evaluator.FindRelatedUsersWhoRatedSameItems( recommender, testDataset, this.maxRelatedUserCount, this.minCommonRatingCount, this.minRelatedUserPoolSize); predictionTime = predictionTimer.Elapsed; Stopwatch evaluationTimer = Stopwatch.StartNew(); metrics = new MetricValueDistributionCollection(); metrics.Add( "Related users L1 Sim NDCG", new MetricValueDistribution(evaluator.RelatedUsersMetric(testDataset, predictions, this.minCommonRatingCount, Metrics.Ndcg, Metrics.NormalizedManhattanSimilarity))); metrics.Add( "Related users L2 Sim NDCG", new MetricValueDistribution(evaluator.RelatedUsersMetric(testDataset, predictions, this.minCommonRatingCount, Metrics.Ndcg, Metrics.NormalizedEuclideanSimilarity))); evaluationTime = evaluationTimer.Elapsed; }
/// <summary> /// This function should be implemented in the derived classes to execute the test for a given recommender and a test dataset. /// </summary> /// <param name="recommender">The recommender to evaluate.</param> /// <param name="evaluator">The evaluator.</param> /// <param name="testDataset">The test dataset.</param> /// <param name="predictionTime">When the method returns, this parameter contains the total prediction time.</param> /// <param name="evaluationTime">When the method returns, this parameter contains the total evaluation time.</param> /// <param name="metrics">When the method returns, this parameter contains names and values of the computed metrics.</param> public abstract void Execute( Recommender recommender, Evaluator evaluator, SplitInstanceSource <RecommenderDataset> testDataset, out TimeSpan predictionTime, out TimeSpan evaluationTime, out MetricValueDistributionCollection metrics);
/// <summary> /// Runs the module. /// </summary> /// <param name="args">The command line arguments for the module.</param> /// <param name="usagePrefix">The prefix to print before the usage string.</param> /// <returns>True if the run was successful, false otherwise.</returns> public override bool Run(string[] args, string usagePrefix) { string inputDatasetFile = string.Empty; string outputTrainingDatasetFile = string.Empty; string outputTestDatasetFile = string.Empty; double trainingOnlyUserFraction = 0.5; double testUserRatingTrainingFraction = 0.25; double coldUserFraction = 0; double coldItemFraction = 0; double ignoredUserFraction = 0; double ignoredItemFraction = 0; bool removeOccasionalColdItems = false; var parser = new CommandLineParser(); parser.RegisterParameterHandler("--input-data", "FILE", "Dataset to split", v => inputDatasetFile = v, CommandLineParameterType.Required); parser.RegisterParameterHandler("--output-data-train", "FILE", "Training part of the split dataset", v => outputTrainingDatasetFile = v, CommandLineParameterType.Required); parser.RegisterParameterHandler("--output-data-test", "FILE", "Test part of the split dataset", v => outputTestDatasetFile = v, CommandLineParameterType.Required); parser.RegisterParameterHandler("--training-users", "NUM", "Fraction of training-only users; defaults to 0.5", (double v) => trainingOnlyUserFraction = v, CommandLineParameterType.Optional); parser.RegisterParameterHandler("--test-user-training-ratings", "NUM", "Fraction of test user ratings for training; defaults to 0.25", (double v) => testUserRatingTrainingFraction = v, CommandLineParameterType.Optional); parser.RegisterParameterHandler("--cold-users", "NUM", "Fraction of cold (test-only) users; defaults to 0", (double v) => coldUserFraction = v, CommandLineParameterType.Optional); parser.RegisterParameterHandler("--cold-items", "NUM", "Fraction of cold (test-only) items; defaults to 0", (double v) => coldItemFraction = v, CommandLineParameterType.Optional); parser.RegisterParameterHandler("--ignored-users", "NUM", "Fraction of ignored users; defaults to 0", (double v) => ignoredUserFraction = v, CommandLineParameterType.Optional); parser.RegisterParameterHandler("--ignored-items", "NUM", "Fraction of ignored items; defaults to 0", (double v) => ignoredItemFraction = v, CommandLineParameterType.Optional); parser.RegisterParameterHandler("--remove-occasional-cold-items", "Remove occasionally produced cold items", () => removeOccasionalColdItems = true); if (!parser.TryParse(args, usagePrefix)) { return(false); } var splittingMapping = Mappings.StarRatingRecommender.SplitToTrainTest( trainingOnlyUserFraction, testUserRatingTrainingFraction, coldUserFraction, coldItemFraction, ignoredUserFraction, ignoredItemFraction, removeOccasionalColdItems); var inputDataset = RecommenderDataset.Load(inputDatasetFile); var outputTrainingDataset = new RecommenderDataset( splittingMapping.GetInstances(SplitInstanceSource.Training(inputDataset)), inputDataset.StarRatingInfo); outputTrainingDataset.Save(outputTrainingDatasetFile); var outputTestDataset = new RecommenderDataset( splittingMapping.GetInstances(SplitInstanceSource.Test(inputDataset)), inputDataset.StarRatingInfo); outputTestDataset.Save(outputTestDatasetFile); return(true); }
/// <summary> /// Predictions based on 10-star rating input data and features /// </summary> /// <param name="traitsCounts"> Number of item traits </param> /// <returns> Metrics </returns> public MetricValues PredictionsOnDataWithFeatures(IList <int> traitsCounts) { var starRatingTrainTestSplittingMapping = RecommenderMappingFactory.GetStarsMapping(true); var binaryRatingTrainTestSplittingMapping = RecommenderMappingFactory.BinarizeMapping(starRatingTrainTestSplittingMapping); var trainSource = SplitInstanceSource.Training(RatingsPath); var testSource = SplitInstanceSource.Test(RatingsPath); var binaryRatingEvaluator = new RecommenderEvaluator <SplitInstanceSource <string>, string, Movie, int, int, IDictionary <int, double> >(binaryRatingTrainTestSplittingMapping.ForEvaluation()); var starsRatingEvaluator = new RecommenderEvaluator <SplitInstanceSource <string>, string, Movie, int, int, IDictionary <int, double> >(starRatingTrainTestSplittingMapping.ForEvaluation()); var correctFractions = new Dictionary <string, double>(); var ndcgs = new Dictionary <string, double>(); var maes = new Dictionary <string, double>(); foreach (var traitCount in traitsCounts) { Console.WriteLine($"Running metrics calculation for data with features and a model with {traitCount} traits."); Rand.Restart(RandomSeed); var recommender = GetRecommender(starRatingTrainTestSplittingMapping, traitCount); recommender.Settings.Training.UseItemFeatures = true; recommender.Settings.Training.UseSharedUserThresholds = true; recommender.Settings.Training.Advanced.UserThresholdPriorVariance = 10; recommender.Train(trainSource); var distribution = recommender.PredictDistribution(testSource); var binarizedPredictions = BinarizePredictions(distribution); var predictions = recommender.Predict(testSource); var correctFraction = 1.0 - binaryRatingEvaluator.RatingPredictionMetric(testSource, binarizedPredictions, Metrics.ZeroOneError); correctFractions.Add(traitCount.ToString(), correctFraction); var itemRecommendationsForEvaluation = starsRatingEvaluator.RecommendRatedItems(recommender, testSource, 5, 5); var ndcg = starsRatingEvaluator.ItemRecommendationMetric(testSource, itemRecommendationsForEvaluation, Metrics.Ndcg); ndcgs.Add(traitCount.ToString(), ndcg); var mae = starsRatingEvaluator.RatingPredictionMetric(testSource, predictions, Metrics.AbsoluteError); //Divide maes by 2 to convert 10-star rating to 5-star rating maes.Add(traitCount.ToString(), mae / 2.0); } return(new MetricValues(correctFractions, ndcgs, maes)); }
/// <summary> /// Takes test data from data source and represents it in the form of jagged array. /// The first dimension represents users, the second dimension represents movies. /// </summary> /// <param name="mapping">A mapping to convert ratings to a scale used exactly in the current experiment. </param> /// <returns></returns> public double[][] GetGroundTruth( IStarRatingRecommenderMapping <SplitInstanceSource <string>, RatingTriple, string, Movie, int, NoFeatureSource, Vector> mapping ) { Rand.Restart(RandomSeed); var testSource = SplitInstanceSource.Test(RatingsPath); var mappingForEvaluation = mapping.ForEvaluation(); var users = mappingForEvaluation.GetUsers(testSource); var ratings = users.Select(u => mappingForEvaluation.GetItemsRatedByUser(testSource, u) .Select(m => (double)mappingForEvaluation.GetRating(testSource, u, m))); var groundTruthArray = GetJaggedDoubles(ratings); return(groundTruthArray); }
/// <summary> /// Predictions based on like/dislike input data /// </summary> /// <param name="traitsCounts"> Number of item traits </param> /// <returns>A tuple of probability of like and metrics </returns> public (Dictionary <string, double[][]> likeProbability, MetricValues metricValues) PredictionsOnBinaryData( IList <int> traitsCounts ) { var starRatingTrainTestSplittingMapping = RecommenderMappingFactory.GetStarsMapping(true); var binaryRatingTrainTestSplittingMapping = RecommenderMappingFactory.BinarizeMapping(starRatingTrainTestSplittingMapping); var trainSource = SplitInstanceSource.Training(RatingsPath); var testSource = SplitInstanceSource.Test(RatingsPath); var binaryRatingEvaluator = new RecommenderEvaluator <SplitInstanceSource <string>, string, Movie, int, int, IDictionary <int, double> >(binaryRatingTrainTestSplittingMapping.ForEvaluation()); var starsRatingEvaluator = new RecommenderEvaluator <SplitInstanceSource <string>, string, Movie, int, int, IDictionary <int, double> >(starRatingTrainTestSplittingMapping.ForEvaluation()); var correctFractions = new Dictionary <string, double>(); var ndcgs = new Dictionary <string, double>(); var likeProbability = new Dictionary <string, double[][]>(); foreach (var traitCount in traitsCounts) { Console.WriteLine($"Running metrics calculation for binarized data and a model with {traitCount} traits."); Rand.Restart(RandomSeed); var recommender = GetRecommender(binaryRatingTrainTestSplittingMapping, traitCount); recommender.Settings.Training.Advanced.UserThresholdPriorVariance = EpsilonPriorVariance; recommender.Train(trainSource); var predictions = recommender.Predict(testSource); likeProbability.Add(traitCount.ToString(), GetLikeProbability(recommender.PredictDistribution(testSource))); var correctFraction = 1.0 - binaryRatingEvaluator.RatingPredictionMetric(testSource, predictions, Metrics.ZeroOneError); correctFractions.Add(traitCount.ToString(), correctFraction); var itemRecommendationsForEvaluation = starsRatingEvaluator.RecommendRatedItems(recommender, testSource, 5, 5); var ndcg = starsRatingEvaluator.ItemRecommendationMetric(testSource, itemRecommendationsForEvaluation, Metrics.Ndcg); ndcgs.Add(traitCount.ToString(), ndcg); } return(likeProbability, new MetricValues(correctFractions, ndcgs)); }
/// <summary> /// Generates a random dataset of the specified size, splits it as requested and checks the correctness of the resulting split. /// </summary> /// <param name="userCount">The number of users in the dataset.</param> /// <param name="itemCount">The number of items in the dataset.</param> /// <param name="sparsity">The probability of a random item to be rated by a random user.</param> /// <param name="trainingOnlyUserFraction">The fraction of users presented only in the training set.</param> /// <param name="testUserTrainingRatingFraction">The fraction of ratings in the training set for each user who is presented in both sets.</param> /// <param name="coldUserFraction">The fraction of users presented only in test set.</param> /// <param name="coldItemFraction">The fraction of items presented only in test set.</param> /// <param name="ignoredUserFraction">The fraction of users not presented in any of the sets.</param> /// <param name="ignoredItemFraction">The fraction of items not presented in any of the sets.</param> /// <param name="removeOccasionalColdItems">Specifies whether the occasionally produced cold items should be removed from the test set.</param> /// <returns>A triple containing the generated dataset, the training subset, and the test subset.</returns> private static Tuple <Dataset, Dataset, Dataset> TestSplittingHelper( int userCount, int itemCount, double sparsity, double trainingOnlyUserFraction, double testUserTrainingRatingFraction, double coldUserFraction, double coldItemFraction, double ignoredUserFraction, double ignoredItemFraction, bool removeOccasionalColdItems) { Dataset dataset = GenerateDataset(userCount, itemCount, sparsity); var mapping = new Mapping(); var splittingMapping = mapping.SplitToTrainTest( trainingOnlyUserFraction, testUserTrainingRatingFraction, coldUserFraction, coldItemFraction, ignoredUserFraction, ignoredItemFraction, removeOccasionalColdItems); Dataset trainingDataset = splittingMapping.GetInstances(SplitInstanceSource.Training(dataset)); Dataset testDataset = splittingMapping.GetInstances(SplitInstanceSource.Test(dataset)); CheckDatasetSplitCorrectness( dataset, trainingDataset, testDataset, coldUserFraction > 0, coldItemFraction > 0, ignoredUserFraction > 0, ignoredItemFraction > 0, removeOccasionalColdItems); return(Tuple.Create(dataset, trainingDataset, testDataset)); }
/// <summary> /// Executes the test for a given recommender and a test dataset. /// </summary> /// <param name="recommender">The recommender to evaluate.</param> /// <param name="evaluator">The evaluator.</param> /// <param name="testDataset">The test dataset.</param> /// <param name="predictionTime">When the method returns, this parameter contains the total prediction time.</param> /// <param name="evaluationTime">When the method returns, this parameter contains the total evaluation time.</param> /// <param name="metrics">When the method returns, this parameter contains names and values of the computed metrics.</param> public override void Execute( Recommender recommender, Evaluator evaluator, SplitInstanceSource <RecommenderDataset> testDataset, out TimeSpan predictionTime, out TimeSpan evaluationTime, out MetricValueDistributionCollection metrics) { var predictionTimer = Stopwatch.StartNew(); IDictionary <User, IEnumerable <Item> > recommendations = evaluator.RecommendRatedItems( recommender, testDataset, this.maxRecommendedItemCount, this.minRecommendationPoolSize); predictionTime = predictionTimer.Elapsed; var evaluationTimer = Stopwatch.StartNew(); metrics = new MetricValueDistributionCollection(); Func <int, double> ratingToGainConverter = r => r - testDataset.InstanceSource.StarRatingInfo.MinStarRating + 1; // Prevent non-positive gains metrics.Add("Recommendation NDCG", new MetricValueDistribution(evaluator.ItemRecommendationMetric(testDataset, recommendations, Metrics.Ndcg, ratingToGainConverter))); metrics.Add("Recommendation GAP", new MetricValueDistribution(evaluator.ItemRecommendationMetric(testDataset, recommendations, Metrics.GradedAveragePrecision, ratingToGainConverter))); evaluationTime = evaluationTimer.Elapsed; }
/// <summary> /// Predictions based on 10-star rating input data /// </summary> /// <param name="traitsCounts"> Number of item traits </param> /// <returns>A tuple of probability of thresholds posterior distributions, most probable ratings and metrics </returns> public (Dictionary <string, IDictionary <string, Gaussian> > posteriorDistributionsOfThresholds, Dictionary <string, double[][]> mostProbableRatings, MetricValues metricValues) PredictionsOnStarRatings( IList <int> traitsCounts ) { var starRatingTrainTestSplittingMapping = RecommenderMappingFactory.GetStarsMapping(true); var binaryRatingTrainTestSplittingMapping = RecommenderMappingFactory.BinarizeMapping(starRatingTrainTestSplittingMapping); var trainSource = SplitInstanceSource.Training(RatingsPath); var testSource = SplitInstanceSource.Test(RatingsPath); var binaryRatingEvaluator = new RecommenderEvaluator <SplitInstanceSource <string>, string, Movie, int, int, IDictionary <int, double> >(binaryRatingTrainTestSplittingMapping.ForEvaluation()); var starsRatingEvaluator = new RecommenderEvaluator <SplitInstanceSource <string>, string, Movie, int, int, IDictionary <int, double> >(starRatingTrainTestSplittingMapping.ForEvaluation()); var correctFractions = new Dictionary <string, double>(); var ndcgs = new Dictionary <string, double>(); var maes = new Dictionary <string, double>(); var mostProbableRatings = new Dictionary <string, double[][]>(); var posteriorDistributionsOfThresholds = new Dictionary <string, IDictionary <string, Gaussian> >(); foreach (var traitCount in traitsCounts) { Console.WriteLine($"Running metrics calculation for 10-star data and a model with {traitCount} traits."); Rand.Restart(RandomSeed); var recommender = GetRecommender(starRatingTrainTestSplittingMapping, traitCount); recommender.Settings.Training.UseSharedUserThresholds = true; recommender.Settings.Training.Advanced.UserThresholdPriorVariance = 10; recommender.Train(trainSource); var distributions = recommender.PredictDistribution(testSource); var predictions = recommender.Predict(testSource); mostProbableRatings.Add(traitCount.ToString(), GetJaggedDoubles(predictions.Select(userRating => userRating.Value.Select(movieRating => (double)movieRating.Value)))); var posteriorDistributionOfThresholds = recommender.GetPosteriorDistributions().Users.First().Value.Thresholds.ToList(); var posteriorDistributionOfThresholdsDict = BeautifyPosteriorDistribution(posteriorDistributionOfThresholds); posteriorDistributionsOfThresholds.Add(traitCount.ToString(), posteriorDistributionOfThresholdsDict); var binarizedPredictions = BinarizePredictions(distributions); var correctFraction = 1.0 - binaryRatingEvaluator.RatingPredictionMetric(testSource, binarizedPredictions, Metrics.ZeroOneError); correctFractions.Add(traitCount.ToString(), correctFraction); var itemRecommendationsForEvaluation = starsRatingEvaluator.RecommendRatedItems(recommender, testSource, 5, 5); var ndcg = starsRatingEvaluator.ItemRecommendationMetric(testSource, itemRecommendationsForEvaluation, Metrics.Ndcg); ndcgs.Add(traitCount.ToString(), ndcg); var mae = starsRatingEvaluator.RatingPredictionMetric(testSource, predictions, Metrics.AbsoluteError); //Divide maes by 2 to convert 10-star rating to 5-star rating maes.Add(traitCount.ToString(), mae / 2.0); } return(posteriorDistributionsOfThresholds, mostProbableRatings, new MetricValues(correctFractions, ndcgs, maes)); }
/// <summary> /// Executes the test for a given recommender under a specified name. /// </summary> public void Execute() { // Report that the run has been started if (this.Started != null) { this.Started(this, EventArgs.Empty); } try { Rand.Restart(1984); // Run should produce the same results every time TimeSpan totalTrainingTime = TimeSpan.Zero; TimeSpan totalPredictionTime = TimeSpan.Zero; TimeSpan totalEvaluationTime = TimeSpan.Zero; Stopwatch totalTimer = Stopwatch.StartNew(); MetricValueDistributionCollection metrics = null; for (int i = 0; i < this.FoldCount; ++i) { // Start timer measuring total time spent on this fold Stopwatch totalFoldTimer = Stopwatch.StartNew(); SplittingMapping splittingMapping = this.SplittingMappingFactory(); Recommender recommender = this.RecommenderFactory(splittingMapping); Evaluator evaluator = new Evaluator(new EvaluatorMapping(splittingMapping)); // Train the recommender Stopwatch foldTrainingTimer = Stopwatch.StartNew(); recommender.Train(SplitInstanceSource.Training(this.RecommenderDataset)); TimeSpan foldTrainingTime = foldTrainingTimer.Elapsed; // Run each test on the trained recommender var foldMetrics = new MetricValueDistributionCollection(); TimeSpan foldPredictionTime = TimeSpan.Zero; TimeSpan foldEvaluationTime = TimeSpan.Zero; foreach (RecommenderTest test in this.Tests) { // Perform the test TimeSpan testPredictionTime, testEvaluationTime; MetricValueDistributionCollection testMetrics; test.Execute( recommender, evaluator, SplitInstanceSource.Test(this.RecommenderDataset), out testPredictionTime, out testEvaluationTime, out testMetrics); // Merge the timings and the metrics foldPredictionTime += testPredictionTime; foldEvaluationTime += testEvaluationTime; foldMetrics.SetToUnionWith(testMetrics); } // Stop timer measuring total time spent on this fold TimeSpan totalFoldTime = totalFoldTimer.Elapsed; // Report that the fold has been processed if (this.FoldProcessed != null) { this.FoldProcessed( this, new RecommenderRunFoldProcessedEventArgs(i, totalFoldTime, foldTrainingTime, foldPredictionTime, foldEvaluationTime, foldMetrics)); } // Merge the timings totalTrainingTime += foldTrainingTime; totalPredictionTime += foldPredictionTime; totalEvaluationTime += foldEvaluationTime; // Merge the metrics if (metrics == null) { metrics = foldMetrics; } else { metrics.MergeWith(foldMetrics); } } // Report that the run has been completed TimeSpan totalTime = totalTimer.Elapsed; if (this.Completed != null) { this.Completed( this, new RecommenderRunCompletedEventArgs(totalTime, totalTrainingTime, totalPredictionTime, totalEvaluationTime, metrics)); } } catch (Exception e) { if (this.Interrupted != null) { this.Interrupted(this, new RecommenderRunInterruptedEventArgs(e)); } } }
/// <summary> /// Provides the object describing how ratings provided by the instance source map to stars by delegating the call to the wrapped mapping. /// </summary> /// <param name="instanceSource">The instance source.</param> /// <returns>The object describing how ratings provided by the instance source map to stars.</returns> public IStarRatingInfo <TRating> GetRatingInfo(SplitInstanceSource <TInstanceSource> instanceSource) { return(this.mapping.GetRatingInfo(instanceSource.InstanceSource)); }