/// <summary> /// Runs the module. /// </summary> /// <param name="args">The command line arguments for the module.</param> /// <param name="usagePrefix">The prefix to print before the usage string.</param> /// <returns>True if the run was successful, false otherwise.</returns> public override bool Run(string[] args, string usagePrefix) { string testDatasetFile = string.Empty; string predictionsFile = string.Empty; string reportFile = string.Empty; var parser = new CommandLineParser(); parser.RegisterParameterHandler("--test-data", "FILE", "Test dataset used to obtain ground truth", v => testDatasetFile = v, CommandLineParameterType.Required); parser.RegisterParameterHandler("--predictions", "FILE", "Predictions to evaluate", v => predictionsFile = v, CommandLineParameterType.Required); parser.RegisterParameterHandler("--report", "FILE", "Evaluation report file", v => reportFile = v, CommandLineParameterType.Required); if (!parser.TryParse(args, usagePrefix)) { return(false); } RecommenderDataset testDataset = RecommenderDataset.Load(testDatasetFile); IDictionary <User, IDictionary <Item, int> > ratingPredictions = RecommenderPersistenceUtils.LoadPredictedRatings(predictionsFile); var evaluatorMapping = Mappings.StarRatingRecommender.ForEvaluation(); var evaluator = new StarRatingRecommenderEvaluator <RecommenderDataset, User, Item, int>(evaluatorMapping); using (var writer = new StreamWriter(reportFile)) { writer.WriteLine( "Mean absolute error: {0:0.000}", evaluator.RatingPredictionMetric(testDataset, ratingPredictions, Metrics.AbsoluteError)); writer.WriteLine( "Root mean squared error: {0:0.000}", Math.Sqrt(evaluator.RatingPredictionMetric(testDataset, ratingPredictions, Metrics.SquaredError))); } return(true); }
/// <summary> /// Prepares the environment (dataset, predictions, evaluation engine etc) before each test. /// </summary> public RecommenderEvaluatorTests() { this.dataset = new[] { // user, item, predicted rating, prediction distribution, true rating (data domain) Tuple.Create <string, string, int, RatingDistribution, double>( "A", "a", 4, new SortedDictionary <int, double> { { 0, 0.0 }, { 1, 0.2 }, { 2, 0.0 }, { 3, 0.0 }, { 4, 0.6 }, { 5, 0.2 } }, 1.1), Tuple.Create <string, string, int, RatingDistribution, double>( "B", "a", 1, new SortedDictionary <int, double> { { 0, 0.0 }, { 1, 0.5 }, { 2, 0.0 }, { 3, 0.0 }, { 4, 0.5 }, { 5, 0.0 } }, 4.1), Tuple.Create <string, string, int, RatingDistribution, double>( "D", "b", 2, new SortedDictionary <int, double> { { 0, 0.0 }, { 1, 0.0 }, { 2, 1.0 }, { 3, 0.0 }, { 4, 0.0 }, { 5, 0.0 } }, 1.9), Tuple.Create <string, string, int, RatingDistribution, double>( "A", "b", 5, new SortedDictionary <int, double> { { 0, 0.0 }, { 1, 0.0 }, { 2, 0.0 }, { 3, 0.0 }, { 4, 0.2 }, { 5, 0.8 } }, 5.3), Tuple.Create <string, string, int, RatingDistribution, double>( "A", "c", 3, new SortedDictionary <int, double> { { 0, 0.0 }, { 1, 0.0 }, { 2, 0.0 }, { 3, 0.6 }, { 4, 0.2 }, { 5, 0.2 } }, 4.7), Tuple.Create <string, string, int, RatingDistribution, double>( "A", "e", 1, new SortedDictionary <int, double> { { 0, 0.0 }, { 1, 0.5 }, { 2, 0.0 }, { 3, 0.0 }, { 4, 0.5 }, { 5, 0.0 } }, 3.6), Tuple.Create <string, string, int, RatingDistribution, double>( "B", "c", 2, new SortedDictionary <int, double> { { 0, 0.0 }, { 1, 0.0 }, { 2, 0.8 }, { 3, 0.2 }, { 4, 0.0 }, { 5, 0.0 } }, 3.1) }; var recommenderMapping = new StarRatingRecommenderMapping(); var evaluatorMapping = recommenderMapping.ForEvaluation(); this.starRatingInfo = recommenderMapping.GetRatingInfo(null); this.evaluator = new StarRatingRecommenderEvaluator <IEnumerable <Instance>, string, string, double>(evaluatorMapping); this.predictions = BuildPredictionDictionary(this.dataset, i => i.Item3); this.uncertainPredictions = BuildPredictionDictionary(this.dataset, i => i.Item4); }
/// <summary> /// Runs the module. /// </summary> /// <param name="args">The command line arguments for the module.</param> /// <param name="usagePrefix">The prefix to print before the usage string.</param> /// <returns>True if the run was successful, false otherwise.</returns> public override bool Run(string[] args, string usagePrefix) { string testDatasetFile = string.Empty; string predictionsFile = string.Empty; string reportFile = string.Empty; var parser = new CommandLineParser(); parser.RegisterParameterHandler("--test-data", "FILE", "Test dataset used to obtain ground truth", v => testDatasetFile = v, CommandLineParameterType.Required); parser.RegisterParameterHandler("--predictions", "FILE", "Predictions to evaluate", v => predictionsFile = v, CommandLineParameterType.Required); parser.RegisterParameterHandler("--report", "FILE", "Evaluation report file", v => reportFile = v, CommandLineParameterType.Required); if (!parser.TryParse(args, usagePrefix)) { return(false); } RecommenderDataset testDataset = RecommenderDataset.Load(testDatasetFile); int minRating = Mappings.StarRatingRecommender.GetRatingInfo(testDataset).MinStarRating; IDictionary <User, IEnumerable <Item> > recommendedItems = RecommenderPersistenceUtils.LoadRecommendedItems(predictionsFile); var evaluatorMapping = Mappings.StarRatingRecommender.ForEvaluation(); var evaluator = new StarRatingRecommenderEvaluator <RecommenderDataset, User, Item, int>(evaluatorMapping); using (var writer = new StreamWriter(reportFile)) { writer.WriteLine( "NDCG: {0:0.000}", evaluator.ItemRecommendationMetric( testDataset, recommendedItems, Metrics.Ndcg, rating => Convert.ToDouble(rating) - minRating + 1)); } return(true); }
/// <summary> /// Runs the module. /// </summary> /// <param name="args">The command line arguments for the module.</param> /// <param name="usagePrefix">The prefix to print before the usage string.</param> /// <returns>True if the run was successful, false otherwise.</returns> public override bool Run(string[] args, string usagePrefix) { string testDatasetFile = string.Empty; string predictionsFile = string.Empty; string reportFile = string.Empty; int minCommonRatingCount = 5; var parser = new CommandLineParser(); parser.RegisterParameterHandler("--test-data", "FILE", "Test dataset used to obtain ground truth", v => testDatasetFile = v, CommandLineParameterType.Required); parser.RegisterParameterHandler("--predictions", "FILE", "Predictions to evaluate", v => predictionsFile = v, CommandLineParameterType.Required); parser.RegisterParameterHandler("--report", "FILE", "Evaluation report file", v => reportFile = v, CommandLineParameterType.Required); parser.RegisterParameterHandler("--min-common-items", "NUM", "Minimum number of users that the query item and the related item should have been rated by in common; defaults to 5", v => minCommonRatingCount = v, CommandLineParameterType.Optional); if (!parser.TryParse(args, usagePrefix)) { return(false); } RecommenderDataset testDataset = RecommenderDataset.Load(testDatasetFile); IDictionary <Item, IEnumerable <Item> > relatedItems = RecommenderPersistenceUtils.LoadRelatedItems(predictionsFile); var evaluatorMapping = Mappings.StarRatingRecommender.ForEvaluation(); var evaluator = new StarRatingRecommenderEvaluator <RecommenderDataset, User, Item, int>(evaluatorMapping); using (var writer = new StreamWriter(reportFile)) { writer.WriteLine( "L1 Sim NDCG: {0:0.000}", evaluator.RelatedItemsMetric(testDataset, relatedItems, minCommonRatingCount, Metrics.Ndcg, Metrics.NormalizedManhattanSimilarity)); writer.WriteLine( "L2 Sim NDCG: {0:0.000}", evaluator.RelatedItemsMetric(testDataset, relatedItems, minCommonRatingCount, Metrics.Ndcg, Metrics.NormalizedEuclideanSimilarity)); } return(true); }