/// <summary>
/// Runs the module.
/// </summary>
/// <param name="args">The command line arguments for the module.</param>
/// <param name="usagePrefix">The prefix to print before the usage string.</param>
/// <returns>True if the run was successful, false otherwise.</returns>
public override bool Run(string[] args, string usagePrefix)
{
string testDatasetFile = string.Empty;
string predictionsFile = string.Empty;
string reportFile = string.Empty;
var parser = new CommandLineParser();
parser.RegisterParameterHandler("--test-data", "FILE", "Test dataset used to obtain ground truth", v => testDatasetFile = v, CommandLineParameterType.Required);
parser.RegisterParameterHandler("--predictions", "FILE", "Predictions to evaluate", v => predictionsFile = v, CommandLineParameterType.Required);
parser.RegisterParameterHandler("--report", "FILE", "Evaluation report file", v => reportFile = v, CommandLineParameterType.Required);
if (!parser.TryParse(args, usagePrefix))
{
return(false);
}
RecommenderDataset testDataset = RecommenderDataset.Load(testDatasetFile);
IDictionary <User, IDictionary <Item, int> > ratingPredictions = RecommenderPersistenceUtils.LoadPredictedRatings(predictionsFile);
var evaluatorMapping = Mappings.StarRatingRecommender.ForEvaluation();
var evaluator = new StarRatingRecommenderEvaluator <RecommenderDataset, User, Item, int>(evaluatorMapping);
using (var writer = new StreamWriter(reportFile))
{
writer.WriteLine(
"Mean absolute error: {0:0.000}",
evaluator.RatingPredictionMetric(testDataset, ratingPredictions, Metrics.AbsoluteError));
writer.WriteLine(
"Root mean squared error: {0:0.000}",
Math.Sqrt(evaluator.RatingPredictionMetric(testDataset, ratingPredictions, Metrics.SquaredError)));
}
return(true);
}
/// <summary>
/// Prepares the environment (dataset, predictions, evaluation engine etc) before each test.
/// </summary>
public RecommenderEvaluatorTests()
{
this.dataset = new[]
{
// user, item, predicted rating, prediction distribution, true rating (data domain)
Tuple.Create <string, string, int, RatingDistribution, double>(
"A", "a", 4, new SortedDictionary <int, double> {
{ 0, 0.0 }, { 1, 0.2 }, { 2, 0.0 }, { 3, 0.0 }, { 4, 0.6 }, { 5, 0.2 }
}, 1.1),
Tuple.Create <string, string, int, RatingDistribution, double>(
"B", "a", 1, new SortedDictionary <int, double> {
{ 0, 0.0 }, { 1, 0.5 }, { 2, 0.0 }, { 3, 0.0 }, { 4, 0.5 }, { 5, 0.0 }
}, 4.1),
Tuple.Create <string, string, int, RatingDistribution, double>(
"D", "b", 2, new SortedDictionary <int, double> {
{ 0, 0.0 }, { 1, 0.0 }, { 2, 1.0 }, { 3, 0.0 }, { 4, 0.0 }, { 5, 0.0 }
}, 1.9),
Tuple.Create <string, string, int, RatingDistribution, double>(
"A", "b", 5, new SortedDictionary <int, double> {
{ 0, 0.0 }, { 1, 0.0 }, { 2, 0.0 }, { 3, 0.0 }, { 4, 0.2 }, { 5, 0.8 }
}, 5.3),
Tuple.Create <string, string, int, RatingDistribution, double>(
"A", "c", 3, new SortedDictionary <int, double> {
{ 0, 0.0 }, { 1, 0.0 }, { 2, 0.0 }, { 3, 0.6 }, { 4, 0.2 }, { 5, 0.2 }
}, 4.7),
Tuple.Create <string, string, int, RatingDistribution, double>(
"A", "e", 1, new SortedDictionary <int, double> {
{ 0, 0.0 }, { 1, 0.5 }, { 2, 0.0 }, { 3, 0.0 }, { 4, 0.5 }, { 5, 0.0 }
}, 3.6),
Tuple.Create <string, string, int, RatingDistribution, double>(
"B", "c", 2, new SortedDictionary <int, double> {
{ 0, 0.0 }, { 1, 0.0 }, { 2, 0.8 }, { 3, 0.2 }, { 4, 0.0 }, { 5, 0.0 }
}, 3.1)
};
var recommenderMapping = new StarRatingRecommenderMapping();
var evaluatorMapping = recommenderMapping.ForEvaluation();
this.starRatingInfo = recommenderMapping.GetRatingInfo(null);
this.evaluator = new StarRatingRecommenderEvaluator <IEnumerable <Instance>, string, string, double>(evaluatorMapping);
this.predictions = BuildPredictionDictionary(this.dataset, i => i.Item3);
this.uncertainPredictions = BuildPredictionDictionary(this.dataset, i => i.Item4);
}
/// <summary>
/// Runs the module.
/// </summary>
/// <param name="args">The command line arguments for the module.</param>
/// <param name="usagePrefix">The prefix to print before the usage string.</param>
/// <returns>True if the run was successful, false otherwise.</returns>
public override bool Run(string[] args, string usagePrefix)
{
string testDatasetFile = string.Empty;
string predictionsFile = string.Empty;
string reportFile = string.Empty;
var parser = new CommandLineParser();
parser.RegisterParameterHandler("--test-data", "FILE", "Test dataset used to obtain ground truth", v => testDatasetFile = v, CommandLineParameterType.Required);
parser.RegisterParameterHandler("--predictions", "FILE", "Predictions to evaluate", v => predictionsFile = v, CommandLineParameterType.Required);
parser.RegisterParameterHandler("--report", "FILE", "Evaluation report file", v => reportFile = v, CommandLineParameterType.Required);
if (!parser.TryParse(args, usagePrefix))
{
return(false);
}
RecommenderDataset testDataset = RecommenderDataset.Load(testDatasetFile);
int minRating = Mappings.StarRatingRecommender.GetRatingInfo(testDataset).MinStarRating;
IDictionary <User, IEnumerable <Item> > recommendedItems = RecommenderPersistenceUtils.LoadRecommendedItems(predictionsFile);
var evaluatorMapping = Mappings.StarRatingRecommender.ForEvaluation();
var evaluator = new StarRatingRecommenderEvaluator <RecommenderDataset, User, Item, int>(evaluatorMapping);
using (var writer = new StreamWriter(reportFile))
{
writer.WriteLine(
"NDCG: {0:0.000}",
evaluator.ItemRecommendationMetric(
testDataset,
recommendedItems,
Metrics.Ndcg,
rating => Convert.ToDouble(rating) - minRating + 1));
}
return(true);
}
/// <summary>
/// Runs the module.
/// </summary>
/// <param name="args">The command line arguments for the module.</param>
/// <param name="usagePrefix">The prefix to print before the usage string.</param>
/// <returns>True if the run was successful, false otherwise.</returns>
public override bool Run(string[] args, string usagePrefix)
{
string testDatasetFile = string.Empty;
string predictionsFile = string.Empty;
string reportFile = string.Empty;
int minCommonRatingCount = 5;
var parser = new CommandLineParser();
parser.RegisterParameterHandler("--test-data", "FILE", "Test dataset used to obtain ground truth", v => testDatasetFile = v, CommandLineParameterType.Required);
parser.RegisterParameterHandler("--predictions", "FILE", "Predictions to evaluate", v => predictionsFile = v, CommandLineParameterType.Required);
parser.RegisterParameterHandler("--report", "FILE", "Evaluation report file", v => reportFile = v, CommandLineParameterType.Required);
parser.RegisterParameterHandler("--min-common-items", "NUM", "Minimum number of users that the query item and the related item should have been rated by in common; defaults to 5", v => minCommonRatingCount = v, CommandLineParameterType.Optional);
if (!parser.TryParse(args, usagePrefix))
{
return(false);
}
RecommenderDataset testDataset = RecommenderDataset.Load(testDatasetFile);
IDictionary <Item, IEnumerable <Item> > relatedItems = RecommenderPersistenceUtils.LoadRelatedItems(predictionsFile);
var evaluatorMapping = Mappings.StarRatingRecommender.ForEvaluation();
var evaluator = new StarRatingRecommenderEvaluator <RecommenderDataset, User, Item, int>(evaluatorMapping);
using (var writer = new StreamWriter(reportFile))
{
writer.WriteLine(
"L1 Sim NDCG: {0:0.000}",
evaluator.RelatedItemsMetric(testDataset, relatedItems, minCommonRatingCount, Metrics.Ndcg, Metrics.NormalizedManhattanSimilarity));
writer.WriteLine(
"L2 Sim NDCG: {0:0.000}",
evaluator.RelatedItemsMetric(testDataset, relatedItems, minCommonRatingCount, Metrics.Ndcg, Metrics.NormalizedEuclideanSimilarity));
}
return(true);
}
