/// <summary>
/// Calculates absolute error between prediction and ground truth
/// </summary>
/// <param name="testSource">A source of ratings made by users</param>
/// <param name="mapping">A mapping to convert source ratings into experiment specific ones</param>
/// <param name="predictions">Ratings which are predicted by the recommender</param>
/// <returns></returns>
private static Dictionary <Movie, List <double> > PredictionError(
SplitInstanceSource <string> testSource,
IStarRatingRecommenderMapping <SplitInstanceSource <string>, RatingTriple, string, Movie, int, NoFeatureSource, Vector> mapping,
IDictionary <string, IDictionary <Movie, IDictionary <int, double> > > predictions)
{
var evMapping = mapping.ForEvaluation();
var allErrors = new Dictionary <Movie, List <double> >();
foreach (var userWithPredictionList in predictions)
{
foreach (var itemPrediction in userWithPredictionList.Value)
{
var prediction = itemPrediction.Value;
var groundTruth = evMapping.GetRating(testSource, userWithPredictionList.Key, itemPrediction.Key);
var predictedRating = prediction.Aggregate((l, r) => l.Value > r.Value ? l : r).Key;
var error = Math.Abs(groundTruth - predictedRating);
if (allErrors.TryGetValue(itemPrediction.Key, out var values))
{
values.Add(error);
}
else
{
allErrors.Add(itemPrediction.Key, new List <double>()
{
error
});
}
}
}
return(allErrors);
}
/// <summary>
/// Initializes a new instance of the <see cref="MahoutRecommender{TInstanceSource}"/> class. Downloads recommender if necessary.
/// </summary>
/// <param name="mapping">The mapping used to access the data.</param>
public static MahoutRecommender <TInstanceSource> Create(
IStarRatingRecommenderMapping <TInstanceSource, RatedUserItem, User, Item, int, DummyFeatureSource, Vector> mapping)
{
if (!File.Exists(PathToJar))
{
// getting zip
var zipName = "mahout.zip";
var tmpFile = Path.Combine(InstallationWorkDir, zipName);
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
new WebClient().DownloadFile(ReleaseUrl, tmpFile);
// extracting zip
using (var file = File.OpenRead(tmpFile))
{
var zip = new System.IO.Compression.ZipArchive(file);
var entry = zip.GetEntry(CoreJobJarPath);
using (Stream source = entry.Open(), dest = File.OpenWrite(PathToJar))
{
source.CopyTo(dest);
}
}
// deleting temp files
File.Delete(tmpFile);
}
return(new MahoutRecommender <TInstanceSource>(mapping));
}
/// <summary>
/// Creates an instance of the Matchbox-based recommender using the settings from this configuration element.
/// </summary>
/// <param name="mapping">The mapping for the recommender being created.</param>
/// <returns>An instance of the Matchbox-based recommender.</returns>
public override IRecommender <SplitInstanceSource <RecommenderDataset>, User, Item, int, RatingDistribution, DummyFeatureSource> Create(
IStarRatingRecommenderMapping <SplitInstanceSource <RecommenderDataset>, RatedUserItem, User, Item, int, DummyFeatureSource, Vector> mapping)
{
var inferNetRecommender = MatchboxRecommender.Create(mapping);
this.SetupMatchboxModel(inferNetRecommender.Settings);
return(inferNetRecommender);
}
/// <summary>
/// Initializes a new instance of the
/// <see cref="StandardDataFormatMatchboxRecommender{TInstanceSource,TInstance,TUser,TItem,TDataRating,TFeatureSource}"/>
/// class.
/// </summary>
/// <param name="topLevelMapping">The mapping used for accessing data.</param>
internal StandardDataFormatMatchboxRecommender(IStarRatingRecommenderMapping <TInstanceSource, TInstance, TUser, TItem, TDataRating, TFeatureSource, Vector> topLevelMapping)
{
this.topLevelMapping = topLevelMapping;
this.nativeMapping = new NativeRecommenderMapping(topLevelMapping);
this.recommender = new NativeDataFormatMatchboxRecommender <TInstanceSource, TFeatureSource>(this.nativeMapping);
this.indexedUserSet = new IndexedEntitySet <TUser>();
this.indexedItemSet = new IndexedEntitySet <TItem>();
}
/// <summary>
/// Initializes a new instance of the <see cref="MahoutRecommender{TInstanceSource}"/> class.
/// </summary>
/// <param name="mapping">The mapping used to access the data.</param>
private MahoutRecommender(
IStarRatingRecommenderMapping <TInstanceSource, RatedUserItem, User, Item, int, DummyFeatureSource, Vector> mapping)
{
Debug.Assert(mapping != null, "A valid mapping should be provided.");
this.mapping = mapping;
this.Settings = new MahoutRecommenderSettings();
}
Create <TInstanceSource, TInstance, TUser, TItem, TRating, TFeatureSource>(
IStarRatingRecommenderMapping <TInstanceSource, TInstance, TUser, TItem, TRating, TFeatureSource, Vector> mapping)
{
if (mapping == null)
{
throw new ArgumentNullException("mapping");
}
return(new StandardDataFormatMatchboxRecommender <TInstanceSource, TInstance, TUser, TItem, TRating, TFeatureSource>(mapping));
}
/// <summary>
/// Initializes a new instance of the
/// <see cref="RandomStarRatingRecommender{TInstanceSource,TInstance,TUser,TItem,TDataRating,TFeatureSource,TFeatureValues}"/> class.
/// </summary>
/// <param name="mapping">The mapping used to access the data.</param>
/// <exception cref="ArgumentNullException">Thrown if one of the required parameters is null.</exception>
public RandomStarRatingRecommender(
IStarRatingRecommenderMapping <TInstanceSource, TInstance, TUser, TItem, TDataRating, TFeatureSource, TFeatureValues> mapping)
{
if (mapping == null)
{
throw new ArgumentNullException(nameof(mapping));
}
this.capabilities = new RandomStarRatingRecommenderCapabilities();
this.mapping = mapping;
}
LoadBackwardCompatible <TInstanceSource, TInstance, TUser, TItem, TRating, TFeatureSource>(
Stream stream, IStarRatingRecommenderMapping <TInstanceSource, TInstance, TUser, TItem, TRating, TFeatureSource, Vector> mapping)
{
if (stream == null)
{
throw new ArgumentNullException("stream");
}
using (var reader = new BinaryReader(stream, Encoding.UTF8, true))
{
return(LoadBackwardCompatible(reader, mapping));
}
}
LoadBackwardCompatible <TInstanceSource, TInstance, TUser, TItem, TRating, TFeatureSource>(
string fileName, IStarRatingRecommenderMapping <TInstanceSource, TInstance, TUser, TItem, TRating, TFeatureSource, Vector> mapping)
{
if (fileName == null)
{
throw new ArgumentNullException("fileName");
}
using (Stream stream = File.Open(fileName, FileMode.Open))
{
return(LoadBackwardCompatible(stream, mapping));
}
}
/// <summary>
/// Initializes a new instance of the
/// <see cref="TrainTestSplittingStarRatingRecommenderMapping{TInstanceSource,TInstance,TUser,TItem,TRating,TFeatureSource,TFeatureValues}"/> class.
/// </summary>
/// <param name="mapping">The wrapped recommender mapping.</param>
/// <param name="trainingOnlyUserFraction">The fraction of users included in the training set only.</param>
/// <param name="testUserRatingTrainingFraction">The fraction of ratings in the training set for each user who is presented in both sets.</param>
/// <param name="coldUserFraction">The fraction of users included in the test set only.</param>
/// <param name="coldItemFraction">The fraction of items included in the test set only.</param>
/// <param name="ignoredUserFraction">The fraction of users not included in either the training or the test set.</param>
/// <param name="ignoredItemFraction">The fraction of items not included in either the training or the test set.</param>
/// <param name="removeOccasionalColdItems">Whether the occasionally produced cold items should be removed from the test set.</param>
public TrainTestSplittingStarRatingRecommenderMapping(
IStarRatingRecommenderMapping <TInstanceSource, TInstance, TUser, TItem, TRating, TFeatureSource, TFeatureValues> mapping,
double trainingOnlyUserFraction,
double testUserRatingTrainingFraction,
double coldUserFraction = 0,
double coldItemFraction = 0,
double ignoredUserFraction = 0,
double ignoredItemFraction = 0,
bool removeOccasionalColdItems = false)
: base(mapping, trainingOnlyUserFraction, testUserRatingTrainingFraction, coldUserFraction, coldItemFraction, ignoredUserFraction, ignoredItemFraction, removeOccasionalColdItems)
{
this.mapping = mapping;
}
SplitToTrainTest <TInstanceSource, TInstance, TUser, TItem, TRating, TFeatureSource, TFeatureValues>(
this IStarRatingRecommenderMapping <TInstanceSource, TInstance, TUser, TItem, TRating, TFeatureSource, TFeatureValues> mapping,
double trainingOnlyUserFraction,
double testUserRatingTrainingFraction,
double coldUserFraction = 0,
double coldItemFraction = 0,
double ignoredUserFraction = 0,
double ignoredItemFraction = 0,
bool removeOccasionalColdItems = false)
{
return(new TrainTestSplittingStarRatingRecommenderMapping <TInstanceSource, TInstance, TUser, TItem, TRating, TFeatureSource, TFeatureValues>(
mapping, trainingOnlyUserFraction, testUserRatingTrainingFraction, coldUserFraction, coldItemFraction, ignoredUserFraction, ignoredItemFraction, removeOccasionalColdItems));
}
/// <summary>
/// Creates an instance of the Mahout recommender using the settings from this configuration element.
/// </summary>
/// <param name="mapping">The mapping for the recommender being created.</param>
/// <returns>An instance of the Mahout recommender.</returns>
public override IRecommender <SplitInstanceSource <RecommenderDataset>, User, Item, int, RatingDistribution, DummyFeatureSource> Create(
IStarRatingRecommenderMapping <SplitInstanceSource <RecommenderDataset>, RatedUserItem, User, Item, int, DummyFeatureSource, Vector> mapping)
{
var recommender = MahoutRecommender <SplitInstanceSource <RecommenderDataset> > .Create(mapping);
recommender.Settings.RatingSimilarity = this.RatingSimilarity.Value;
recommender.Settings.RatingPredictionAlgorithm = this.RatingPredictionAlgorithm.Value;
recommender.Settings.MissingRatingPredictionAlgorithm = this.MissingRatingPredictionAlgorithm.Value;
recommender.Settings.UserNeighborhoodSize = this.UserNeighborhoodSize.Value;
recommender.Settings.TraitCount = this.TraitCount.Value;
recommender.Settings.IterationCount = this.IterationCount.Value;
return(recommender);
}
LoadBackwardCompatible <TInstanceSource, TInstance, TUser, TItem, TRating, TFeatureSource>(
BinaryReader reader, IStarRatingRecommenderMapping <TInstanceSource, TInstance, TUser, TItem, TRating, TFeatureSource, Vector> mapping)
{
if (reader == null)
{
throw new ArgumentNullException("reader");
}
if (mapping == null)
{
throw new ArgumentNullException("mapping");
}
return(new StandardDataFormatMatchboxRecommender <TInstanceSource, TInstance, TUser, TItem, TRating, TFeatureSource>(reader, mapping));
}
/// <summary>
/// Takes test data from data source and represents it in the form of jagged array.
/// The first dimension represents users, the second dimension represents movies.
/// </summary>
/// <param name="mapping">A mapping to convert ratings to a scale used exactly in the current experiment. </param>
/// <returns></returns>
public double[][] GetGroundTruth(
IStarRatingRecommenderMapping <SplitInstanceSource <string>, RatingTriple, string, Movie, int, NoFeatureSource, Vector> mapping
)
{
Rand.Restart(RandomSeed);
var testSource = SplitInstanceSource.Test(RatingsPath);
var mappingForEvaluation = mapping.ForEvaluation();
var users = mappingForEvaluation.GetUsers(testSource);
var ratings = users.Select(u =>
mappingForEvaluation.GetItemsRatedByUser(testSource, u)
.Select(m => (double)mappingForEvaluation.GetRating(testSource, u, m)));
var groundTruthArray = GetJaggedDoubles(ratings);
return(groundTruthArray);
}
/// <summary>
/// Creates an instance of the VW recommender using the settings from this configuration element.
/// </summary>
/// <param name="mapping">The mapping for the recommender being created.</param>
/// <returns>An instance of the VW recommender.</returns>
public override IRecommender <SplitInstanceSource <RecommenderDataset>, User, Item, int, RatingDistribution, DummyFeatureSource> Create(
IStarRatingRecommenderMapping <SplitInstanceSource <RecommenderDataset>, RatedUserItem, User, Item, int, DummyFeatureSource, Vector> mapping)
{
var recommender = VowpalWabbitRecommender <SplitInstanceSource <RecommenderDataset> > .Create(mapping);
recommender.Settings.TraitCount = this.TraitCount.Value;
recommender.Settings.BitPrecision = this.BitPrecision.Value;
recommender.Settings.LearningRate = this.LearningRate.Value;
recommender.Settings.LearningRateDecay = this.LearningRateDecay.Value;
recommender.Settings.PassCount = this.PassCount.Value;
recommender.Settings.L1Regularization = this.L1Regularization.Value;
recommender.Settings.L2Regularization = this.L2Regularization.Value;
recommender.Settings.UseUserFeatures = this.UseUserFeatures.Value;
recommender.Settings.UseItemFeatures = this.UseItemFeatures.Value;
return(recommender);
}
GetRecommender(
IStarRatingRecommenderMapping <SplitInstanceSource <string>, RatingTriple, string, Movie, int, NoFeatureSource, Vector> dataMapping,
int traitCount)
{
// In this chapter Infer.NET Learner is used to make recommendations
// You can find its source code here:
// https://github.com/dotnet/infer/tree/master/src/Learners/Recommender
// This recommender has a model that matches the one described in the book.
// Its source code is situated here:
// https://github.com/dotnet/infer/blob/master/src/Learners/RecommenderModels/Models.cs
var recommender = MatchboxRecommender.Create(dataMapping);
if (traitCount == 0)
{
//workaround to make recommender work with 0 traits
var training = recommender.Settings.Training;
var countField = training.GetType().GetField("traitCount", BindingFlags.NonPublic | BindingFlags.Instance);
countField.SetValue(training, 0);
}
else
{
recommender.Settings.Training.TraitCount = traitCount;
}
// This value allows to normalize the sum of trait affinity variances to 1.0.
var traitVariance = 1.0 / Math.Sqrt(traitCount);
recommender.Settings.Training.IterationCount = IterationCount;
recommender.Settings.Training.BatchCount = 1;
recommender.Settings.Training.UseUserFeatures = false;
recommender.Settings.Training.UseItemFeatures = false;
recommender.Settings.Training.Advanced.AffinityNoiseVariance = 1.0;
recommender.Settings.Training.Advanced.UserThresholdNoiseVariance = EpsilonPriorVariance;
recommender.Settings.Training.Advanced.ItemTraitVariance = traitVariance;
recommender.Settings.Training.Advanced.ItemBiasVariance = 1.0;
recommender.Settings.Training.Advanced.UserTraitVariance = traitVariance;
recommender.Settings.Training.Advanced.UserBiasVariance = 1.0;
recommender.Settings.Prediction.SetPredictionLossFunction(LossFunction.ZeroOne);
return(recommender);
}
/// <summary>
/// Initializes a new instance of the <see cref="NativeRecommenderMapping"/> class.
/// </summary>
/// <param name="topLevelMapping">The wrapped mapping for accessing data in standard format.</param>
public NativeRecommenderMapping(
IStarRatingRecommenderMapping <TInstanceSource, TInstance, TUser, TItem, TDataRating, TFeatureSource, Vector> topLevelMapping)
{
this.topLevelMapping = topLevelMapping;
}
/// <summary>
/// Converts a 10-star rating mapping to a binary (like/dislike) one. It allows to save order of mapping output.
/// </summary>
/// <param name="mapping"></param>
/// <returns></returns>
public RatingBinarizingMapping <SplitInstanceSource <string>, Movie> BinarizeMapping(IStarRatingRecommenderMapping <SplitInstanceSource <string>, RatingTriple, string, Movie, int, NoFeatureSource, Vector> mapping)
{
return(new RatingBinarizingMapping <SplitInstanceSource <string>, Movie>(mapping));
}
/// <summary>
/// Creates an instance of the recommender described by this configuration element.
/// </summary>
/// <param name="mapping">The mapping for the recommender being created.</param>
/// <returns>The created recommender instance.</returns>
public abstract IRecommender <SplitInstanceSource <RecommenderDataset>, User, Item, int, RatingDistribution, DummyFeatureSource> Create(
IStarRatingRecommenderMapping <SplitInstanceSource <RecommenderDataset>, RatedUserItem, User, Item, int, DummyFeatureSource, Vector> mapping);
/// <summary>
/// Initializes a new instance of the
/// <see cref="StarRatingRecommenderEvaluatorMapping{TInstanceSource,TInstance,TUser,TItem,TRating,TFeatureSource,TFeatureValues}"/> class.
/// </summary>
/// <param name="mapping">The recommender mapping.</param>
public StarRatingRecommenderEvaluatorMapping(
IStarRatingRecommenderMapping <TInstanceSource, TInstance, TUser, TItem, TRating, TFeatureSource, TFeatureValues> mapping)
: base(mapping)
{
this.mapping = mapping;
}
/// <summary>
/// A mapping constructor
/// </summary>
/// <param name="mapping">A 10-star rating which will be converted to like/dislike rating mapping. </param>
public RatingBinarizingMapping(IStarRatingRecommenderMapping <TInstanceSource, RatingTriple, string, TItem, int, NoFeatureSource, Vector> mapping)
{
this.mapping = mapping;
}
/// <summary>
/// Creates an instance of the random recommender.
/// </summary>
/// <param name="mapping">The mapping for the recommender being created.</param>
/// <returns>An instance of the random recommender.</returns>
public override IRecommender <SplitInstanceSource <RecommenderDataset>, User, Item, int, RatingDistribution, DummyFeatureSource> Create(
IStarRatingRecommenderMapping <SplitInstanceSource <RecommenderDataset>, RatedUserItem, User, Item, int, DummyFeatureSource, Vector> mapping)
{
return(new RandomStarRatingRecommender <SplitInstanceSource <RecommenderDataset>, RatedUserItem, User, Item, int, DummyFeatureSource, Vector>(mapping));
}
ForEvaluation <TInstanceSource, TInstance, TUser, TItem, TRating, TFeatureSource, TFeatureValues>(
this IStarRatingRecommenderMapping <TInstanceSource, TInstance, TUser, TItem, TRating, TFeatureSource, TFeatureValues> mapping)
{
return(new StarRatingRecommenderEvaluatorMapping <TInstanceSource, TInstance, TUser, TItem, TRating, TFeatureSource, TFeatureValues>(mapping));
}
