static void Main(string[] args)
{
TopNEvaluator evaluator = new TopNEvaluator();
var recommendersUnderTest = new List<ITrainableRecSource<MalTrainingData, MalUserListEntries, IEnumerable<IRecommendation>, IRecommendation>>();
List<List<EvaluationResults>> resultsForEachRecommender = new List<List<EvaluationResults>>();
const int minEpisodesToCountIncomplete = 26;
const double targetPercentile = 0.25;
//var averageScoreRecSourceWithoutDropped = new MalAverageScoreRecSource(minEpisodesToCountIncomplete, useDropped: false, minUsersToCountAnime: 50);
//var mostPopularRecSourceWithoutDropped = new MalMostPopularRecSource(minEpisodesToCountIncomplete, useDropped: false);
//var defaultBiasedMatrixFactorizationRecSource = new MalMyMediaLiteRatingPredictionRecSource<BiasedMatrixFactorization>
// (new BiasedMatrixFactorization(), minEpisodesToCountIncomplete, useDropped: true, minUsersToCountAnime: 50);
//var biasedMatrixFactorizationRecSourceWithBoldDriver = new MalMyMediaLiteRatingPredictionRecSource<BiasedMatrixFactorization>
// (new BiasedMatrixFactorization() { BoldDriver = true }, minEpisodesToCountIncomplete, useDropped: true, minUsersToCountAnime: 50);
//var biasedMatrixFactorizationRecSourceWithFactors = new MalMyMediaLiteRatingPredictionRecSource<BiasedMatrixFactorization>
// (new BiasedMatrixFactorization() { BoldDriver = true, FrequencyRegularization = true, NumFactors = 50 },
// minEpisodesToCountIncomplete, useDropped: true, minUsersToCountAnime: 50);
//var biasedMatrixFactorizationRecSourceWithFactorsAndIters = new MalMyMediaLiteRatingPredictionRecSource<BiasedMatrixFactorization>
// (new BiasedMatrixFactorization() { BoldDriver = true, FrequencyRegularization = true, NumFactors = 50, NumIter = 50 },
// minEpisodesToCountIncomplete, useDropped: true, minUsersToCountAnime: 50);
//var defaultMatrixFactorizationRecSource = new MalMyMediaLiteRatingPredictionRecSource<MatrixFactorization>
// (new MatrixFactorization(), minEpisodesToCountIncomplete, useDropped: true, minUsersToCountAnime: 50);
//var animeRecsRecSource35 = new MalAnimeRecsRecSourceWithConstantPercentTarget(
// numRecommendersToUse: 100,
// fractionConsideredRecommended: 0.35,
// targetFraction: 0.35,
// minEpisodesToClassifyIncomplete: minEpisodesToCountIncomplete
//);
//var animeRecsRecSource25 = new MalAnimeRecsRecSourceWithConstantPercentTarget(
// numRecommendersToUse: 100,
// fractionConsideredRecommended: 0.25,
// targetFraction: 0.25,
// minEpisodesToClassifyIncomplete: minEpisodesToCountIncomplete
//);
var bprmfRecSource = new MalMyMediaLiteItemRecommenderRecSourceWithConstantPercentTarget<BPRMF>(
new BPRMF() { BiasReg = .01f },
fractionConsideredRecommended: 0.25,
minEpisodesToClassifyIncomplete: minEpisodesToCountIncomplete,
minUsersToCountAnime: 30,
targetFraction: 0.25
);
//recommendersUnderTest.Add(averageScoreRecSourceWithoutDropped);
//recommendersUnderTest.Add(mostPopularRecSourceWithoutDropped);
//recommendersUnderTest.Add(defaultBiasedMatrixFactorizationRecSource);
//recommendersUnderTest.Add(biasedMatrixFactorizationRecSourceWithBoldDriver);
//recommendersUnderTest.Add(biasedMatrixFactorizationRecSourceWithFactors);
//recommendersUnderTest.Add(biasedMatrixFactorizationRecSourceWithFactorsAndIters);
//recommendersUnderTest.Add(defaultMatrixFactorizationRecSource);
//recommendersUnderTest.Add(animeRecsRecSource35);
//recommendersUnderTest.Add(animeRecsRecSource25);
recommendersUnderTest.Add(bprmfRecSource);
for (int i = 0; i < recommendersUnderTest.Count; i++)
{
resultsForEachRecommender.Add(new List<EvaluationResults>());
}
IUserInputClassifier<MalUserListEntries> targetClassifier = new MalPercentageRatingClassifier(targetPercentile, minEpisodesToCountIncomplete);
MalTrainingData rawData;
string postgresConnectionString = ConfigurationManager.ConnectionStrings["Postgres"].ConnectionString;
using (PgMalDataLoader loader = new PgMalDataLoader(postgresConnectionString))
{
rawData = loader.LoadMalTrainingData();
}
const int numEvaluations = 5;
const int numRecsToGet = 25;
for (int pass = 0; pass < numEvaluations; pass++)
{
for (int recSourceIndex = 0; recSourceIndex < recommendersUnderTest.Count; recSourceIndex++)
{
ITrainableRecSource<MalTrainingData, MalUserListEntries, IEnumerable<IRecommendation>, IRecommendation> recSource = recommendersUnderTest[recSourceIndex];
Tuple<MalTrainingData, ICollection<MalUserListEntries>> dataForTrainingAndEvaluation = GetDataForTrainingAndEvaluation(rawData);
MalTrainingData trainingData = dataForTrainingAndEvaluation.Item1;
ICollection<MalUserListEntries> dataForEvaluation = dataForTrainingAndEvaluation.Item2;
recSource.Train(trainingData);
EvaluationResults results = evaluator.Evaluate(
recSource: recSource,
users: dataForEvaluation,
goodBadClassifier: targetClassifier,
inputDivisionFunc: MalUserListEntries.DivideClassifiedForInputAndEvaluation,
numRecsToTryToGet: numRecsToGet
);
resultsForEachRecommender[recSourceIndex].Add(results);
}
}
for(int recSourceIndex = 0; recSourceIndex < recommendersUnderTest.Count; recSourceIndex++)
{
ITrainableRecSource<MalTrainingData, MalUserListEntries, IEnumerable<IRecommendation>, IRecommendation> recSource = recommendersUnderTest[recSourceIndex];
Console.WriteLine(recSource);
foreach (EvaluationResults resultsForPass in resultsForEachRecommender[recSourceIndex])
{
Console.WriteLine("Precision: {0:P2}\tRecall: {1:P2}",
resultsForPass.Precision, resultsForPass.Recall);
}
Console.WriteLine();
Console.WriteLine();
Console.WriteLine();
Console.WriteLine();
}
}
static void Main(string[] args)
{
TopNEvaluator evaluator = new TopNEvaluator();
var recommendersUnderTest = new List <ITrainableRecSource <MalTrainingData, MalUserListEntries, IEnumerable <IRecommendation>, IRecommendation> >();
List <List <EvaluationResults> > resultsForEachRecommender = new List <List <EvaluationResults> >();
const int minEpisodesToCountIncomplete = 26;
const double targetPercentile = 0.25;
//var averageScoreRecSourceWithoutDropped = new MalAverageScoreRecSource(minEpisodesToCountIncomplete, useDropped: false, minUsersToCountAnime: 50);
//var mostPopularRecSourceWithoutDropped = new MalMostPopularRecSource(minEpisodesToCountIncomplete, useDropped: false);
//var defaultBiasedMatrixFactorizationRecSource = new MalMyMediaLiteRatingPredictionRecSource<BiasedMatrixFactorization>
// (new BiasedMatrixFactorization(), minEpisodesToCountIncomplete, useDropped: true, minUsersToCountAnime: 50);
//var biasedMatrixFactorizationRecSourceWithBoldDriver = new MalMyMediaLiteRatingPredictionRecSource<BiasedMatrixFactorization>
// (new BiasedMatrixFactorization() { BoldDriver = true }, minEpisodesToCountIncomplete, useDropped: true, minUsersToCountAnime: 50);
//var biasedMatrixFactorizationRecSourceWithFactors = new MalMyMediaLiteRatingPredictionRecSource<BiasedMatrixFactorization>
// (new BiasedMatrixFactorization() { BoldDriver = true, FrequencyRegularization = true, NumFactors = 50 },
// minEpisodesToCountIncomplete, useDropped: true, minUsersToCountAnime: 50);
//var biasedMatrixFactorizationRecSourceWithFactorsAndIters = new MalMyMediaLiteRatingPredictionRecSource<BiasedMatrixFactorization>
// (new BiasedMatrixFactorization() { BoldDriver = true, FrequencyRegularization = true, NumFactors = 50, NumIter = 50 },
// minEpisodesToCountIncomplete, useDropped: true, minUsersToCountAnime: 50);
//var defaultMatrixFactorizationRecSource = new MalMyMediaLiteRatingPredictionRecSource<MatrixFactorization>
// (new MatrixFactorization(), minEpisodesToCountIncomplete, useDropped: true, minUsersToCountAnime: 50);
var animeRecsRecSource35 = new MalAnimeRecsRecSourceWithConstantPercentTarget(
numRecommendersToUse: 100,
fractionConsideredRecommended: 0.35,
targetFraction: 0.35,
minEpisodesToClassifyIncomplete: minEpisodesToCountIncomplete
);
//var animeRecsRecSource25 = new MalAnimeRecsRecSourceWithConstantPercentTarget(
// numRecommendersToUse: 100,
// fractionConsideredRecommended: 0.25,
// targetFraction: 0.25,
// minEpisodesToClassifyIncomplete: minEpisodesToCountIncomplete
//);
#if MYMEDIALITE
var bprmfRecSource = new MalMyMediaLiteItemRecommenderRecSourceWithConstantPercentTarget <BPRMF>(
new BPRMF()
{
BiasReg = .01f
},
fractionConsideredRecommended: 0.25,
minEpisodesToClassifyIncomplete: minEpisodesToCountIncomplete,
minUsersToCountAnime: 30,
targetFraction: 0.25
);
#endif
//recommendersUnderTest.Add(averageScoreRecSourceWithoutDropped);
//recommendersUnderTest.Add(mostPopularRecSourceWithoutDropped);
//recommendersUnderTest.Add(defaultBiasedMatrixFactorizationRecSource);
//recommendersUnderTest.Add(biasedMatrixFactorizationRecSourceWithBoldDriver);
//recommendersUnderTest.Add(biasedMatrixFactorizationRecSourceWithFactors);
//recommendersUnderTest.Add(biasedMatrixFactorizationRecSourceWithFactorsAndIters);
//recommendersUnderTest.Add(defaultMatrixFactorizationRecSource);
recommendersUnderTest.Add(animeRecsRecSource35);
//recommendersUnderTest.Add(animeRecsRecSource25);
#if MYMEDIALITE
recommendersUnderTest.Add(bprmfRecSource);
#endif
for (int i = 0; i < recommendersUnderTest.Count; i++)
{
resultsForEachRecommender.Add(new List <EvaluationResults>());
}
IUserInputClassifier <MalUserListEntries> targetClassifier = new MalPercentageRatingClassifier(targetPercentile, minEpisodesToCountIncomplete);
MalTrainingData rawData;
IConfigurationBuilder configBuilder = new ConfigurationBuilder()
.AddXmlFile("config_base.xml")
.AddXmlFile("config_overrides.xml", optional: true);
IConfigurationRoot rawConfig = configBuilder.Build();
Config config = rawConfig.Get <Config>();
string postgresConnectionString = config.ConnectionStrings.AnimeRecs;
using (PgMalDataLoader loader = new PgMalDataLoader(postgresConnectionString))
using (CancellationTokenSource timeout = new CancellationTokenSource(TimeSpan.FromSeconds(60)))
{
rawData = loader.LoadMalTrainingDataAsync(timeout.Token).ConfigureAwait(false).GetAwaiter().GetResult();
}
const int numEvaluations = 5;
const int numRecsToGet = 25;
for (int pass = 0; pass < numEvaluations; pass++)
{
for (int recSourceIndex = 0; recSourceIndex < recommendersUnderTest.Count; recSourceIndex++)
{
ITrainableRecSource <MalTrainingData, MalUserListEntries, IEnumerable <IRecommendation>, IRecommendation> recSource = recommendersUnderTest[recSourceIndex];
Tuple <MalTrainingData, ICollection <MalUserListEntries> > dataForTrainingAndEvaluation = GetDataForTrainingAndEvaluation(rawData);
MalTrainingData trainingData = dataForTrainingAndEvaluation.Item1;
ICollection <MalUserListEntries> dataForEvaluation = dataForTrainingAndEvaluation.Item2;
recSource.Train(trainingData);
EvaluationResults results = evaluator.Evaluate(
recSource: recSource,
users: dataForEvaluation,
goodBadClassifier: targetClassifier,
inputDivisionFunc: MalUserListEntries.DivideClassifiedForInputAndEvaluation,
numRecsToTryToGet: numRecsToGet
);
resultsForEachRecommender[recSourceIndex].Add(results);
}
}
for (int recSourceIndex = 0; recSourceIndex < recommendersUnderTest.Count; recSourceIndex++)
{
ITrainableRecSource <MalTrainingData, MalUserListEntries, IEnumerable <IRecommendation>, IRecommendation> recSource = recommendersUnderTest[recSourceIndex];
Console.WriteLine(recSource);
foreach (EvaluationResults resultsForPass in resultsForEachRecommender[recSourceIndex])
{
Console.WriteLine("Precision: {0:P2}\tRecall: {1:P2}",
resultsForPass.Precision, resultsForPass.Recall);
}
Console.WriteLine();
Console.WriteLine();
Console.WriteLine();
Console.WriteLine();
}
}
