private MalTrainingData FilterOutSpecialsAndTrim(MalTrainingData trainingData)
{
Dictionary <int, MalUserListEntries> filteredUsers = new Dictionary <int, MalUserListEntries>();
foreach (int userId in trainingData.Users.Keys.Take(m_recommender.NumRecommenders))
{
List <ReadOnlyMalListEntryDictionary.ListEntryAndAnimeId> filteredEntryList = new List <ReadOnlyMalListEntryDictionary.ListEntryAndAnimeId>();
foreach (int animeId in trainingData.Users[userId].Entries.Keys)
{
if (trainingData.Animes[animeId].Type != MalAnimeType.Special)
{
filteredEntryList.Add(new ReadOnlyMalListEntryDictionary.ListEntryAndAnimeId(animeId, trainingData.Users[userId].Entries[animeId]));
}
}
ReadOnlyMalListEntryDictionary filteredEntries = new ReadOnlyMalListEntryDictionary(filteredEntryList);
MalUserListEntries filteredUser = new MalUserListEntries(filteredEntries, trainingData.Users[userId].AnimesEligibleForRecommendation,
trainingData.Users[userId].MalUsername, trainingData.Users[userId].OkToRecommendPredicate);
filteredUsers[userId] = filteredUser;
}
MalTrainingData filteredTrainingData = new MalTrainingData(filteredUsers, trainingData.Animes);
return(filteredTrainingData);
}
public IEnumerable <RatingPredictionRecommendation> GetRecommendations(MalUserListEntries inputForUser, int numRecommendationsToTryToGet)
{
IBasicInputForUser basicInput = inputForUser.AsBasicInput(m_minEpisodesToCountIncomplete, m_useDropped,
additionalOkToRecommendPredicate: (animeId) => m_userCountByAnime.ContainsKey(animeId) && m_userCountByAnime[animeId] >= m_minUsersToCountAnime
);
return(m_recommender.GetRecommendations(basicInput, numRecommendationsToTryToGet));
}
private static bool AnimeListContainsPrerequisitesFor(MalUserListEntries userAnimeList, IDictionary <int, IList <int> > prereqs,
int animeId)
{
IList <int> animePrereqs;
if (!prereqs.TryGetValue(animeId, out animePrereqs))
{
// No prereqs
return(true);
}
foreach (int prereqId in animePrereqs)
{
// For each prereq, user must have it on their list as completed
if (!userAnimeList.Entries.ContainsKey(prereqId) || userAnimeList.Entries[prereqId].Status != CompletionStatus.Completed)
{
return(false);
}
}
return(true);
}
private static bool AnimeListContainsPrerequisitesFor(MalUserListEntries userAnimeList, IDictionary<int, IList<int>> prereqs,
int animeId)
{
IList<int> animePrereqs;
if (!prereqs.TryGetValue(animeId, out animePrereqs))
{
// No prereqs
return true;
}
foreach (int prereqId in animePrereqs)
{
// For each prereq, user must have it on their list as completed
if (!userAnimeList.Entries.ContainsKey(prereqId) || userAnimeList.Entries[prereqId].Status != CompletionStatus.Completed)
{
return false;
}
}
return true;
}
public MalPositiveFeedbackRecResults GetRecommendations(MalUserListEntries inputForUser, int numRecommendationsToTryToGet)
{
MalPositiveFeedbackInput input = new MalPositiveFeedbackInput(inputForUser, targetFraction: m_targetFraction);
return(m_underlyingRecSource.GetRecommendations(input, numRecommendationsToTryToGet));
}
public MalTrainingData LoadMalTrainingData()
{
// Load all anime, then all users, then all entries
Dictionary<int, mal_user> dbUsers = new Dictionary<int, mal_user>();
Logging.Log.Debug("Slurping anime from the database.");
IEnumerable<mal_anime> dbAnimeSlurp = mal_anime.GetAll(m_conn, transaction: null);
Logging.Log.Debug("Processing anime from the database.");
Dictionary<int, MalAnime> animes = new Dictionary<int, MalAnime>(dbAnimeSlurp.Count());
foreach (mal_anime dbAnime in dbAnimeSlurp)
{
MalAnime anime = new MalAnime(
malAnimeId: dbAnime.mal_anime_id,
type: (MalAnimeType)dbAnime.mal_anime_type_id,
title: dbAnime.title
);
animes[dbAnime.mal_anime_id] = anime;
}
Logging.Log.DebugFormat("Done processing {0} anime from the database.", animes.Count);
Logging.Log.Debug("Slurping users from the database.");
IEnumerable<mal_user> dbUserSlurp = mal_user.GetAll(m_conn, transaction: null);
Logging.Log.Debug("Processing users from the database.");
foreach (mal_user dbUser in dbUserSlurp)
{
dbUsers[dbUser.mal_user_id] = dbUser;
}
Logging.Log.DebugFormat("Done processing {0} users from the database.", dbUsers.Count);
string allEntriesSlimSql = @"
SELECT mal_user_id, mal_anime_id, rating, mal_list_entry_status_id, num_episodes_watched
FROM mal_list_entry
";
Logging.Log.Debug("Slurping list entries from the database.");
// Do not buffer list entries
IEnumerable<mal_list_entry_slim> dbEntrySlurp = m_conn.Query<mal_list_entry_slim>(allEntriesSlimSql, buffered: false, commandTimeout: 60);
Logging.Log.Debug("Processing list entries from the database.");
long entryCount = 0;
Dictionary<int, List<ReadOnlyMalListEntryDictionary.ListEntryAndAnimeId>> entriesByUser
= new Dictionary<int, List<ReadOnlyMalListEntryDictionary.ListEntryAndAnimeId>>();
foreach (mal_list_entry_slim dbEntry in dbEntrySlurp)
{
entryCount++;
mal_user dbUser;
if (!dbUsers.TryGetValue(dbEntry.mal_user_id, out dbUser) || !animes.ContainsKey(dbEntry.mal_anime_id))
{
// Entry for an anime or user that wasn't in the database...there must have been an update going on between the time we got users, anime, and list entries
continue;
}
List<ReadOnlyMalListEntryDictionary.ListEntryAndAnimeId> animeList;
if (!entriesByUser.TryGetValue(dbEntry.mal_user_id, out animeList))
{
animeList = new List<ReadOnlyMalListEntryDictionary.ListEntryAndAnimeId>();
entriesByUser[dbEntry.mal_user_id] = animeList;
}
animeList.Add(new ReadOnlyMalListEntryDictionary.ListEntryAndAnimeId(
animeId: dbEntry.mal_anime_id,
entry: new MalListEntry(
rating: (byte?)dbEntry.rating,
status: (CompletionStatus)dbEntry.mal_list_entry_status_id,
numEpisodesWatched: dbEntry.num_episodes_watched
)
));
}
Dictionary<int, MalUserListEntries> users = new Dictionary<int, MalUserListEntries>(dbUserSlurp.Count());
foreach (int userId in entriesByUser.Keys)
{
List<ReadOnlyMalListEntryDictionary.ListEntryAndAnimeId> animeList = entriesByUser[userId];
animeList.Capacity = animeList.Count;
ReadOnlyMalListEntryDictionary listEntries = new ReadOnlyMalListEntryDictionary(animeList);
users[userId] = new MalUserListEntries(listEntries, animes, dbUsers[userId].mal_name, okToRecommendPredicate: null);
}
Logging.Log.DebugFormat("Done processing {0} list entries.", entryCount);
return new MalTrainingData(users, animes);
}
public IEnumerable <MostPopularRecommendation> GetRecommendations(MalUserListEntries inputForUser, int numRecommendationsToTryToGet)
{
return(m_recommender.GetRecommendations(inputForUser, numRecommendationsToTryToGet));
}
public MalAnimeRecsInput(MalUserListEntries animeList, double targetFraction)
{
AnimeList = animeList;
TargetFraction = targetFraction;
}
public MalAnimeRecsInput(MalUserListEntries animeList, decimal targetScore)
{
AnimeList = animeList;
TargetScore = targetScore;
}
public MalPositiveFeedbackInput(MalUserListEntries animeList, decimal targetScore)
{
AnimeList = animeList;
TargetScore = targetScore;
}
public MalPositiveFeedbackInput(MalUserListEntries animeList, double targetFraction)
{
AnimeList = animeList;
TargetFraction = targetFraction;
}
public MalPositiveFeedbackInput(MalUserListEntries animeList, decimal targetScore)
{
AnimeList = animeList;
TargetScore = targetScore;
}
public MalPositiveFeedbackInput(MalUserListEntries animeList, double targetFraction)
{
AnimeList = animeList;
TargetFraction = targetFraction;
}
public GetMalRecsResponse GetMalRecs(GetMalRecsRequest request)
{
request.AssertArgumentNotNull("Payload");
request.RecSourceName.AssertArgumentNotNull("Payload.RecSourceName");
request.AnimeList.AssertArgumentNotNull("Payload.AnimeList");
request.AnimeList.Entries.AssertArgumentNotNull("Payload.AnimeList.Entries");
if (request.TargetScore == null && request.TargetFraction == null)
{
Error error = new Error(ErrorCodes.InvalidArgument, "Payload.TargetScore or Payload.TargetFraction must be set.");
throw new RecServiceErrorException(error);
}
string targetScoreString;
if (request.TargetFraction != null)
{
targetScoreString = request.TargetFraction.Value.ToString("P2");
}
else
{
targetScoreString = request.TargetScore.Value.ToString();
}
Logging.Log.InfoFormat("Request for {0} MAL recs using rec source {1}. User has {2} anime list entries. Target score is {3}.",
request.NumRecsDesired, request.RecSourceName, request.AnimeList.Entries.Count, targetScoreString);
// Acquire read lock on rec sources
bool enteredLock;
const int lockTimeoutInMs = 3000;
using (var trainingDataReadLock = m_trainingDataLock.ScopedReadLock(lockTimeoutInMs, out enteredLock))
{
// If we couldn't get a read lock within 3 seconds, a reload/retrain is probably going on
if (!enteredLock)
{
Error error = new Error(errorCode: ErrorCodes.Maintenance,
message: "The rec service is currently undergoing maintenance and cannot respond to rec requests.");
throw new RecServiceErrorException(error);
}
using (var recSourcesReadLock = m_recSourcesLock.ScopedReadLock())
{
// Get rec source by name
if (!m_recSources.ContainsKey(request.RecSourceName))
{
Error error = new Error(errorCode: ErrorCodes.NoSuchRecSource,
message: string.Format("No rec source called \"{0}\" is loaded.", request.RecSourceName));
throw new RecServiceErrorException(error);
}
ITrainableJsonRecSource recSource = m_recSources[request.RecSourceName];
// Convert DTO anime list to RecEngine anime list
Dictionary<int, AnimeRecs.RecEngine.MAL.MalListEntry> entries = new Dictionary<int, RecEngine.MAL.MalListEntry>();
foreach (AnimeRecs.RecService.DTO.MalListEntry dtoEntry in request.AnimeList.Entries)
{
AnimeRecs.RecEngine.MAL.MalListEntry recEngineEntry = new RecEngine.MAL.MalListEntry(dtoEntry.Rating, dtoEntry.Status, dtoEntry.NumEpisodesWatched);
entries[dtoEntry.MalAnimeId] = recEngineEntry;
}
MalUserListEntries animeList = new MalUserListEntries(ratings: entries, animes: m_animes,
malUsername: null, prerequisites: m_prereqs);
Stopwatch timer = Stopwatch.StartNew();
GetMalRecsResponse response = recSource.GetRecommendations(animeList, request);
timer.Stop();
Logging.Log.InfoFormat("Got recommendations from rec source {0}. Took {1}.", request.RecSourceName, timer.Elapsed);
return response;
}
}
}
