public GetRecsViewModel(MalRecResults<IEnumerable<IRecommendation>> results, int userId, string userName,
MalUserLookupResults userLookup, IDictionary<int, MalListEntry> userAnimeList,
int maximumRecommendationsToReturn, int maximumRecommendersToReturn, IDictionary<int, MalListEntry> animeWithheld, IAnimeRecsDbConnectionFactory dbConnectionFactory)
{
Results = results;
UserId = userId;
UserName = userName;
UserLookup = userLookup;
UserAnimeList = userAnimeList;
MaximumRecommendationsToReturn = maximumRecommendationsToReturn;
MaximumRecommendersToReturn = maximumRecommendersToReturn;
DbConnectionFactory = dbConnectionFactory;
StreamsByAnime = new Dictionary<int, ICollection<streaming_service_anime_map>>();
AnimeWithheld = animeWithheld;
}
public bool GetListForUser(string user, out MalUserLookupResults animeList)
{
m_cacheLock.EnterReadLock();
try
{
if (m_expiration == null)
{
if (m_animeListCache.TryGetValue(user, out animeList))
{
return true;
}
else
{
animeList = null;
return false;
}
}
LinkedListNode<Tuple<string, DateTime>> userAndTimeInsertedNode;
// Check if this user is in the cache and if the cache entry is not stale
if (m_cachePutTimesByName.TryGetValue(user, out userAndTimeInsertedNode))
{
DateTime expirationTime = userAndTimeInsertedNode.Value.Item2 + m_expiration.Value;
if (DateTime.UtcNow < expirationTime)
{
animeList = m_animeListCache[user];
return true;
}
else
{
animeList = null;
return false;
}
}
else
{
animeList = null;
return false;
}
}
finally
{
m_cacheLock.ExitReadLock();
}
}
public bool GetListForUser(string user, out MalUserLookupResults animeList)
{
m_cacheLock.EnterReadLock();
try
{
if (m_expiration == null)
{
if (m_animeListCache.TryGetValue(user, out animeList))
{
return(true);
}
else
{
animeList = null;
return(false);
}
}
// Check if this user is in the cache and if the cache entry is not stale
if (m_cachePutTimesByName.TryGetValue(user, out LinkedListNode <Tuple <string, DateTime> > userAndTimeInsertedNode))
{
DateTime expirationTime = userAndTimeInsertedNode.Value.Item2 + m_expiration.Value;
if (DateTime.UtcNow < expirationTime)
{
animeList = m_animeListCache[user];
return(true);
}
else
{
animeList = null;
return(false);
}
}
else
{
animeList = null;
return(false);
}
}
finally
{
m_cacheLock.ExitReadLock();
}
}
/// <summary>
/// Gets a user's manga list. This method requires a MAL API key.
/// </summary>
/// <param name="user"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <exception cref="MalApi.MalUserNotFoundException"></exception>
/// <exception cref="MalApi.MalApiException"></exception>
public async Task <MalUserLookupResults> GetMangaListForUserAsync(string user,
CancellationToken cancellationToken)
{
const string malAppMangaInfoUriFormatString = "https://myanimelist.net/malappinfo.php?status=all&type=manga&u={0}";
string userInfoUri = string.Format(malAppMangaInfoUriFormatString, Uri.EscapeDataString(user));
Logging.Log.InfoFormat("Getting manga list for MAL user {0} using URI {1}", user, userInfoUri);
Func <string, MalUserLookupResults> responseProcessingFunc = (xml) =>
{
using (TextReader xmlTextReader = new StringReader(xml))
{
try
{
return(MalAppInfoXml.Parse(xmlTextReader));
}
catch (MalUserNotFoundException ex)
{
throw new MalUserNotFoundException(string.Format("No MAL list exists for {0}.", user), ex);
}
}
};
try
{
HttpRequestMessage request = InitNewRequest(userInfoUri, HttpMethod.Get);
MalUserLookupResults parsedList = await ProcessRequestAsync(request, responseProcessingFunc,
cancellationToken : cancellationToken,
baseErrorMessage : string.Format("Failed getting manga list for user {0} using url {1}", user,
userInfoUri)).ConfigureAwait(continueOnCapturedContext: false);
Logging.Log.InfoFormat("Successfully retrieved manga list for user {0}", user);
return(parsedList);
}
catch (OperationCanceledException)
{
Logging.Log.InfoFormat("Canceled getting manga list for MAL user {0}", user);
throw;
}
}
public void PutListForUser(string user, MalUserLookupResults animeList)
{
m_cacheLock.EnterWriteLock();
try
{
if (m_expiration == null)
{
m_animeListCache[user] = animeList;
return;
}
if (m_cachePutTimesByName.TryGetValue(user, out LinkedListNode <Tuple <string, DateTime> > nodeForLastInsert))
{
m_cachePutTimesSortedByTime.Remove(nodeForLastInsert);
}
DateTime nowUtc = DateTime.UtcNow;
DateTime deleteOlderThan = nowUtc - m_expiration.Value;
var newNode = m_cachePutTimesSortedByTime.AddFirst(new Tuple <string, DateTime>(user, nowUtc));
m_cachePutTimesByName[user] = newNode;
m_animeListCache[user] = animeList;
// Check for old entries and remove them
while (m_cachePutTimesSortedByTime.Count > 0 && m_cachePutTimesSortedByTime.Last.Value.Item2 < deleteOlderThan)
{
string oldUser = m_cachePutTimesSortedByTime.Last.Value.Item1;
m_animeListCache.Remove(oldUser);
m_cachePutTimesByName.Remove(oldUser);
m_cachePutTimesSortedByTime.RemoveLast();
}
}
finally
{
m_cacheLock.ExitWriteLock();
}
}
/// <summary>
/// Gets a user's anime list.
/// </summary>
/// <param name="user"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <MalUserLookupResults> GetAnimeListForUserAsync(string user, CancellationToken cancellationToken)
{
Logging.Log.InfoFormat("Checking cache for user {0}.", user);
if (m_cache.GetListForUser(user, out MalUserLookupResults cachedAnimeList))
{
if (cachedAnimeList != null)
{
Logging.Log.InfoFormat("Got anime list for {0} from cache.", user);
return(cachedAnimeList);
}
else
{
// User does not have an anime list/no such user exists
Logging.Log.InfoFormat("Cache indicates that user {0} does not have an anime list.", user);
throw new MalUserNotFoundException(string.Format("No MAL list exists for {0}.", user));
}
}
else
{
Logging.Log.InfoFormat("Cache did not contain anime list for {0}.", user);
try
{
MalUserLookupResults animeList = await m_underlyingApi.GetAnimeListForUserAsync(user, cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false);
m_cache.PutListForUser(user, animeList);
return(animeList);
}
catch (MalUserNotFoundException)
{
// Cache the fact that the user does not have an anime list
m_cache.PutListForUser(user, null);
throw;
}
}
}
/// <summary>
/// Parses XML obtained from malappinfo.php.
/// </summary>
/// <param name="doc"></param>
/// <returns></returns>
public static MalUserLookupResults Parse(XDocument doc)
{
Logging.Log.Trace("Parsing XML.");
XElement error = doc.Root.Element("error");
if (error != null && (string)error == "Invalid username")
{
throw new MalUserNotFoundException("No MAL list exists for this user.");
}
else if (error != null)
{
throw new MalApiException((string)error);
}
if (!doc.Root.HasElements)
{
throw new MalUserNotFoundException("No MAL list exists for this user.");
}
XElement myinfo = GetExpectedElement(doc.Root, "myinfo");
int userId = GetElementValueInt(myinfo, "user_id");
string canonicalUserName = GetElementValueString(myinfo, "user_name");
List <MyAnimeListEntry> entries = new List <MyAnimeListEntry>();
IEnumerable <XElement> animes = doc.Root.Elements("anime");
foreach (XElement anime in animes)
{
int animeId = GetElementValueInt(anime, "series_animedb_id");
string title = GetElementValueString(anime, "series_title");
string synonymList = GetElementValueString(anime, "series_synonyms");
string[] rawSynonyms = synonymList.Split(SynonymSeparator, StringSplitOptions.RemoveEmptyEntries);
// filter out synonyms that are the same as the main title
HashSet <string> synonyms = new HashSet <string>(rawSynonyms.Where(synonym => !synonym.Equals(title, StringComparison.Ordinal)));
int seriesTypeInt = GetElementValueInt(anime, "series_type");
MalAnimeType seriesType = (MalAnimeType)seriesTypeInt;
int numEpisodes = GetElementValueInt(anime, "series_episodes");
int seriesStatusInt = GetElementValueInt(anime, "series_status");
MalSeriesStatus seriesStatus = (MalSeriesStatus)seriesStatusInt;
string seriesStartString = GetElementValueString(anime, "series_start");
UncertainDate seriesStart = UncertainDate.FromMalDateString(seriesStartString);
string seriesEndString = GetElementValueString(anime, "series_end");
UncertainDate seriesEnd = UncertainDate.FromMalDateString(seriesEndString);
string seriesImage = GetElementValueString(anime, "series_image");
MalAnimeInfoFromUserLookup animeInfo = new MalAnimeInfoFromUserLookup(animeId: animeId, title: title,
type: seriesType, synonyms: synonyms, status: seriesStatus, numEpisodes: numEpisodes, startDate: seriesStart,
endDate: seriesEnd, imageUrl: seriesImage);
int numEpisodesWatched = GetElementValueInt(anime, "my_watched_episodes");
string myStartDateString = GetElementValueString(anime, "my_start_date");
UncertainDate myStartDate = UncertainDate.FromMalDateString(myStartDateString);
string myFinishDateString = GetElementValueString(anime, "my_finish_date");
UncertainDate myFinishDate = UncertainDate.FromMalDateString(myFinishDateString);
decimal rawScore = GetElementValueDecimal(anime, "my_score");
decimal?myScore = rawScore == 0 ? (decimal?)null : rawScore;
int completionStatusInt = GetElementValueInt(anime, "my_status");
CompletionStatus completionStatus = (CompletionStatus)completionStatusInt;
long lastUpdatedUnixTimestamp = GetElementValueLong(anime, "my_last_updated");
DateTime lastUpdated = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc) + TimeSpan.FromSeconds(lastUpdatedUnixTimestamp);
string rawTagsString = GetElementValueString(anime, "my_tags");
string[] untrimmedTags = rawTagsString.Split(TagSeparator, StringSplitOptions.RemoveEmptyEntries);
List <string> tags = new List <string>(untrimmedTags.Select(tag => tag.Trim()));
MyAnimeListEntry entry = new MyAnimeListEntry(score: myScore, status: completionStatus, numEpisodesWatched: numEpisodesWatched,
myStartDate: myStartDate, myFinishDate: myFinishDate, myLastUpdate: lastUpdated, animeInfo: animeInfo, tags: tags);
entries.Add(entry);
}
MalUserLookupResults results = new MalUserLookupResults(userId: userId, canonicalUserName: canonicalUserName, animeList: entries);
Logging.Log.Trace("Parsed XML.");
return(results);
}
public void PutListForUser(string user, MalUserLookupResults animeList)
{
m_cacheLock.EnterWriteLock();
try
{
if (m_expiration == null)
{
m_animeListCache[user] = animeList;
return;
}
LinkedListNode<Tuple<string, DateTime>> nodeForLastInsert;
if (m_cachePutTimesByName.TryGetValue(user, out nodeForLastInsert))
{
m_cachePutTimesSortedByTime.Remove(nodeForLastInsert);
}
DateTime nowUtc = DateTime.UtcNow;
DateTime deleteOlderThan = nowUtc - m_expiration.Value;
var newNode = m_cachePutTimesSortedByTime.AddFirst(new Tuple<string, DateTime>(user, nowUtc));
m_cachePutTimesByName[user] = newNode;
m_animeListCache[user] = animeList;
// Check for old entries and remove them
while (m_cachePutTimesSortedByTime.Count > 0 && m_cachePutTimesSortedByTime.Last.Value.Item2 < deleteOlderThan)
{
string oldUser = m_cachePutTimesSortedByTime.Last.Value.Item1;
m_animeListCache.Remove(oldUser);
m_cachePutTimesByName.Remove(oldUser);
m_cachePutTimesSortedByTime.RemoveLast();
}
}
finally
{
m_cacheLock.ExitWriteLock();
}
}
static void InsertUserAndRatingsInDatabase(MalUserLookupResults userLookup, NpgsqlConnection conn, NpgsqlTransaction transaction)
{
Logging.Log.InfoFormat("Inserting anime and list entries for {0} ({1} entries).", userLookup.CanonicalUserName, userLookup.AnimeList.Count);
List<mal_anime> animesToUpsert = new List<mal_anime>();
Dictionary<int, List<mal_anime_synonym>> synonymsToUpsert = new Dictionary<int, List<mal_anime_synonym>>();
List<mal_list_entry> entriesToInsert = new List<mal_list_entry>();
List<mal_list_entry_tag> tagsToInsert = new List<mal_list_entry_tag>();
// Buffer animes, anime synonyms, list entries, and tags.
// For animes not upserted this session, upsert animes all at once, clear synonyms, insert synonyms
// insert user
// insert list entries all at once
// insert tags all at once
foreach (MyAnimeListEntry anime in userLookup.AnimeList)
{
if (!AnimesUpserted.ContainsKey(anime.AnimeInfo.AnimeId))
{
mal_anime animeRow = new mal_anime(
_mal_anime_id: anime.AnimeInfo.AnimeId,
_title: anime.AnimeInfo.Title,
_mal_anime_type_id: (int)anime.AnimeInfo.Type,
_num_episodes: anime.AnimeInfo.NumEpisodes,
_mal_anime_status_id: (int)anime.AnimeInfo.Status,
_start_year: (short?)anime.AnimeInfo.StartDate.Year,
_start_month: (short?)anime.AnimeInfo.StartDate.Month,
_start_day: (short?)anime.AnimeInfo.StartDate.Day,
_end_year: (short?)anime.AnimeInfo.EndDate.Year,
_end_month: (short?)anime.AnimeInfo.EndDate.Month,
_end_day: (short?)anime.AnimeInfo.EndDate.Day,
_image_url: anime.AnimeInfo.ImageUrl,
_last_updated: DateTime.UtcNow
);
animesToUpsert.Add(animeRow);
List<mal_anime_synonym> synonymRowsForThisAnime = new List<mal_anime_synonym>();
foreach (string synonym in anime.AnimeInfo.Synonyms)
{
mal_anime_synonym synonymRow = new mal_anime_synonym(
_mal_anime_id: anime.AnimeInfo.AnimeId,
_synonym: synonym
);
synonymRowsForThisAnime.Add(synonymRow);
}
synonymsToUpsert[anime.AnimeInfo.AnimeId] = synonymRowsForThisAnime;
}
mal_list_entry dbListEntry = new mal_list_entry(
_mal_user_id: userLookup.UserId,
_mal_anime_id: anime.AnimeInfo.AnimeId,
_rating: (short?)anime.Score,
_mal_list_entry_status_id: (short)anime.Status,
_num_episodes_watched: (short)anime.NumEpisodesWatched,
_started_watching_year: (short?)anime.MyStartDate.Year,
_started_watching_month: (short?)anime.MyStartDate.Month,
_started_watching_day: (short?)anime.MyStartDate.Day,
_finished_watching_year: (short?)anime.MyFinishDate.Year,
_finished_watching_month: (short?)anime.MyFinishDate.Month,
_finished_watching_day: (short?)anime.MyFinishDate.Day,
_last_mal_update: anime.MyLastUpdate
);
entriesToInsert.Add(dbListEntry);
foreach (string tag in anime.Tags)
{
mal_list_entry_tag dbTag = new mal_list_entry_tag(
_mal_user_id: userLookup.UserId,
_mal_anime_id: anime.AnimeInfo.AnimeId,
_tag: tag
);
tagsToInsert.Add(dbTag);
}
}
// For animes not upserted this session, upsert animes, clear synonyms all at once, insert synonyms all at once
Logging.Log.DebugFormat("Upserting {0} animes.", animesToUpsert.Count);
foreach (mal_anime animeToUpsert in animesToUpsert)
{
Logging.Log.TraceFormat("Checking if anime \"{0}\" is in the database.", animeToUpsert.title);
bool animeIsInDb = mal_anime.IsInDatabase(animeToUpsert.mal_anime_id, conn, transaction);
if (!animeIsInDb)
{
// Not worth optimizing this by batching inserts because once there are a couple hundred users in the database,
// inserts will be relatively few in number.
Logging.Log.Trace("Not in database. Inserting it.");
animeToUpsert.Insert(conn, transaction);
Logging.Log.TraceFormat("Inserted anime \"{0}\" in database.", animeToUpsert.title);
AnimesUpserted[animeToUpsert.mal_anime_id] = animeToUpsert;
}
else
{
Logging.Log.TraceFormat("Already in database. Updating it.");
animeToUpsert.Update(conn, transaction);
Logging.Log.TraceFormat("Updated anime \"{0}\".", animeToUpsert.title);
AnimesUpserted[animeToUpsert.mal_anime_id] = animeToUpsert;
}
}
Logging.Log.DebugFormat("Upserted {0} animes.", animesToUpsert.Count);
if (synonymsToUpsert.Count > 0)
{
List<mal_anime_synonym> flattenedSynonyms = synonymsToUpsert.Values.SelectMany(synonyms => synonyms).ToList();
// clear synonyms for all these animes
Logging.Log.DebugFormat("Clearing {0} synonyms for this batch.", flattenedSynonyms.Count);
mal_anime_synonym.Delete(synonymsToUpsert.Keys, conn, transaction);
Logging.Log.DebugFormat("Cleared {0} synonyms for this batch.", flattenedSynonyms.Count);
// insert synonyms for all these animes
Logging.Log.DebugFormat("Inserting {0} synonyms for this batch.", flattenedSynonyms.Count);
mal_anime_synonym.Insert(flattenedSynonyms, conn, transaction);
Logging.Log.DebugFormat("Inserted {0} synonyms for this batch.", flattenedSynonyms.Count);
}
else
{
Logging.Log.Debug("No synonyms in this batch.");
}
// Insert user
mal_user user = new mal_user(
_mal_user_id: userLookup.UserId,
_mal_name: userLookup.CanonicalUserName,
_time_added: DateTime.UtcNow
);
Logging.Log.DebugFormat("Inserting {0} into DB.", userLookup.CanonicalUserName);
user.Insert(conn, transaction);
Logging.Log.DebugFormat("Inserted {0} into DB.", userLookup.CanonicalUserName);
// insert list entries all at once
if (entriesToInsert.Count > 0)
{
Logging.Log.DebugFormat("Inserting {0} list entries for user \"{1}\".", entriesToInsert.Count, userLookup.CanonicalUserName);
mal_list_entry.Insert(entriesToInsert, conn, transaction);
Logging.Log.DebugFormat("Inserted {0} list entries for user \"{1}\".", entriesToInsert.Count, userLookup.CanonicalUserName);
}
// insert tags all at once
if (tagsToInsert.Count > 0)
{
Logging.Log.DebugFormat("Inserting {0} tags by user \"{1}\".", tagsToInsert.Count, userLookup.CanonicalUserName);
mal_list_entry_tag.Insert(tagsToInsert, conn, transaction);
Logging.Log.DebugFormat("Inserted {0} tags by user \"{1}\".", tagsToInsert.Count, userLookup.CanonicalUserName);
}
Logging.Log.InfoFormat("Done inserting anime and list entries for {0}.", userLookup.CanonicalUserName);
}
static bool UserMeetsCriteria(MalUserLookupResults userLookup, NpgsqlConnection conn, NpgsqlTransaction transaction)
{
// completed, rated >= X, and user is not in DB
int completedRated = userLookup.AnimeList.Count(anime => anime.Score.HasValue && anime.Status == CompletionStatus.Completed);
if (completedRated < config.MinimumAnimesCompletedAndRated)
{
return false;
}
Logging.Log.DebugFormat("Really checking if {0} is in the database by user id.", userLookup.CanonicalUserName);
bool isInDb = mal_user.UserIsInDb(userLookup.UserId, conn, transaction);
Logging.Log.DebugFormat("{0} really in database = {1}", userLookup.CanonicalUserName, isInDb);
return !isInDb;
}
