private SVR_AniDB_File TryGetAniDBFileFromAniDB(SVR_VideoLocal vidLocal, Dictionary <int, bool> animeIDs)
{
// check if we already have a record
SVR_AniDB_File aniFile = RepoFactory.AniDB_File.GetByHashAndFileSize(vidLocal.Hash, vlocal.FileSize);
if (aniFile == null || aniFile.FileSize != vlocal.FileSize)
{
ForceAniDB = true;
}
if (ForceAniDB)
{
if (!ShokoService.AnidbProcessor.IsUdpBanned)
{
// get info from AniDB
logger.Debug("Getting AniDB_File record from AniDB....");
Raw_AniDB_File fileInfo = ShokoService.AnidbProcessor.GetFileInfo(vidLocal);
if (fileInfo != null)
{
aniFile ??= new SVR_AniDB_File();
SVR_AniDB_File.Populate(aniFile, fileInfo);
}
}
else
{
CommandRequest_GetFile fileCommand = new CommandRequest_GetFile(vlocal.VideoLocalID, true);
fileCommand.Save();
}
}
if (aniFile == null)
{
return(null);
}
//overwrite with local file name
string localFileName = vidLocal.GetBestVideoLocalPlace()?.FullServerPath;
localFileName = !string.IsNullOrEmpty(localFileName)
? Path.GetFileName(localFileName)
: vidLocal.FileName;
aniFile.FileName = localFileName;
RepoFactory.AniDB_File.Save(aniFile, false);
aniFile.CreateLanguages();
aniFile.CreateCrossEpisodes(localFileName);
aniFile.Episodes.Select(a => a.AnimeID).Distinct().ForEach(animeID =>
{
if (animeIDs.ContainsKey(animeID))
{
animeIDs[animeID] = false;
}
else
{
animeIDs.Add(animeID, false);
}
});
return(aniFile);
}
public AniDB(SVR_AniDB_File anidb)
{
ID = anidb.FileID;
Source = anidb.File_Source;
ReleaseGroup = new AniDB.AniDBReleaseGroup
{
ID = anidb.GroupID,
Name = anidb.Anime_GroupName,
ShortName = anidb.Anime_GroupNameShort,
};
ReleaseDate = anidb.File_ReleaseDate == 0 ? null : Commons.Utils.AniDB.GetAniDBDateAsDate(anidb.File_ReleaseDate);
Version = anidb.FileVersion;
IsDeprecated = anidb.IsDeprecated == 1;
IsCensored = anidb.IsCensored == 1;
Chaptered = anidb.IsChaptered == 1;
Duration = (new TimeSpan(0, 0, anidb.File_LengthSeconds));
Resolution = anidb.File_VideoResolution;
VideoCodec = anidb.File_VideoCodec;
OriginalFileName = anidb.FileName;
FileSize = anidb.FileSize;
Description = anidb.File_Description;
Updated = anidb.DateTimeUpdated;
AudioCodecs = anidb.File_AudioCodec.Split(new[] { '\'', '`', '"' }, StringSplitOptions.RemoveEmptyEntries)
.ToList();
AudioLanguages = anidb.Languages.Select(a => a.LanguageName).ToList();
SubLanguages = anidb.Subtitles.Select(a => a.LanguageName).ToList();
}
public static string GetResolution(SVR_VideoLocal videoLocal, SVR_AniDB_File aniFile = null)
{
if (aniFile == null)
{
aniFile = videoLocal?.GetAniDBFile();
}
return(MediaInfoUtils.GetStandardResolution(GetResolutionInternal(videoLocal, aniFile)));
}
public override void ProcessCommand()
{
logger.Info("Get AniDB file info: {0}", VideoLocalID);
try
{
if (vlocal == null)
{
vlocal = RepoFactory.VideoLocal.GetByID(VideoLocalID);
}
if (vlocal == null)
{
return;
}
lock (vlocal)
{
SVR_AniDB_File aniFile = RepoFactory.AniDB_File.GetByHashAndFileSize(vlocal.Hash, vlocal.FileSize);
Raw_AniDB_File fileInfo = null;
if (aniFile == null || ForceAniDB)
{
fileInfo = ShokoService.AnidbProcessor.GetFileInfo(vlocal);
}
if (fileInfo != null)
{
// save to the database
if (aniFile == null)
{
aniFile = new SVR_AniDB_File();
}
SVR_AniDB_File.Populate(aniFile, fileInfo);
//overwrite with local file name
string localFileName = vlocal.FileName;
aniFile.FileName = localFileName;
RepoFactory.AniDB_File.Save(aniFile, false);
aniFile.CreateLanguages();
aniFile.CreateCrossEpisodes(localFileName);
SVR_AniDB_Anime anime = RepoFactory.AniDB_Anime.GetByAnimeID(aniFile.AnimeID);
if (anime != null)
{
RepoFactory.AniDB_Anime.Save(anime);
}
SVR_AnimeSeries series = RepoFactory.AnimeSeries.GetByAnimeID(aniFile.AnimeID);
series.UpdateStats(true, true, true);
}
}
}
catch (Exception ex)
{
logger.Error("Error processing CommandRequest_GetFile: {0} - {1}", VideoLocalID, ex);
}
}
public override void ProcessCommand()
{
logger.Info("Get AniDB file info: {0}", VideoLocalID);
try
{
if (vlocal == null)
{
vlocal = Repo.Instance.VideoLocal.GetByID(VideoLocalID);
}
if (vlocal == null)
{
return;
}
lock (vlocal)
{
SVR_AniDB_File aniFile = Repo.Instance.AniDB_File.GetByHashAndFileSize(vlocal.Hash, vlocal.FileSize);
Raw_AniDB_File fileInfo = null;
if (aniFile == null || ForceAniDB)
{
fileInfo = ShokoService.AnidbProcessor.GetFileInfo(vlocal);
}
if (fileInfo != null)
{
using (var upd = Repo.Instance.AniDB_File.BeginAddOrUpdate(() => aniFile))
{
upd.Entity.Populate_RA(fileInfo);
//overwrite with local file name
string localFileName = vlocal.FileName;
upd.Entity.FileName = localFileName;
aniFile = upd.Commit();
}
aniFile.CreateLanguages();
aniFile.CreateCrossEpisodes(vlocal.FileName);
//TODO: Look at why this might be worth it?
//SVR_AniDB_Anime anime = Repo.Instance.AniDB_Anime.GetByAnimeID(aniFile.AnimeID);
//if (anime != null) Repo.Instance.AniDB_Anime.Save(anime);
SVR_AnimeSeries series = Repo.Instance.AnimeSeries.GetByAnimeID(aniFile.AnimeID);
series.UpdateStats(true, true, true);
}
}
}
catch (Exception ex)
{
logger.Error("Error processing CommandRequest_GetFile: {0} - {1}", VideoLocalID, ex);
}
}
private static int CompareResolutionTo(SVR_VideoLocal newFile, SVR_VideoLocal oldFile,
SVR_AniDB_File newAniFile, SVR_AniDB_File oldAniFile)
{
string oldRes = GetResolution(oldFile, oldAniFile);
string newRes = GetResolution(newFile, newAniFile);
if (newRes == null && oldRes == null)
{
return(0);
}
if (newRes == null)
{
return(1);
}
if (oldRes == null)
{
return(-1);
}
string[] res = Settings.PreferredResolutions.ToArray();
if (!res.Contains(newRes) && !res.Contains(oldRes))
{
return(0);
}
if (!res.Contains(newRes))
{
return(1);
}
if (!res.Contains(oldRes))
{
return(-1);
}
int newIndex = Array.IndexOf(res, newRes);
int oldIndex = Array.IndexOf(res, oldRes);
return(newIndex.CompareTo(oldIndex));
}
public override void ProcessCommand()
{
logger.Info("Get AniDB file info: {0}", VideoLocalID);
try
{
vlocal = RepoFactory.VideoLocal.GetByID(VideoLocalID);
if (vlocal == null)
{
return;
}
lock (vlocal)
{
SVR_AniDB_File aniFile = RepoFactory.AniDB_File.GetByHashAndFileSize(vlocal.Hash, vlocal.FileSize);
/*// get anidb file info from web cache
* if (aniFile == null && ServerSettings.WebCache_AniDB_File_Get)
* {
* AniDB_FileRequest fr = XMLService.Get_AniDB_File(vlocal.Hash, vlocal.FileSize);
* if (fr != null)
* {
* aniFile = new AniDB_File();
* aniFile.Populate(fr);
*
* //overwrite with local file name
* string localFileName = Path.GetFileName(vlocal.FilePath);
* aniFile.FileName = localFileName;
*
* repAniFile.Save(aniFile, false);
* aniFile.CreateLanguages();
* aniFile.CreateCrossEpisodes(localFileName);
*
* StatsCache.Instance.UpdateUsingAniDBFile(vlocal.Hash);
* }
* }*/
Raw_AniDB_File fileInfo = null;
if (aniFile == null || ForceAniDB)
{
fileInfo = ShokoService.AnidbProcessor.GetFileInfo(vlocal);
}
if (fileInfo != null)
{
// save to the database
if (aniFile == null)
{
aniFile = new SVR_AniDB_File();
}
SVR_AniDB_File.Populate(aniFile, fileInfo);
//overwrite with local file name
string localFileName = vlocal.FileName;
aniFile.FileName = localFileName;
RepoFactory.AniDB_File.Save(aniFile, false);
aniFile.CreateLanguages();
aniFile.CreateCrossEpisodes(localFileName);
if (!string.IsNullOrEmpty(fileInfo.OtherEpisodesRAW))
{
string[] epIDs = fileInfo.OtherEpisodesRAW.Split(',');
foreach (string epid in epIDs)
{
int id = 0;
if (int.TryParse(epid, out id))
{
CommandRequest_GetEpisode cmdEp = new CommandRequest_GetEpisode(id);
cmdEp.Save();
}
}
}
SVR_AniDB_Anime anime = RepoFactory.AniDB_Anime.GetByAnimeID(aniFile.AnimeID);
if (anime != null)
{
using (var session = DatabaseFactory.SessionFactory.OpenSession())
{
anime.UpdateContractDetailed(session.Wrap());
}
}
SVR_AnimeSeries series = RepoFactory.AnimeSeries.GetByAnimeID(aniFile.AnimeID);
series.UpdateStats(false, true, true);
// StatsCache.Instance.UpdateUsingAniDBFile(vlocal.Hash);
}
}
}
catch (Exception ex)
{
logger.Error("Error processing CommandRequest_GetFile: {0} - {1}", VideoLocalID, ex.ToString());
return;
}
}
public static bool CheckFileKeep(SVR_VideoLocal file)
{
bool result = true;
SVR_AniDB_File aniFile = file.GetAniDBFile();
// Don't delete files with missing info. If it's not getting updated, then do it manually
if (aniFile != null)
{
if (aniFile.File_Source.Equals("unknown"))
{
return(true);
}
if (aniFile.File_VideoResolution.Equals("0x0"))
{
return(true);
}
}
foreach (var type in Settings._requiredtypes)
{
if (!result)
{
break;
}
switch (type)
{
case FileQualityFilterType.AUDIOCODEC:
result &= CheckAudioCodec(file);
break;
case FileQualityFilterType.AUDIOSTREAMCOUNT:
result &= CheckAudioStreamCount(file);
break;
case FileQualityFilterType.CHAPTER:
if (aniFile == null)
{
return(false);
}
result &= CheckChaptered(file);
break;
case FileQualityFilterType.RESOLUTION:
result &= CheckResolution(file, aniFile);
break;
case FileQualityFilterType.SOURCE:
if (aniFile == null)
{
return(false);
}
result &= CheckSource(aniFile);
break;
case FileQualityFilterType.SUBGROUP:
if (aniFile == null)
{
return(false);
}
result &= CheckSubGroup(aniFile);
break;
case FileQualityFilterType.SUBSTREAMCOUNT:
result &= CheckSubStreamCount(file);
break;
case FileQualityFilterType.VERSION:
if (aniFile == null)
{
return(false);
}
result &= CheckDeprecated(aniFile);
break;
case FileQualityFilterType.VIDEOCODEC:
if (aniFile == null)
{
return(false);
}
result &= CheckVideoCodec(file);
break;
}
}
return(result);
}
private static Tuple <int, int> GetResolutionInternal(SVR_VideoLocal videoLocal, SVR_AniDB_File aniFile)
{
string[] res = aniFile?.File_VideoResolution?.Split('x');
if (res == null || res.Length != 2 || res[0] == "0" && res[1] == "0")
{
var stream = videoLocal?.Media?.Parts?.SelectMany(a => a.Streams)
?.FirstOrDefault(a => a.StreamType == "1");
if (stream != null)
{
res = new[] { stream.Width, stream.Height }
}
;
}
if (res == null || res.Length != 2 || res[0] == "0" && res[1] == "0")
{
return(null);
}
if (!int.TryParse(res[0], out int oldWidth))
{
return(null);
}
if (!int.TryParse(res[1], out int oldHeight))
{
return(null);
}
if (oldWidth == 0 || oldHeight == 0)
{
return(null);
}
return(new Tuple <int, int>(oldWidth, oldHeight));
}
#endregion
}
public static string GetResolution(SVR_VideoLocal videoLocal, SVR_AniDB_File aniFile)
{
return(GetResolution(GetResolutionInternal(videoLocal, aniFile)));
}
private static int CompareResolutionTo(SVR_VideoLocal newFile, SVR_VideoLocal oldFile, SVR_AniDB_File newAniFile, SVR_AniDB_File oldAniFile)
{
string oldRes = GetResolution(oldFile, oldAniFile);
string newRes = GetResolution(newFile, newAniFile);
if (newRes == null || oldRes == null)
{
return(0);
}
if (!Settings._resolutions.Contains(newRes))
{
return(0);
}
if (!Settings._resolutions.Contains(oldRes))
{
return(-1);
}
int newIndex = Array.IndexOf(Settings._resolutions, newRes);
int oldIndex = Array.IndexOf(Settings._resolutions, oldRes);
return(newIndex.CompareTo(oldIndex));
}
private static bool CheckResolution(SVR_VideoLocal videoLocal, SVR_AniDB_File aniFile)
{
Tuple <int, int> resTuple = GetResolutionInternal(videoLocal, aniFile);
string res = GetResolution(resTuple);
if (res == null)
{
return(false);
}
int resArea = resTuple.Item1 * resTuple.Item2;
FileQualityFilterOperationType operationType = Settings.RequiredResolutionOperator;
switch (operationType)
{
case FileQualityFilterOperationType.EQUALS:
return(res.Equals(Settings.RequiredResolutions.FirstOrDefault()));
case FileQualityFilterOperationType.GREATER_EQ:
List <int> keysGT = ResolutionArea.Keys.Where(a => resArea >= a).ToList();
keysGT.AddRange(ResolutionAreaOld.Keys.Where(a => resArea >= a));
List <string> valuesGT = new List <string>();
foreach (int key in keysGT)
{
if (ResolutionArea.ContainsKey(key))
{
valuesGT.Add(ResolutionArea[key]);
}
if (ResolutionAreaOld.ContainsKey(key))
{
valuesGT.Add(ResolutionAreaOld[key]);
}
}
if (valuesGT.FindInEnumerable(Settings.RequiredResolutions))
{
return(true);
}
break;
case FileQualityFilterOperationType.LESS_EQ:
List <int> keysLT = ResolutionArea.Keys.Where(a => resArea <= a).ToList();
keysLT.AddRange(ResolutionAreaOld.Keys.Where(a => resArea <= a));
List <string> valuesLT = new List <string>();
foreach (int key in keysLT)
{
if (ResolutionArea.ContainsKey(key))
{
valuesLT.Add(ResolutionArea[key]);
}
if (ResolutionAreaOld.ContainsKey(key))
{
valuesLT.Add(ResolutionAreaOld[key]);
}
}
if (valuesLT.FindInEnumerable(Settings.RequiredResolutions))
{
return(true);
}
break;
case FileQualityFilterOperationType.IN:
return(Settings.RequiredResolutions.Contains(res));
case FileQualityFilterOperationType.NOTIN:
return(!Settings.RequiredResolutions.Contains(res));
}
return(false);
}
public string GetFileName(SVR_VideoLocal video)
{
if (video == null)
{
return("*Error: Unable to access file");
}
// base data
string scriptCode = _rawScript.Script;
SVR_AniDB_File file = video.GetAniDBFile();
List <AniDB_Episode> episodes = new List <AniDB_Episode>();
SVR_AniDB_Anime anime;
// error handling (not all is in this)
if (file == null)
{
var epIntmdry = video.GetAnimeEpisodes();
if (epIntmdry.Count == 0)
{
return("*Error: Unable to get episode for file");
}
episodes.Add(epIntmdry[0].AniDB_Episode);
anime = RepoFactory.AniDB_Anime.GetByAnimeID(episodes[0].AnimeID);
if (anime == null)
{
return("*Error: Unable to get anime for file");
}
}
else
{
episodes = file.Episodes;
if (episodes.Count == 0)
{
return("*Error: Unable to get episode for file");
}
anime = RepoFactory.AniDB_Anime.GetByAnimeID(episodes[0].AnimeID);
if (anime == null)
{
return("*Error: Unable to get anime for file");
}
}
if (string.IsNullOrEmpty(scriptCode))
{
return("*Error: No script available for renamer");
}
// TODO: start filling up tables (crudely, will implement better later (maybe (probably (hopefully))))
// future me: actually this makes for a very strong renamer
UserData.RegisterType <SVR_VideoLocal>();
UserData.RegisterType <SVR_AniDB_File>();
UserData.RegisterType <AniDB_Episode>();
UserData.RegisterType <SVR_AniDB_Anime>();
UserData.RegisterType <AniDB_Anime_Title>();
UserData.RegisterExtensionType(typeof(Models)); // extends SVR_AniDB_Anime
UserData.RegisterType <EpisodeType>();
DynValue videoLuaObject = UserData.Create(video);
DynValue fileLuaObject = UserData.Create(file);
DynValue episodeLuaObject = UserData.Create(episodes[0]);
DynValue animeLuaObject = UserData.Create(anime);
_script.Globals["EpisodeType"] = UserData.CreateStatic <EpisodeType>();
_script.Globals.Set("video", videoLuaObject);
_script.Globals.Set("file", fileLuaObject);
_script.Globals.Set("episode", episodeLuaObject);
_script.Globals["episodes"] = episodes;
_script.Globals.Set("anime", animeLuaObject);
// (I have)/(deodex has) no idea if this is dangerous. It's as if the interpreter is as strong as the C# method
// (as long as the private-public keywords are actually used correctly, it should be fine)
// make sure name is a string by initializing it as an empty one (in case its not used)
_script.Globals["name"] = "";
// run the script, get the variable "name" (str)
_script.DoString(scriptCode);
string scriptResult = _script.Globals.Get("name").ToString();
// Substring used since the string is being wrapped with "" for some reason
if (string.IsNullOrEmpty(scriptResult.Substring(1, scriptResult.Length - 2)))
{
return("*Error: the new filename is empty (script error)");
}
//copy-pasted
string pathToVid = video.GetBestVideoLocalPlace().FilePath;
if (string.IsNullOrEmpty(pathToVid))
{
return("*Error: Unable to get the file's old filename");
}
string ext = Path.GetExtension(pathToVid); //Prefer VideoLocal_Place as this is more accurate.
if (string.IsNullOrEmpty(ext))
{
return("*Error: Unable to get the file's extension"); // fail if we get a blank extension, something went wrong
}
string renameTo = $"{scriptResult.Substring(1, scriptResult.Length - 2).Replace("`", "'")}{ext}";
if (File.Exists(Path.Combine(Path.GetDirectoryName(pathToVid), renameTo))) // Has potential null error, im bad pls fix ty
{
return("*Error: A file with this filename already exists");
}
return(Utils.ReplaceInvalidFolderNameCharacters(renameTo));
}
private void ProcessFile_AniDB(SVR_VideoLocal vidLocal)
{
logger.Trace($"Checking for AniDB_File record for: {vidLocal.Hash} --- {vidLocal.FileName}");
// check if we already have this AniDB_File info in the database
lock (vidLocal)
{
SVR_AniDB_File aniFile = null;
if (!ForceAniDB)
{
aniFile = RepoFactory.AniDB_File.GetByHashAndFileSize(vidLocal.Hash, vlocal.FileSize);
if (aniFile == null)
{
logger.Trace("AniDB_File record not found");
}
}
// If cross refs were wiped, but the AniDB_File was not, we unfortunately need to requery the info
List <CrossRef_File_Episode> crossRefs = RepoFactory.CrossRef_File_Episode.GetByHash(vidLocal.Hash);
if (crossRefs == null || crossRefs.Count == 0)
{
aniFile = null;
}
Dictionary <int, bool> animeIDs = new Dictionary <int, bool>();
if (aniFile == null || aniFile.FileSize != vlocal.FileSize)
{
aniFile = TryGetAniDBFileFromAniDB(vidLocal, animeIDs);
}
// if we still haven't got the AniDB_File Info we try the web cache or local records
if (aniFile == null)
{
// check if we have any records from previous imports
crossRefs = RepoFactory.CrossRef_File_Episode.GetByHash(vidLocal.Hash);
if (crossRefs == null || crossRefs.Count == 0)
{
// lets see if we can find the episode/anime info from the web cache
if (TryGetCrossRefsFromWebCache(vidLocal, ref crossRefs))
{
return;
}
}
// we assume that all episodes belong to the same anime
foreach (CrossRef_File_Episode xref in crossRefs)
{
AniDB_Episode ep = RepoFactory.AniDB_Episode.GetByEpisodeID(xref.EpisodeID);
if (animeIDs.ContainsKey(xref.AnimeID))
{
animeIDs[xref.AnimeID] = ep == null;
}
else
{
animeIDs.Add(xref.AnimeID, ep == null);
}
}
}
else
{
// check if we have the episode info
// if we don't, we will need to re-download the anime info (which also has episode info)
if (aniFile.EpisodeCrossRefs.Count == 0)
{
aniFile.Episodes.Select(a => a.AnimeID).Distinct().ForEach(animeID =>
{
if (animeIDs.ContainsKey(animeID))
{
animeIDs[animeID] = true;
}
else
{
animeIDs.Add(animeID, true);
}
});
// if we have the AniDB file, but no cross refs it means something has been broken
logger.Debug($"Could not find any cross ref records for: {vidLocal.ED2KHash}");
}
else
{
foreach (CrossRef_File_Episode xref in aniFile.EpisodeCrossRefs)
{
AniDB_Episode ep = RepoFactory.AniDB_Episode.GetByEpisodeID(xref.EpisodeID);
if (animeIDs.ContainsKey(xref.AnimeID))
{
animeIDs[xref.AnimeID] = ep == null;
}
else
{
animeIDs.Add(xref.AnimeID, ep == null);
}
}
}
}
PopulateAnimeForFile(vidLocal, animeIDs);
// We do this inside, as the info will not be available as needed otherwise
List <SVR_VideoLocal> videoLocals =
aniFile?.EpisodeIDs?.SelectMany(a => RepoFactory.VideoLocal.GetByAniDBEpisodeID(a))
.Where(b => b != null)
.ToList();
if (videoLocals != null)
{
if (ServerSettings.Instance.Import.UseExistingFileWatchedStatus)
{
// Copy over watched states
foreach (var user in RepoFactory.JMMUser.GetAll())
{
var watchedVideo = videoLocals.FirstOrDefault(a =>
a?.GetUserRecord(user.JMMUserID)?.WatchedDate != null);
// No files that are watched
if (watchedVideo == null)
{
continue;
}
var watchedRecord = watchedVideo.GetUserRecord(user.JMMUserID);
var userRecord = vidLocal.GetUserRecord(user.JMMUserID) ?? new VideoLocal_User
{
JMMUserID = user.JMMUserID,
VideoLocalID = vidLocal.VideoLocalID,
};
userRecord.WatchedDate = watchedRecord.WatchedDate;
userRecord.ResumePosition = watchedRecord.ResumePosition;
RepoFactory.VideoLocalUser.Save(userRecord);
}
}
// update stats for groups and series. The series are not saved until here, so it's absolutely necessary!!
animeIDs.Keys.ForEach(SVR_AniDB_Anime.UpdateStatsByAnimeID);
if (ServerSettings.Instance.FileQualityFilterEnabled)
{
videoLocals.Sort(FileQualityFilter.CompareTo);
List <SVR_VideoLocal> keep = videoLocals
.Take(FileQualityFilter.Settings.MaxNumberOfFilesToKeep)
.ToList();
foreach (SVR_VideoLocal vl2 in keep)
{
videoLocals.Remove(vl2);
}
if (!FileQualityFilter.Settings.AllowDeletionOfImportedFiles &&
videoLocals.Contains(vidLocal))
{
videoLocals.Remove(vidLocal);
}
videoLocals = videoLocals.Where(a => !FileQualityFilter.CheckFileKeep(a)).ToList();
videoLocals.ForEach(a => a.Places.ForEach(b => b.RemoveAndDeleteFile()));
}
}
vidLocal.Places.ForEach(a => { a.RenameAndMoveAsRequired(); });
// Add this file to the users list
if (ServerSettings.Instance.AniDb.MyList_AddFiles && !SkipMyList && vidLocal.MyListID <= 0)
{
new CommandRequest_AddFileToMyList(vidLocal.ED2KHash).Save();
}
}
}
private void ProcessFile_AniDB(SVR_VideoLocal vidLocal)
{
logger.Trace("Checking for AniDB_File record for: {0} --- {1}", vidLocal.Hash, vidLocal.FileName);
// check if we already have this AniDB_File info in the database
lock (vidLocal)
{
SVR_AniDB_File aniFile = null;
if (!ForceAniDB)
{
aniFile = RepoFactory.AniDB_File.GetByHashAndFileSize(vidLocal.Hash, vlocal.FileSize);
if (aniFile == null)
{
logger.Trace("AniDB_File record not found");
}
}
int animeID = 0;
if (aniFile == null)
{
// get info from AniDB
logger.Debug("Getting AniDB_File record from AniDB....");
Raw_AniDB_File fileInfo = ShokoService.AnidbProcessor.GetFileInfo(vidLocal);
if (fileInfo != null)
{
// check if we already have a record
aniFile = RepoFactory.AniDB_File.GetByHashAndFileSize(vidLocal.Hash, vlocal.FileSize);
if (aniFile == null)
{
aniFile = new SVR_AniDB_File();
}
SVR_AniDB_File.Populate(aniFile, fileInfo);
//overwrite with local file name
string localFileName = vidLocal.FileName;
aniFile.FileName = localFileName;
RepoFactory.AniDB_File.Save(aniFile, false);
aniFile.CreateLanguages();
aniFile.CreateCrossEpisodes(localFileName);
if (!string.IsNullOrEmpty(fileInfo.OtherEpisodesRAW))
{
string[] epIDs = fileInfo.OtherEpisodesRAW.Split(',');
foreach (string epid in epIDs)
{
int id = 0;
if (int.TryParse(epid, out id))
{
CommandRequest_GetEpisode cmdEp = new CommandRequest_GetEpisode(id);
cmdEp.Save();
}
}
}
animeID = aniFile.AnimeID;
}
}
bool missingEpisodes = false;
// if we still haven't got the AniDB_File Info we try the web cache or local records
if (aniFile == null)
{
// check if we have any records from previous imports
List <CrossRef_File_Episode> crossRefs = RepoFactory.CrossRef_File_Episode.GetByHash(vidLocal.Hash);
if (crossRefs == null || crossRefs.Count == 0)
{
// lets see if we can find the episode/anime info from the web cache
if (ServerSettings.WebCache_XRefFileEpisode_Get)
{
List <Shoko.Models.Azure.Azure_CrossRef_File_Episode> xrefs =
AzureWebAPI.Get_CrossRefFileEpisode(vidLocal);
crossRefs = new List <CrossRef_File_Episode>();
if (xrefs == null || xrefs.Count == 0)
{
logger.Debug(
"Cannot find AniDB_File record or get cross ref from web cache record so exiting: {0}",
vidLocal.ED2KHash);
return;
}
else
{
foreach (Shoko.Models.Azure.Azure_CrossRef_File_Episode xref in xrefs)
{
CrossRef_File_Episode xrefEnt = new CrossRef_File_Episode();
xrefEnt.Hash = vidLocal.ED2KHash;
xrefEnt.FileName = vidLocal.FileName;
xrefEnt.FileSize = vidLocal.FileSize;
xrefEnt.CrossRefSource = (int)CrossRefSource.WebCache;
xrefEnt.AnimeID = xref.AnimeID;
xrefEnt.EpisodeID = xref.EpisodeID;
xrefEnt.Percentage = xref.Percentage;
xrefEnt.EpisodeOrder = xref.EpisodeOrder;
bool duplicate = false;
foreach (CrossRef_File_Episode xrefcheck in crossRefs)
{
if (xrefcheck.AnimeID == xrefEnt.AnimeID &&
xrefcheck.EpisodeID == xrefEnt.EpisodeID &&
xrefcheck.Hash == xrefEnt.Hash)
{
duplicate = true;
}
}
if (!duplicate)
{
crossRefs.Add(xrefEnt);
// in this case we need to save the cross refs manually as AniDB did not provide them
RepoFactory.CrossRef_File_Episode.Save(xrefEnt);
}
}
}
}
else
{
logger.Debug("Cannot get AniDB_File record so exiting: {0}", vidLocal.ED2KHash);
return;
}
}
// we assume that all episodes belong to the same anime
foreach (CrossRef_File_Episode xref in crossRefs)
{
animeID = xref.AnimeID;
AniDB_Episode ep = RepoFactory.AniDB_Episode.GetByEpisodeID(xref.EpisodeID);
if (ep == null)
{
missingEpisodes = true;
}
}
}
else
{
// check if we have the episode info
// if we don't, we will need to re-download the anime info (which also has episode info)
if (aniFile.EpisodeCrossRefs.Count == 0)
{
animeID = aniFile.AnimeID;
// if we have the anidb file, but no cross refs it means something has been broken
logger.Debug("Could not find any cross ref records for: {0}", vidLocal.ED2KHash);
missingEpisodes = true;
}
else
{
foreach (CrossRef_File_Episode xref in aniFile.EpisodeCrossRefs)
{
AniDB_Episode ep = RepoFactory.AniDB_Episode.GetByEpisodeID(xref.EpisodeID);
if (ep == null)
{
missingEpisodes = true;
}
animeID = xref.AnimeID;
}
}
}
// get from DB
SVR_AniDB_Anime anime = RepoFactory.AniDB_Anime.GetByAnimeID(animeID);
bool animeRecentlyUpdated = false;
if (anime != null)
{
TimeSpan ts = DateTime.Now - anime.DateTimeUpdated;
if (ts.TotalHours < 4)
{
animeRecentlyUpdated = true;
}
}
// even if we are missing episode info, don't get data more than once every 4 hours
// this is to prevent banning
if (missingEpisodes && !animeRecentlyUpdated)
{
logger.Debug("Getting Anime record from AniDB....");
anime = ShokoService.AnidbProcessor.GetAnimeInfoHTTP(animeID, true, ServerSettings.AutoGroupSeries);
}
// create the group/series/episode records if needed
SVR_AnimeSeries ser = null;
if (anime != null)
{
logger.Debug("Creating groups, series and episodes....");
// check if there is an AnimeSeries Record associated with this AnimeID
ser = RepoFactory.AnimeSeries.GetByAnimeID(animeID);
if (ser == null)
{
// create a new AnimeSeries record
ser = anime.CreateAnimeSeriesAndGroup();
}
ser.CreateAnimeEpisodes();
// check if we have any group status data for this associated anime
// if not we will download it now
if (RepoFactory.AniDB_GroupStatus.GetByAnimeID(anime.AnimeID).Count == 0)
{
CommandRequest_GetReleaseGroupStatus cmdStatus =
new CommandRequest_GetReleaseGroupStatus(anime.AnimeID, false);
cmdStatus.Save();
}
// update stats
ser.EpisodeAddedDate = DateTime.Now;
RepoFactory.AnimeSeries.Save(ser, false, false);
foreach (SVR_AnimeGroup grp in ser.AllGroupsAbove)
{
grp.EpisodeAddedDate = DateTime.Now;
RepoFactory.AnimeGroup.Save(grp, true, false);
}
if (ServerSettings.FileQualityFilterEnabled)
{
// We do this inside, as the info will not be available as needed otherwise
List <SVR_VideoLocal> videoLocals =
aniFile?.EpisodeIDs?.SelectMany(a => RepoFactory.VideoLocal.GetByAniDBEpisodeID(a))
.Where(b => b != null)
.ToList();
if (videoLocals != null)
{
videoLocals.Sort(FileQualityFilter.CompareTo);
List <SVR_VideoLocal> keep = videoLocals
.Take(FileQualityFilter.Settings.MaxNumberOfFilesToKeep)
.ToList();
foreach (SVR_VideoLocal vl2 in keep)
{
videoLocals.Remove(vl2);
}
if (!FileQualityFilter.Settings.AllowDeletionOfImportedFiles &&
videoLocals.Contains(vidLocal))
{
videoLocals.Remove(vidLocal);
}
videoLocals = videoLocals.Where(a => !FileQualityFilter.CheckFileKeep(a)).ToList();
foreach (SVR_VideoLocal toDelete in videoLocals)
{
toDelete.Places.ForEach(a => a.RemoveAndDeleteFile());
}
}
}
}
vidLocal.Places.ForEach(a => { a.RenameAndMoveAsRequired(); });
// update stats for groups and series
if (ser != null)
{
// update all the groups above this series in the heirarchy
ser.QueueUpdateStats();
//StatsCache.Instance.UpdateUsingSeries(ser.AnimeSeriesID);
}
// Add this file to the users list
if (ServerSettings.AniDB_MyList_AddFiles)
{
CommandRequest_AddFileToMyList cmd = new CommandRequest_AddFileToMyList(vidLocal.ED2KHash);
cmd.Save();
}
}
}
public override void ProcessCommand()
{
logger.Info("Processing CommandRequest_SyncMyList");
try
{
// we will always assume that an anime was downloaded via http first
ScheduledUpdate sched =
RepoFactory.ScheduledUpdate.GetByUpdateType((int)ScheduledUpdateType.AniDBMyListSync);
if (sched == null)
{
sched = new ScheduledUpdate
{
UpdateType = (int)ScheduledUpdateType.AniDBMyListSync,
UpdateDetails = string.Empty
};
}
else
{
int freqHours = Utils.GetScheduledHours(ServerSettings.Instance.AniDb.MyList_UpdateFrequency);
// if we have run this in the last 24 hours and are not forcing it, then exit
TimeSpan tsLastRun = DateTime.Now - sched.LastUpdate;
if (tsLastRun.TotalHours < freqHours)
{
if (!ForceRefresh)
{
return;
}
}
}
// Get the list from AniDB
AniDBHTTPCommand_GetMyList cmd = new AniDBHTTPCommand_GetMyList();
cmd.Init(ServerSettings.Instance.AniDb.Username, ServerSettings.Instance.AniDb.Password);
AniDBUDPResponseCode ev = cmd.Process();
if (ev != AniDBUDPResponseCode.GotMyListHTTP)
{
logger.Warn("AniDB did not return a successful code: " + ev);
return;
}
int totalItems = 0;
int watchedItems = 0;
int modifiedItems = 0;
// Add missing files on AniDB
var onlineFiles = cmd.MyListItems.ToLookup(a => a.FileID);
var dictAniFiles = RepoFactory.AniDB_File.GetAll().ToLookup(a => a.Hash);
int missingFiles = 0;
foreach (SVR_VideoLocal vid in RepoFactory.VideoLocal.GetAll()
.Where(a => !string.IsNullOrEmpty(a.Hash)).ToList())
{
// Does it have a linked AniFile
if (!dictAniFiles.Contains(vid.Hash))
{
continue;
}
int fileID = dictAniFiles[vid.Hash].FirstOrDefault()?.FileID ?? 0;
if (fileID == 0)
{
continue;
}
// Is it in MyList
if (onlineFiles.Contains(fileID))
{
Raw_AniDB_MyListFile file = onlineFiles[fileID].FirstOrDefault(a => a != null);
if (file != null)
{
if (vid.MyListID == 0)
{
vid.MyListID = file.ListID;
RepoFactory.VideoLocal.Save(vid);
}
// Update file state if deleted
if (file.State != (int)ServerSettings.Instance.AniDb.MyList_StorageState)
{
int seconds = Commons.Utils.AniDB.GetAniDBDateAsSeconds(file.WatchedDate);
CommandRequest_UpdateMyListFileStatus cmdUpdateFile =
new CommandRequest_UpdateMyListFileStatus(vid.Hash, file.WatchedDate.HasValue,
false,
seconds);
cmdUpdateFile.Save();
}
continue;
}
}
// means we have found a file in our local collection, which is not recorded online
if (ServerSettings.Instance.AniDb.MyList_AddFiles)
{
CommandRequest_AddFileToMyList cmdAddFile = new CommandRequest_AddFileToMyList(vid.Hash);
cmdAddFile.Save();
}
missingFiles++;
}
logger.Info($"MYLIST Missing Files: {missingFiles} Added to queue for inclusion");
List <SVR_JMMUser> aniDBUsers = RepoFactory.JMMUser.GetAniDBUsers();
LinkedHashSet <SVR_AnimeSeries> modifiedSeries = new LinkedHashSet <SVR_AnimeSeries>();
// Remove Missing Files and update watched states (single loop)
List <int> filesToRemove = new List <int>();
foreach (Raw_AniDB_MyListFile myitem in cmd.MyListItems)
{
try
{
totalItems++;
if (myitem.IsWatched)
{
watchedItems++;
}
string hash = string.Empty;
SVR_AniDB_File anifile = RepoFactory.AniDB_File.GetByFileID(myitem.FileID);
if (anifile != null)
{
hash = anifile.Hash;
}
else
{
// look for manually linked files
List <CrossRef_File_Episode> xrefs =
RepoFactory.CrossRef_File_Episode.GetByEpisodeID(myitem.EpisodeID);
foreach (CrossRef_File_Episode xref in xrefs)
{
if (xref.CrossRefSource == (int)CrossRefSource.AniDB)
{
continue;
}
hash = xref.Hash;
break;
}
}
// We couldn't evem find a hash, so remove it
if (string.IsNullOrEmpty(hash))
{
filesToRemove.Add(myitem.ListID);
continue;
}
// If there's no video local, we don't have it
SVR_VideoLocal vl = RepoFactory.VideoLocal.GetByHash(hash);
if (vl == null)
{
filesToRemove.Add(myitem.ListID);
continue;
}
foreach (SVR_JMMUser juser in aniDBUsers)
{
bool localStatus = false;
// doesn't matter which anidb user we use
int jmmUserID = juser.JMMUserID;
VideoLocal_User userRecord = vl.GetUserRecord(juser.JMMUserID);
if (userRecord != null)
{
localStatus = userRecord.WatchedDate.HasValue;
}
string action = string.Empty;
if (localStatus == myitem.IsWatched)
{
continue;
}
// localStatus and AniDB Status are different
DateTime?watchedDate = myitem.WatchedDate ?? DateTime.Now;
if (localStatus)
{
// local = watched, anidb = unwatched
if (ServerSettings.Instance.AniDb.MyList_ReadUnwatched)
{
modifiedItems++;
vl.ToggleWatchedStatus(false, false, watchedDate,
false, jmmUserID, false,
true);
action = "Used AniDB Status";
}
else if (ServerSettings.Instance.AniDb.MyList_SetWatched)
{
vl.ToggleWatchedStatus(true, true, userRecord.WatchedDate, false, jmmUserID,
false, true);
}
}
else
{
// means local is un-watched, and anidb is watched
if (ServerSettings.Instance.AniDb.MyList_ReadWatched)
{
modifiedItems++;
vl.ToggleWatchedStatus(true, false, watchedDate, false,
jmmUserID, false, true);
action = "Updated Local record to Watched";
}
else if (ServerSettings.Instance.AniDb.MyList_SetUnwatched)
{
vl.ToggleWatchedStatus(false, true, watchedDate, false, jmmUserID,
false, true);
}
}
vl.GetAnimeEpisodes().Select(a => a.GetAnimeSeries()).Where(a => a != null)
.DistinctBy(a => a.AnimeSeriesID).ForEach(a => modifiedSeries.Add(a));
logger.Info(
$"MYLISTDIFF:: File {vl.FileName} - Local Status = {localStatus}, AniDB Status = {myitem.IsWatched} --- {action}");
}
}
catch (Exception ex)
{
logger.Error($"A MyList Item threw an error while syncing: {ex}");
}
}
// Actually remove the files
if (filesToRemove.Count > 0)
{
foreach (int listID in filesToRemove)
{
CommandRequest_DeleteFileFromMyList deleteCommand =
new CommandRequest_DeleteFileFromMyList(listID);
deleteCommand.Save();
}
logger.Info($"MYLIST Missing Files: {filesToRemove.Count} Added to queue for deletion");
}
modifiedSeries.ForEach(a => a.QueueUpdateStats());
logger.Info($"Process MyList: {totalItems} Items, {missingFiles} Added, {filesToRemove.Count} Deleted, {watchedItems} Watched, {modifiedItems} Modified");
sched.LastUpdate = DateTime.Now;
RepoFactory.ScheduledUpdate.Save(sched);
}
catch (Exception ex)
{
logger.Error(ex, "Error processing CommandRequest_SyncMyList: {0} ", ex);
}
}
private static Tuple <int, int> GetResolutionInternal(SVR_VideoLocal videoLocal, SVR_AniDB_File aniFile)
{
string[] res = aniFile?.File_VideoResolution?.Split('x');
int oldHeight = 0, oldWidth = 0;
if (res != null && res.Length == 2 && res[0] == "0" && res[1] == "0")
{
int.TryParse(res[0], out oldWidth);
int.TryParse(res[1], out oldHeight);
}
if (oldHeight == 0 || oldWidth == 0)
{
var stream = videoLocal?.Media?.Parts?.SelectMany(a => a.Streams)
.FirstOrDefault(a => a.StreamType == 1);
if (stream != null)
{
oldWidth = stream.Width;
oldHeight = stream.Height;
}
}
if (oldHeight == 0 || oldWidth == 0)
{
return(null);
}
return(new Tuple <int, int>(oldWidth, oldHeight));
}
public override void ProcessCommand()
{
logger.Info("Processing CommandRequest_SyncMyList");
try
{
// we will always assume that an anime was downloaded via http first
ScheduledUpdate sched =
RepoFactory.ScheduledUpdate.GetByUpdateType((int)ScheduledUpdateType.AniDBMyListSync);
if (sched == null)
{
sched = new ScheduledUpdate
{
UpdateType = (int)ScheduledUpdateType.AniDBMyListSync,
UpdateDetails = ""
};
}
else
{
int freqHours = Utils.GetScheduledHours(ServerSettings.AniDB_MyList_UpdateFrequency);
// if we have run this in the last 24 hours and are not forcing it, then exit
TimeSpan tsLastRun = DateTime.Now - sched.LastUpdate;
if (tsLastRun.TotalHours < freqHours)
{
if (!ForceRefresh)
{
return;
}
}
}
AniDBHTTPCommand_GetMyList cmd = new AniDBHTTPCommand_GetMyList();
cmd.Init(ServerSettings.AniDB_Username, ServerSettings.AniDB_Password);
enHelperActivityType ev = cmd.Process();
if (ev != enHelperActivityType.GotMyListHTTP || cmd.MyListItems.Count <= 1)
{
return;
}
int totalItems = 0;
int watchedItems = 0;
int modifiedItems = 0;
double pct = 0;
// 2. find files locally for the user, which are not recorded on anidb
// and then add them to anidb
Dictionary <int, Raw_AniDB_MyListFile> onlineFiles = new Dictionary <int, Raw_AniDB_MyListFile>();
foreach (Raw_AniDB_MyListFile myitem in cmd.MyListItems)
{
onlineFiles[myitem.FileID] = myitem;
}
Dictionary <string, SVR_AniDB_File> dictAniFiles = new Dictionary <string, SVR_AniDB_File>();
IReadOnlyList <SVR_AniDB_File> allAniFiles = RepoFactory.AniDB_File.GetAll();
foreach (SVR_AniDB_File anifile in allAniFiles)
{
dictAniFiles[anifile.Hash] = anifile;
}
int missingFiles = 0;
foreach (SVR_VideoLocal vid in RepoFactory.VideoLocal.GetAll()
.Where(a => !string.IsNullOrEmpty(a.Hash)).ToList())
{
if (!dictAniFiles.ContainsKey(vid.Hash))
{
continue;
}
int fileID = dictAniFiles[vid.Hash].FileID;
if (onlineFiles.ContainsKey(fileID))
{
continue;
}
// means we have found a file in our local collection, which is not recorded online
CommandRequest_AddFileToMyList cmdAddFile = new CommandRequest_AddFileToMyList(vid.Hash);
cmdAddFile.Save();
missingFiles++;
}
logger.Info($"MYLIST Missing Files: {missingFiles} Added to queue for inclusion");
List <SVR_JMMUser> aniDBUsers = RepoFactory.JMMUser.GetAniDBUsers();
// 1 . sync mylist items
foreach (Raw_AniDB_MyListFile myitem in cmd.MyListItems)
{
// ignore files mark as deleted by the user
if (myitem.State == (int)AniDBFileStatus.Deleted)
{
continue;
}
totalItems++;
if (myitem.IsWatched)
{
watchedItems++;
}
//calculate percentage
pct = totalItems / (double)cmd.MyListItems.Count * 100;
string spct = pct.ToString("#0.0");
string hash = string.Empty;
SVR_AniDB_File anifile = RepoFactory.AniDB_File.GetByFileID(myitem.FileID);
if (anifile != null)
{
hash = anifile.Hash;
}
else
{
// look for manually linked files
List <CrossRef_File_Episode> xrefs =
RepoFactory.CrossRef_File_Episode.GetByEpisodeID(myitem.EpisodeID);
foreach (CrossRef_File_Episode xref in xrefs)
{
if (xref.CrossRefSource == (int)CrossRefSource.AniDB)
{
continue;
}
hash = xref.Hash;
break;
}
}
if (string.IsNullOrEmpty(hash))
{
continue;
}
// find the video associated with this record
SVR_VideoLocal vl = RepoFactory.VideoLocal.GetByHash(hash);
if (vl == null)
{
continue;
}
foreach (SVR_JMMUser juser in aniDBUsers)
{
bool localStatus = false;
// doesn't matter which anidb user we use
int jmmUserID = juser.JMMUserID;
VideoLocal_User userRecord = vl.GetUserRecord(juser.JMMUserID);
if (userRecord != null)
{
localStatus = userRecord.WatchedDate.HasValue;
}
string action = "";
if (localStatus == myitem.IsWatched)
{
continue;
}
if (localStatus)
{
// local = watched, anidb = unwatched
if (ServerSettings.AniDB_MyList_ReadUnwatched)
{
modifiedItems++;
vl.ToggleWatchedStatus(myitem.IsWatched, false, myitem.WatchedDate,
false, false, jmmUserID, false,
true);
action = "Used AniDB Status";
}
}
else
{
// means local is un-watched, and anidb is watched
if (ServerSettings.AniDB_MyList_ReadWatched)
{
modifiedItems++;
vl.ToggleWatchedStatus(true, false, myitem.WatchedDate, false, false,
jmmUserID, false, true);
action = "Updated Local record to Watched";
}
}
string msg =
$"MYLISTDIFF:: File {vl.FileName} - Local Status = {localStatus}, AniDB Status = {myitem.IsWatched} --- {action}";
logger.Info(msg);
}
//string msg = string.Format("MYLIST:: File {0} - Local Status = {1}, AniDB Status = {2} --- {3}",
// vl.FullServerPath, localStatus, myitem.IsWatched, action);
//logger.Info(msg);
}
// now update all stats
Importer.UpdateAllStats();
logger.Info("Process MyList: {0} Items, {1} Watched, {2} Modified", totalItems, watchedItems,
modifiedItems);
sched.LastUpdate = DateTime.Now;
RepoFactory.ScheduledUpdate.Save(sched);
}
catch (Exception ex)
{
logger.Error(ex, "Error processing CommandRequest_SyncMyList: {0} ", ex.Message);
}
}
private void ProcessFile_AniDB(SVR_VideoLocal vidLocal)
{
logger.Trace($"Checking for AniDB_File record for: {vidLocal.Hash} --- {vidLocal.FileName}");
// check if we already have this AniDB_File info in the database
lock (vidLocal)
{
SVR_AniDB_File aniFile = null;
if (!ForceAniDB)
{
aniFile = Repo.Instance.AniDB_File.GetByHashAndFileSize(vidLocal.Hash, vlocal.FileSize);
if (aniFile == null)
{
logger.Trace("AniDB_File record not found");
}
}
// If cross refs were wiped, but the AniDB_File was not, we unfortunately need to requery the info
List <CrossRef_File_Episode> crossRefs = Repo.Instance.CrossRef_File_Episode.GetByHash(vidLocal.Hash);
if (crossRefs == null || crossRefs.Count == 0)
{
aniFile = null;
}
int animeID = 0;
if (aniFile == null)
{
// get info from AniDB
logger.Debug("Getting AniDB_File record from AniDB....");
// check if we already have a record
using (var upd = Repo.Instance.AniDB_File.BeginAddOrUpdate(() => Repo.Instance.AniDB_File.GetByHashAndFileSize(vidLocal.Hash, vlocal.FileSize)))
{
bool skip = false;
if (!upd.IsUpdate)
{
Raw_AniDB_File fileInfo = ShokoService.AnidbProcessor.GetFileInfo(vidLocal);
if (fileInfo != null)
{
upd.Entity.Populate_RA(fileInfo);
}
else
{
skip = true;
}
}
if (!skip)
{
//overwrite with local file name
string localFileName = vidLocal.GetBestVideoLocalPlace()?.FullServerPath;
localFileName = !string.IsNullOrEmpty(localFileName)
? Path.GetFileName(localFileName)
: vidLocal.FileName;
upd.Entity.FileName = localFileName;
aniFile = upd.Commit(false);
aniFile.CreateLanguages();
aniFile.CreateCrossEpisodes(localFileName);
animeID = aniFile.AnimeID;
}
upd.Commit();
}
}
bool missingEpisodes = false;
// if we still haven't got the AniDB_File Info we try the web cache or local records
if (aniFile == null)
{
// check if we have any records from previous imports
crossRefs = Repo.Instance.CrossRef_File_Episode.GetByHash(vidLocal.Hash);
if (crossRefs == null || crossRefs.Count == 0)
{
// lets see if we can find the episode/anime info from the web cache
if (ServerSettings.Instance.WebCache.XRefFileEpisode_Get)
{
List <Azure_CrossRef_File_Episode> xrefs =
AzureWebAPI.Get_CrossRefFileEpisode(vidLocal);
crossRefs = new List <CrossRef_File_Episode>();
if (xrefs == null || xrefs.Count == 0)
{
logger.Debug(
$"Cannot find AniDB_File record or get cross ref from web cache record so exiting: {vidLocal.ED2KHash}");
return;
}
string fileName = vidLocal.GetBestVideoLocalPlace()?.FullServerPath;
fileName = !string.IsNullOrEmpty(fileName) ? Path.GetFileName(fileName) : vidLocal.FileName;
foreach (Azure_CrossRef_File_Episode xref in xrefs)
{
CrossRef_File_Episode xrefEnt = new CrossRef_File_Episode
{
Hash = vidLocal.ED2KHash,
FileName = fileName,
FileSize = vidLocal.FileSize,
CrossRefSource = (int)CrossRefSource.WebCache,
AnimeID = xref.AnimeID,
EpisodeID = xref.EpisodeID,
Percentage = xref.Percentage,
EpisodeOrder = xref.EpisodeOrder
};
bool duplicate = false;
foreach (CrossRef_File_Episode xrefcheck in crossRefs)
{
if (xrefcheck.AnimeID == xrefEnt.AnimeID &&
xrefcheck.EpisodeID == xrefEnt.EpisodeID &&
xrefcheck.Hash == xrefEnt.Hash)
{
duplicate = true;
}
}
if (!duplicate)
{
crossRefs.Add(xrefEnt);
// in this case we need to save the cross refs manually as AniDB did not provide them
Repo.Instance.CrossRef_File_Episode.BeginAdd(xrefEnt).Commit();
}
}
}
else
{
logger.Debug($"Cannot get AniDB_File record so exiting: {vidLocal.ED2KHash}");
return;
}
}
// we assume that all episodes belong to the same anime
foreach (CrossRef_File_Episode xref in crossRefs)
{
animeID = xref.AnimeID;
AniDB_Episode ep = Repo.Instance.AniDB_Episode.GetByEpisodeID(xref.EpisodeID);
if (ep == null)
{
missingEpisodes = true;
}
}
}
else
{
// check if we have the episode info
// if we don't, we will need to re-download the anime info (which also has episode info)
if (aniFile.EpisodeCrossRefs.Count == 0)
{
animeID = aniFile.AnimeID;
// if we have the anidb file, but no cross refs it means something has been broken
logger.Debug($"Could not find any cross ref records for: {vidLocal.ED2KHash}");
missingEpisodes = true;
}
else
{
foreach (CrossRef_File_Episode xref in aniFile.EpisodeCrossRefs)
{
AniDB_Episode ep = Repo.Instance.AniDB_Episode.GetByEpisodeID(xref.EpisodeID);
if (ep == null)
{
missingEpisodes = true;
}
animeID = xref.AnimeID;
}
}
}
// get from DB
SVR_AniDB_Anime anime = Repo.Instance.AniDB_Anime.GetByAnimeID(animeID);
var update = Repo.Instance.AniDB_AnimeUpdate.GetByAnimeID(animeID);
bool animeRecentlyUpdated = false;
if (anime != null && update != null)
{
TimeSpan ts = DateTime.Now - update.UpdatedAt;
if (ts.TotalHours < 4)
{
animeRecentlyUpdated = true;
}
}
else
{
missingEpisodes = true;
}
// even if we are missing episode info, don't get data more than once every 4 hours
// this is to prevent banning
if (missingEpisodes && !animeRecentlyUpdated)
{
logger.Debug("Getting Anime record from AniDB....");
anime = ShokoService.AnidbProcessor.GetAnimeInfoHTTP(animeID, true,
ServerSettings.Instance.AutoGroupSeries || ServerSettings.Instance.AniDb.DownloadRelatedAnime);
}
// create the group/series/episode records if needed
if (anime != null)
{
logger.Debug("Creating groups, series and episodes....");
// check if there is an AnimeSeries Record associated with this AnimeID
SVR_AnimeSeries ser;
using (var upd = Repo.Instance.AnimeSeries.BeginAddOrUpdate(
() => Repo.Instance.AnimeSeries.GetByAnimeID(animeID),
() => anime.CreateAnimeSeriesAndGroup()
))
{
upd.Entity.CreateAnimeEpisodes();
// check if we have any group status data for this associated anime
// if not we will download it now
if (Repo.Instance.AniDB_GroupStatus.GetByAnimeID(anime.AnimeID).Count == 0)
{
CommandRequest_GetReleaseGroupStatus cmdStatus =
new CommandRequest_GetReleaseGroupStatus(anime.AnimeID, false);
cmdStatus.Save();
}
// update stats
upd.Entity.EpisodeAddedDate = DateTime.Now;
ser = upd.Commit();
}
Repo.Instance.AnimeGroup.BatchAction(ser.AllGroupsAbove, ser.AllGroupsAbove.Count, (grp, _) => grp.EpisodeAddedDate = DateTime.Now, (true, false, false));
// We do this inside, as the info will not be available as needed otherwise
List <SVR_VideoLocal> videoLocals =
aniFile?.EpisodeIDs?.SelectMany(a => Repo.Instance.VideoLocal.GetByAniDBEpisodeID(a))
.Where(b => b != null)
.ToList();
if (videoLocals != null)
{
// Copy over watched states
foreach (var user in Repo.Instance.JMMUser.GetAll())
{
var watchedVideo = videoLocals.FirstOrDefault(a =>
a?.GetUserRecord(user.JMMUserID)?.WatchedDate != null);
// No files that are watched
if (watchedVideo == null)
{
continue;
}
var watchedRecord = watchedVideo.GetUserRecord(user.JMMUserID);
using (var upd = Repo.Instance.VideoLocal_User.BeginAddOrUpdate(
() => vidLocal.GetUserRecord(user.JMMUserID),
() => new VideoLocal_User {
JMMUserID = user.JMMUserID, VideoLocalID = vidLocal.VideoLocalID
}
))
{
upd.Entity.WatchedDate = watchedRecord.WatchedDate;
upd.Entity.ResumePosition = watchedRecord.ResumePosition;
upd.Commit();
}
}
if (ServerSettings.Instance.FileQualityFilterEnabled)
{
videoLocals.Sort(FileQualityFilter.CompareTo);
List <SVR_VideoLocal> keep = videoLocals
.Take(FileQualityFilter.Settings.MaxNumberOfFilesToKeep)
.ToList();
foreach (SVR_VideoLocal vl2 in keep)
{
videoLocals.Remove(vl2);
}
if (!FileQualityFilter.Settings.AllowDeletionOfImportedFiles &&
videoLocals.Contains(vidLocal))
{
videoLocals.Remove(vidLocal);
}
videoLocals = videoLocals.Where(a => !FileQualityFilter.CheckFileKeep(a)).ToList();
videoLocals.ForEach(a => a.Places.ForEach(b => b.RemoveAndDeleteFile()));
}
}
// update stats for groups and series
// update all the groups above this series in the heirarchy
SVR_AniDB_Anime.UpdateStatsByAnimeID(animeID);
}
else
{
logger.Warn($"Unable to create AniDB_Anime for file: {vidLocal.FileName}");
}
vidLocal.Places.ForEach(a => { a.RenameAndMoveAsRequired(); });
// Add this file to the users list
if (ServerSettings.Instance.AniDb.MyList_AddFiles)
{
CommandRequest_AddFileToMyList cmd = new CommandRequest_AddFileToMyList(vidLocal.ED2KHash);
cmd.Save();
}
}
}
public override void Run(IProgress <ICommand> progress = null)
{
logger.Info("Get AniDB file info: {0}", VideoLocalID);
try
{
ReportInit(progress);
if (vlocal == null)
{
vlocal = Repo.Instance.VideoLocal.GetByID(VideoLocalID);
}
if (vlocal == null)
{
ReportError(progress, $"VideoLocal with id {VideoLocalID} not found");
return;
}
lock (vlocal)
{
SVR_AniDB_File aniFile = Repo.Instance.AniDB_File.GetByHashAndFileSize(vlocal.Hash, vlocal.FileSize);
ReportUpdate(progress, 20);
Raw_AniDB_File fileInfo = null;
if (aniFile == null || ForceAniDB)
{
fileInfo = ShokoService.AnidbProcessor.GetFileInfo(vlocal);
}
ReportUpdate(progress, 40);
if (fileInfo != null)
{
SVR_AniDB_File file = aniFile;
using (var upd = Repo.Instance.AniDB_File.BeginAddOrUpdate(file))
{
upd.Entity.Populate_RA(fileInfo);
//overwrite with local file name
string localFileName = vlocal.Info;
upd.Entity.FileName = localFileName;
aniFile = upd.Commit();
}
ReportUpdate(progress, 55);
aniFile.CreateLanguages();
ReportUpdate(progress, 70);
aniFile.CreateCrossEpisodes(vlocal.Info);
ReportUpdate(progress, 85);
//TODO: Look at why this might be worth it?
//SVR_AniDB_Anime anime = Repo.Instance.AniDB_Anime.GetByAnimeID(aniFile.AnimeID);
//if (anime != null) Repo.Instance.AniDB_Anime.Save(anime);
SVR_AnimeSeries series = Repo.Instance.AnimeSeries.GetByAnimeID(aniFile.AnimeID);
series.UpdateStats(true, true, true);
}
ReportFinish(progress);
}
}
catch (Exception ex)
{
ReportError(progress, $"Error processing Command AniDb.GetFile: {VideoLocalID} - {ex}", ex);
}
}
private void FillHashesAgainstAniDBRepo(SVR_VideoLocal v)
{
if (!string.IsNullOrEmpty(v.ED2KHash))
{
SVR_AniDB_File f = Repo.Instance.AniDB_File.GetByHash(v.ED2KHash);
if (f != null)
{
if (!string.IsNullOrEmpty(f.CRC))
{
v.CRC32 = f.CRC.ToUpperInvariant();
}
if (!string.IsNullOrEmpty(f.SHA1))
{
v.SHA1 = f.SHA1.ToUpperInvariant();
}
if (!string.IsNullOrEmpty(f.MD5))
{
v.MD5 = f.MD5.ToUpperInvariant();
}
return;
}
}
if (!string.IsNullOrEmpty(v.SHA1))
{
SVR_AniDB_File f = Repo.Instance.AniDB_File.GetBySHA1(v.SHA1);
if (f != null)
{
if (!string.IsNullOrEmpty(f.CRC))
{
v.CRC32 = f.CRC.ToUpperInvariant();
}
if (!string.IsNullOrEmpty(f.Hash))
{
v.ED2KHash = f.Hash.ToUpperInvariant();
}
if (!string.IsNullOrEmpty(f.MD5))
{
v.MD5 = f.MD5.ToUpperInvariant();
}
return;
}
}
if (!string.IsNullOrEmpty(v.MD5))
{
SVR_AniDB_File f = Repo.Instance.AniDB_File.GetByMD5(v.MD5);
if (f != null)
{
if (!string.IsNullOrEmpty(f.CRC))
{
v.CRC32 = f.CRC.ToUpperInvariant();
}
if (!string.IsNullOrEmpty(f.Hash))
{
v.ED2KHash = f.Hash.ToUpperInvariant();
}
if (!string.IsNullOrEmpty(f.SHA1))
{
v.SHA1 = f.SHA1.ToUpperInvariant();
}
}
}
}
public override void Run(IProgress <ICommand> progress = null)
{
try
{
ReportInit(progress);
List <SVR_VideoLocal> allfiles = Repo.Instance.VideoLocal.GetAll().ToList();
List <SVR_VideoLocal> missfiles = allfiles.Where(
a =>
string.IsNullOrEmpty(a.CRC32) || string.IsNullOrEmpty(a.SHA1) ||
string.IsNullOrEmpty(a.MD5) || a.SHA1 == "0000000000000000000000000000000000000000" ||
a.MD5 == "00000000000000000000000000000000")
.ToList();
ReportUpdate(progress, 10);
List <SVR_VideoLocal> withfiles = allfiles.Except(missfiles).ToList();
Dictionary <int, (string ed2k, string crc32, string md5, string sha1)> updates = new Dictionary <int, (string ed2k, string crc32, string md5, string sha1)>();
//Check if we can populate md5,sha and crc from AniDB_Files
List <SVR_VideoLocal> vm = missfiles.ToList();
for (int x = 0; x < vm.Count; x++)
{
double prog = (x + 1) * 60 / (double)missfiles.Count;
SVR_VideoLocal v = vm[x];
PrettyDescription.QueueState = QueueStateEnum.CheckingFile;
PrettyDescription.ExtraParams = new [] { v.Info };
SVR_AniDB_File file = Repo.Instance.AniDB_File.GetByHash(v.ED2KHash);
if (file != null)
{
if (!string.IsNullOrEmpty(file.CRC) && !string.IsNullOrEmpty(file.SHA1) &&
!string.IsNullOrEmpty(file.MD5))
{
updates[v.VideoLocalID] = (file.Hash, file.CRC, file.MD5, file.SHA1);
missfiles.Remove(v);
withfiles.Add(v);
ReportUpdate(progress, 10 + prog);
continue;
}
}
WebCache_FileHash ls = WebCacheAPI.Instance.GetHash(WebCache_HashType.ED2K, v.ED2KHash);
if (ls != null)
{
updates[v.VideoLocalID] = (ls.ED2K.ToUpperInvariant(), ls.CRC32.ToUpperInvariant(), ls.MD5.ToUpperInvariant(), ls.SHA1.ToUpperInvariant());
missfiles.Remove(v);
}
ReportUpdate(progress, 10 + prog);
}
//We need to recalculate the sha1, md5 and crc32 of the missing ones.
List <ICommand> tohash = new List <ICommand>();
foreach (SVR_VideoLocal v in missfiles)
{
try
{
SVR_VideoLocal_Place p = v.GetBestVideoLocalPlace();
if (p != null && p.ImportFolder.CloudID == 0)
{
tohash.Add(new CmdHashFile(p.FullServerPath, true));
}
}
catch
{
//Ignored
}
}
Queue.Instance.AddRange(tohash);
ReportUpdate(progress, 80);
if (updates.Count > 0)
{
using (var upd = Repo.Instance.VideoLocal.BeginBatchUpdate(() => Repo.Instance.VideoLocal.GetMany(updates.Keys)))
{
foreach (SVR_VideoLocal v in upd)
{
(string ed2k, string crc32, string md5, string sha1)t = updates[v.VideoLocalID];
v.Hash = t.ed2k;
v.CRC32 = t.crc32;
v.MD5 = t.md5;
v.SHA1 = t.sha1;
upd.Update(v);
}
upd.Commit();
}
}
ReportUpdate(progress, 90);
//Send the hashes
WebCacheAPI.Instance.AddHashes(withfiles.Select(a => a.ToHashRequest()));
logger.Info("Sync Hashes Complete");
ReportFinish(progress);
}
catch (Exception ex)
{
ReportError(progress, $"Error processing ServerSyncHashes: {ex}", ex);
}
}
