public static void FixHashes() { try { VideoLocalRepository repVids = new VideoLocalRepository(); foreach (VideoLocal vid in repVids.GetAll()) { bool fixedHash = false; if (vid.CRC32.Equals("00000000")) { vid.CRC32 = null; fixedHash = true; } if (vid.MD5.Equals("00000000000000000000000000000000")) { vid.MD5 = null; fixedHash = true; } if (vid.SHA1.Equals("0000000000000000000000000000000000000000")) { vid.SHA1 = null; fixedHash = true; } if (fixedHash) { repVids.Save(vid); logger.Info("Fixed hashes on file: {0}", vid.FullServerPath); } } } catch (Exception ex) { logger.ErrorException(ex.ToString(), ex); } }
void workerMediaInfo_DoWork(object sender, DoWorkEventArgs e) { VideoLocalRepository repVidLocals = new VideoLocalRepository(); // first build a list of files that we already know about, as we don't want to process them again List<VideoLocal> filesAll = repVidLocals.GetAll(); Dictionary<string, VideoLocal> dictFilesExisting = new Dictionary<string, VideoLocal>(); foreach (VideoLocal vl in filesAll) { CommandRequest_ReadMediaInfo cr = new CommandRequest_ReadMediaInfo(vl.VideoLocalID); cr.Save(); } }
public static void RemoveRecordsWithoutPhysicalFiles() { VideoLocalRepository repVidLocals = new VideoLocalRepository(); CrossRef_File_EpisodeRepository repXRefs = new CrossRef_File_EpisodeRepository(); // get a full list of files List<VideoLocal> filesAll = repVidLocals.GetAll(); foreach (VideoLocal vl in filesAll) { if (!File.Exists(vl.FullServerPath)) { // delete video local record logger.Info("RemoveRecordsWithoutPhysicalFiles : {0}", vl.FullServerPath); repVidLocals.Delete(vl.VideoLocalID); CommandRequest_DeleteFileFromMyList cmdDel = new CommandRequest_DeleteFileFromMyList(vl.Hash, vl.FileSize); cmdDel.Save(); } } UpdateAllStats(); }
public static void RunImport_NewFiles() { VideoLocalRepository repVidLocals = new VideoLocalRepository(); // first build a list of files that we already know about, as we don't want to process them again List<VideoLocal> filesAll = repVidLocals.GetAll(); Dictionary<string, VideoLocal> dictFilesExisting = new Dictionary<string, VideoLocal>(); foreach (VideoLocal vl in filesAll) { try { dictFilesExisting[vl.FullServerPath] = vl; } catch (Exception ex) { string msg = string.Format("Error RunImport_NewFiles XREF: {0} - {1}", vl.ToStringDetailed(), ex.ToString()); logger.Info(msg); //throw; } } // Steps for processing a file // 1. Check if it is a video file // 2. Check if we have a VideoLocal record for that file // ......... // get a complete list of files List<string> fileList = new List<string>(); ImportFolderRepository repNetShares = new ImportFolderRepository(); foreach (ImportFolder share in repNetShares.GetAll()) { logger.Debug("ImportFolder: {0} || {1}", share.ImportFolderName, share.ImportFolderLocation); try { Utils.GetFilesForImportFolder(share.ImportFolderLocation, ref fileList); } catch (Exception ex) { logger.ErrorException(ex.ToString(), ex); } } // get a list fo files that we haven't processed before List<string> fileListNew = new List<string>(); foreach (string fileName in fileList) { if (!dictFilesExisting.ContainsKey(fileName)) fileListNew.Add(fileName); } // get a list of all the shares we are looking at int filesFound = 0, videosFound = 0; int i = 0; // get a list of all files in the share foreach (string fileName in fileListNew) { i++; filesFound++; logger.Info("Processing File {0}/{1} --- {2}", i, fileList.Count, fileName); if (!FileHashHelper.IsVideo(fileName)) continue; videosFound++; CommandRequest_HashFile cr_hashfile = new CommandRequest_HashFile(fileName, false); cr_hashfile.Save(); } logger.Debug("Found {0} files", filesFound); logger.Debug("Found {0} videos", videosFound); }
public static void RunImport_IntegrityCheck() { VideoLocalRepository repVidLocals = new VideoLocalRepository(); AniDB_FileRepository repAniFile = new AniDB_FileRepository(); AniDB_EpisodeRepository repAniEps = new AniDB_EpisodeRepository(); AniDB_AnimeRepository repAniAnime = new AniDB_AnimeRepository(); // files which don't have a valid import folder List<VideoLocal> filesToDelete = repVidLocals.GetVideosWithoutImportFolder(); foreach (VideoLocal vl in filesToDelete) repVidLocals.Delete(vl.VideoLocalID); // files which have not been hashed yet // or files which do not have a VideoInfo record List<VideoLocal> filesToHash = repVidLocals.GetVideosWithoutHash(); Dictionary<int, VideoLocal> dictFilesToHash = new Dictionary<int, VideoLocal>(); foreach (VideoLocal vl in filesToHash) { dictFilesToHash[vl.VideoLocalID] = vl; CommandRequest_HashFile cmd = new CommandRequest_HashFile(vl.FullServerPath, false); cmd.Save(); } List<VideoLocal> filesToRehash = repVidLocals.GetVideosWithoutVideoInfo(); Dictionary<int, VideoLocal> dictFilesToRehash = new Dictionary<int, VideoLocal>(); foreach (VideoLocal vl in filesToHash) { dictFilesToRehash[vl.VideoLocalID] = vl; // don't use if it is in the previous list if (!dictFilesToHash.ContainsKey(vl.VideoLocalID)) { try { CommandRequest_HashFile cmd = new CommandRequest_HashFile(vl.FullServerPath, false); cmd.Save(); } catch (Exception ex) { string msg = string.Format("Error RunImport_IntegrityCheck XREF: {0} - {1}", vl.ToStringDetailed(), ex.ToString()); logger.Info(msg); } } } // files which have been hashed, but don't have an associated episode List<VideoLocal> filesWithoutEpisode = repVidLocals.GetVideosWithoutEpisode(); Dictionary<int, VideoLocal> dictFilesWithoutEpisode = new Dictionary<int, VideoLocal>(); foreach (VideoLocal vl in filesWithoutEpisode) dictFilesWithoutEpisode[vl.VideoLocalID] = vl; // check that all the episode data is populated List<VideoLocal> filesAll = repVidLocals.GetAll(); Dictionary<string, VideoLocal> dictFilesAllExisting = new Dictionary<string, VideoLocal>(); foreach (VideoLocal vl in filesAll) { try { dictFilesAllExisting[vl.FullServerPath] = vl; } catch (Exception ex) { string msg = string.Format("Error RunImport_IntegrityCheck XREF: {0} - {1}", vl.ToStringDetailed(), ex.ToString()); logger.Error(msg); continue; } // check if it has an episode if (dictFilesWithoutEpisode.ContainsKey(vl.VideoLocalID)) { CommandRequest_ProcessFile cmd = new CommandRequest_ProcessFile(vl.VideoLocalID, false); cmd.Save(); continue; } // if the file is not manually associated, then check for AniDB_File info AniDB_File aniFile = repAniFile.GetByHash(vl.Hash); foreach (CrossRef_File_Episode xref in vl.EpisodeCrossRefs) { if (xref.CrossRefSource != (int)CrossRefSource.AniDB) continue; if (aniFile == null) { CommandRequest_ProcessFile cmd = new CommandRequest_ProcessFile(vl.VideoLocalID, false); cmd.Save(); continue; } } if (aniFile == null) continue; // the cross ref is created before the actually episode data is downloaded // so lets check for that bool missingEpisodes = false; foreach (CrossRef_File_Episode xref in aniFile.EpisodeCrossRefs) { AniDB_Episode ep = repAniEps.GetByEpisodeID(xref.EpisodeID); if (ep == null) missingEpisodes = true; } if (missingEpisodes) { // this will then download the anime etc CommandRequest_ProcessFile cmd = new CommandRequest_ProcessFile(vl.VideoLocalID, false); cmd.Save(); continue; } } }
public static void RunImport_ScanFolder(int importFolderID) { // get a complete list of files List<string> fileList = new List<string>(); ImportFolderRepository repFolders = new ImportFolderRepository(); int filesFound = 0, videosFound = 0; int i = 0; try { ImportFolder fldr = repFolders.GetByID(importFolderID); if (fldr == null) return; VideoLocalRepository repVidLocals = new VideoLocalRepository(); // first build a list of files that we already know about, as we don't want to process them again List<VideoLocal> filesAll = repVidLocals.GetAll(); Dictionary<string, VideoLocal> dictFilesExisting = new Dictionary<string, VideoLocal>(); foreach (VideoLocal vl in filesAll) { try { dictFilesExisting[vl.FullServerPath] = vl; } catch (Exception ex) { string msg = string.Format("Error RunImport_ScanFolder XREF: {0} - {1}", vl.ToStringDetailed(), ex.ToString()); logger.Info(msg); } } logger.Debug("ImportFolder: {0} || {1}", fldr.ImportFolderName, fldr.ImportFolderLocation); Utils.GetFilesForImportFolder(fldr.ImportFolderLocation, ref fileList); // get a list of all files in the share foreach (string fileName in fileList) { i++; if (dictFilesExisting.ContainsKey(fileName)) continue; filesFound++; logger.Info("Processing File {0}/{1} --- {2}", i, fileList.Count, fileName); if (!FileHashHelper.IsVideo(fileName)) continue; videosFound++; CommandRequest_HashFile cr_hashfile = new CommandRequest_HashFile(fileName, false); cr_hashfile.Save(); } logger.Debug("Found {0} new files", filesFound); logger.Debug("Found {0} videos", videosFound); } catch (Exception ex) { logger.ErrorException(ex.ToString(), ex); } }
public override void ProcessCommand() { logger.Info("Processing CommandRequest_SyncMyList"); try { // we will always assume that an anime was downloaded via http first ScheduledUpdateRepository repSched = new ScheduledUpdateRepository(); AniDB_FileRepository repAniFile = new AniDB_FileRepository(); VideoLocalRepository repVidLocals = new VideoLocalRepository(); ScheduledUpdate sched = repSched.GetByUpdateType((int)ScheduledUpdateType.AniDBMyListSync); if (sched == null) { sched = new ScheduledUpdate(); sched.UpdateType = (int)ScheduledUpdateType.AniDBMyListSync; sched.UpdateDetails = ""; } else { int freqHours = Utils.GetScheduledHours(ServerSettings.AniDB_MyList_UpdateFrequency); // if we have run this in the last 24 hours and are not forcing it, then exit TimeSpan tsLastRun = DateTime.Now - sched.LastUpdate; if (tsLastRun.TotalHours < freqHours) { if (!ForceRefresh) return; } } AniDBHTTPCommand_GetMyList cmd = new AniDBHTTPCommand_GetMyList(); cmd.Init(ServerSettings.AniDB_Username, ServerSettings.AniDB_Password); enHelperActivityType ev = cmd.Process(); if (ev == enHelperActivityType.GotMyListHTTP && cmd.MyListItems.Count > 1) { int totalItems = 0; int watchedItems = 0; int modifiedItems = 0; double pct = 0; // 2. find files locally for the user, which are not recorded on anidb // and then add them to anidb Dictionary<int, Raw_AniDB_MyListFile> onlineFiles = new Dictionary<int, Raw_AniDB_MyListFile>(); foreach (Raw_AniDB_MyListFile myitem in cmd.MyListItems) onlineFiles[myitem.FileID] = myitem; Dictionary<string, AniDB_File> dictAniFiles = new Dictionary<string, AniDB_File>(); List<AniDB_File> allAniFiles = repAniFile.GetAll(); foreach (AniDB_File anifile in allAniFiles) dictAniFiles[anifile.Hash] = anifile; int missingFiles = 0; foreach (VideoLocal vid in repVidLocals.GetAll()) { if (!dictAniFiles.ContainsKey(vid.Hash)) continue; int fileID = dictAniFiles[vid.Hash].FileID; if (!onlineFiles.ContainsKey(fileID)) { // means we have found a file in our local collection, which is not recorded online CommandRequest_AddFileToMyList cmdAddFile = new CommandRequest_AddFileToMyList(vid.Hash); cmdAddFile.Save(); missingFiles++; } } logger.Info(string.Format("MYLIST Missing Files: {0} Added to queue for inclusion", missingFiles)); JMMUserRepository repUsers = new JMMUserRepository(); List<JMMUser> aniDBUsers = repUsers.GetAniDBUsers(); VideoLocal_UserRepository repVidUsers = new VideoLocal_UserRepository(); CrossRef_File_EpisodeRepository repFileEp = new CrossRef_File_EpisodeRepository(); // 1 . sync mylist items foreach (Raw_AniDB_MyListFile myitem in cmd.MyListItems) { // ignore files mark as deleted by the user if (myitem.State == (int)AniDBFileStatus.Deleted) continue; totalItems++; if (myitem.IsWatched) watchedItems++; //calculate percentage pct = (double)totalItems / (double)cmd.MyListItems.Count * (double)100; string spct = pct.ToString("#0.0"); string hash = string.Empty; AniDB_File anifile = repAniFile.GetByFileID(myitem.FileID); if (anifile != null) hash = anifile.Hash; else { // look for manually linked files List<CrossRef_File_Episode> xrefs = repFileEp.GetByEpisodeID(myitem.EpisodeID); foreach (CrossRef_File_Episode xref in xrefs) { if (xref.CrossRefSource != (int)CrossRefSource.AniDB) { hash = xref.Hash; break; } } } if (!string.IsNullOrEmpty(hash)) { // find the video associated with this record VideoLocal vl = repVidLocals.GetByHash(hash); if (vl == null) continue; foreach (JMMUser juser in aniDBUsers) { bool localStatus = false; int? jmmUserID = null; // doesn't matter which anidb user we use jmmUserID = juser.JMMUserID; VideoLocal_User userRecord = vl.GetUserRecord(juser.JMMUserID); if (userRecord != null) localStatus = true; string action = ""; if (localStatus != myitem.IsWatched) { if (localStatus == true) { // local = watched, anidb = unwatched if (ServerSettings.AniDB_MyList_ReadUnwatched) { modifiedItems++; if (jmmUserID.HasValue) vl.ToggleWatchedStatus(myitem.IsWatched, false, myitem.WatchedDate, false, false, jmmUserID.Value, false, true); action = "Used AniDB Status"; } } else { // means local is un-watched, and anidb is watched if (ServerSettings.AniDB_MyList_ReadWatched) { modifiedItems++; if (jmmUserID.HasValue) vl.ToggleWatchedStatus(true, false, myitem.WatchedDate, false, false, jmmUserID.Value, false, true); action = "Updated Local record to Watched"; } } string msg = string.Format("MYLISTDIFF:: File {0} - Local Status = {1}, AniDB Status = {2} --- {3}", vl.FullServerPath, localStatus, myitem.IsWatched, action); logger.Info(msg); } } //string msg = string.Format("MYLIST:: File {0} - Local Status = {1}, AniDB Status = {2} --- {3}", // vl.FullServerPath, localStatus, myitem.IsWatched, action); //logger.Info(msg); } } // now update all stats Importer.UpdateAllStats(); logger.Info("Process MyList: {0} Items, {1} Watched, {2} Modified", totalItems, watchedItems, modifiedItems); sched.LastUpdate = DateTime.Now; repSched.Save(sched); } } catch (Exception ex) { logger.Error("Error processing CommandRequest_SyncMyList: {0} ", ex.ToString()); return; } }