public static void FixHashes()
{
try
{
VideoLocalRepository repVids = new VideoLocalRepository();
foreach (VideoLocal vid in repVids.GetAll())
{
bool fixedHash = false;
if (vid.CRC32.Equals("00000000"))
{
vid.CRC32 = null;
fixedHash = true;
}
if (vid.MD5.Equals("00000000000000000000000000000000"))
{
vid.MD5 = null;
fixedHash = true;
}
if (vid.SHA1.Equals("0000000000000000000000000000000000000000"))
{
vid.SHA1 = null;
fixedHash = true;
}
if (fixedHash)
{
repVids.Save(vid);
logger.Info("Fixed hashes on file: {0}", vid.FullServerPath);
}
}
}
catch (Exception ex)
{
logger.ErrorException(ex.ToString(), ex);
}
}
void workerMediaInfo_DoWork(object sender, DoWorkEventArgs e)
{
VideoLocalRepository repVidLocals = new VideoLocalRepository();
// first build a list of files that we already know about, as we don't want to process them again
List<VideoLocal> filesAll = repVidLocals.GetAll();
Dictionary<string, VideoLocal> dictFilesExisting = new Dictionary<string, VideoLocal>();
foreach (VideoLocal vl in filesAll)
{
CommandRequest_ReadMediaInfo cr = new CommandRequest_ReadMediaInfo(vl.VideoLocalID);
cr.Save();
}
}
public static void RemoveRecordsWithoutPhysicalFiles()
{
VideoLocalRepository repVidLocals = new VideoLocalRepository();
CrossRef_File_EpisodeRepository repXRefs = new CrossRef_File_EpisodeRepository();
// get a full list of files
List<VideoLocal> filesAll = repVidLocals.GetAll();
foreach (VideoLocal vl in filesAll)
{
if (!File.Exists(vl.FullServerPath))
{
// delete video local record
logger.Info("RemoveRecordsWithoutPhysicalFiles : {0}", vl.FullServerPath);
repVidLocals.Delete(vl.VideoLocalID);
CommandRequest_DeleteFileFromMyList cmdDel = new CommandRequest_DeleteFileFromMyList(vl.Hash, vl.FileSize);
cmdDel.Save();
}
}
UpdateAllStats();
}
public static void RunImport_NewFiles()
{
VideoLocalRepository repVidLocals = new VideoLocalRepository();
// first build a list of files that we already know about, as we don't want to process them again
List<VideoLocal> filesAll = repVidLocals.GetAll();
Dictionary<string, VideoLocal> dictFilesExisting = new Dictionary<string, VideoLocal>();
foreach (VideoLocal vl in filesAll)
{
try
{
dictFilesExisting[vl.FullServerPath] = vl;
}
catch (Exception ex)
{
string msg = string.Format("Error RunImport_NewFiles XREF: {0} - {1}", vl.ToStringDetailed(), ex.ToString());
logger.Info(msg);
//throw;
}
}
// Steps for processing a file
// 1. Check if it is a video file
// 2. Check if we have a VideoLocal record for that file
// .........
// get a complete list of files
List<string> fileList = new List<string>();
ImportFolderRepository repNetShares = new ImportFolderRepository();
foreach (ImportFolder share in repNetShares.GetAll())
{
logger.Debug("ImportFolder: {0} || {1}", share.ImportFolderName, share.ImportFolderLocation);
try
{
Utils.GetFilesForImportFolder(share.ImportFolderLocation, ref fileList);
}
catch (Exception ex)
{
logger.ErrorException(ex.ToString(), ex);
}
}
// get a list fo files that we haven't processed before
List<string> fileListNew = new List<string>();
foreach (string fileName in fileList)
{
if (!dictFilesExisting.ContainsKey(fileName))
fileListNew.Add(fileName);
}
// get a list of all the shares we are looking at
int filesFound = 0, videosFound = 0;
int i = 0;
// get a list of all files in the share
foreach (string fileName in fileListNew)
{
i++;
filesFound++;
logger.Info("Processing File {0}/{1} --- {2}", i, fileList.Count, fileName);
if (!FileHashHelper.IsVideo(fileName)) continue;
videosFound++;
CommandRequest_HashFile cr_hashfile = new CommandRequest_HashFile(fileName, false);
cr_hashfile.Save();
}
logger.Debug("Found {0} files", filesFound);
logger.Debug("Found {0} videos", videosFound);
}
public static void RunImport_IntegrityCheck()
{
VideoLocalRepository repVidLocals = new VideoLocalRepository();
AniDB_FileRepository repAniFile = new AniDB_FileRepository();
AniDB_EpisodeRepository repAniEps = new AniDB_EpisodeRepository();
AniDB_AnimeRepository repAniAnime = new AniDB_AnimeRepository();
// files which don't have a valid import folder
List<VideoLocal> filesToDelete = repVidLocals.GetVideosWithoutImportFolder();
foreach (VideoLocal vl in filesToDelete)
repVidLocals.Delete(vl.VideoLocalID);
// files which have not been hashed yet
// or files which do not have a VideoInfo record
List<VideoLocal> filesToHash = repVidLocals.GetVideosWithoutHash();
Dictionary<int, VideoLocal> dictFilesToHash = new Dictionary<int, VideoLocal>();
foreach (VideoLocal vl in filesToHash)
{
dictFilesToHash[vl.VideoLocalID] = vl;
CommandRequest_HashFile cmd = new CommandRequest_HashFile(vl.FullServerPath, false);
cmd.Save();
}
List<VideoLocal> filesToRehash = repVidLocals.GetVideosWithoutVideoInfo();
Dictionary<int, VideoLocal> dictFilesToRehash = new Dictionary<int, VideoLocal>();
foreach (VideoLocal vl in filesToHash)
{
dictFilesToRehash[vl.VideoLocalID] = vl;
// don't use if it is in the previous list
if (!dictFilesToHash.ContainsKey(vl.VideoLocalID))
{
try
{
CommandRequest_HashFile cmd = new CommandRequest_HashFile(vl.FullServerPath, false);
cmd.Save();
}
catch (Exception ex)
{
string msg = string.Format("Error RunImport_IntegrityCheck XREF: {0} - {1}", vl.ToStringDetailed(), ex.ToString());
logger.Info(msg);
}
}
}
// files which have been hashed, but don't have an associated episode
List<VideoLocal> filesWithoutEpisode = repVidLocals.GetVideosWithoutEpisode();
Dictionary<int, VideoLocal> dictFilesWithoutEpisode = new Dictionary<int, VideoLocal>();
foreach (VideoLocal vl in filesWithoutEpisode)
dictFilesWithoutEpisode[vl.VideoLocalID] = vl;
// check that all the episode data is populated
List<VideoLocal> filesAll = repVidLocals.GetAll();
Dictionary<string, VideoLocal> dictFilesAllExisting = new Dictionary<string, VideoLocal>();
foreach (VideoLocal vl in filesAll)
{
try
{
dictFilesAllExisting[vl.FullServerPath] = vl;
}
catch (Exception ex)
{
string msg = string.Format("Error RunImport_IntegrityCheck XREF: {0} - {1}", vl.ToStringDetailed(), ex.ToString());
logger.Error(msg);
continue;
}
// check if it has an episode
if (dictFilesWithoutEpisode.ContainsKey(vl.VideoLocalID))
{
CommandRequest_ProcessFile cmd = new CommandRequest_ProcessFile(vl.VideoLocalID, false);
cmd.Save();
continue;
}
// if the file is not manually associated, then check for AniDB_File info
AniDB_File aniFile = repAniFile.GetByHash(vl.Hash);
foreach (CrossRef_File_Episode xref in vl.EpisodeCrossRefs)
{
if (xref.CrossRefSource != (int)CrossRefSource.AniDB) continue;
if (aniFile == null)
{
CommandRequest_ProcessFile cmd = new CommandRequest_ProcessFile(vl.VideoLocalID, false);
cmd.Save();
continue;
}
}
if (aniFile == null) continue;
// the cross ref is created before the actually episode data is downloaded
// so lets check for that
bool missingEpisodes = false;
foreach (CrossRef_File_Episode xref in aniFile.EpisodeCrossRefs)
{
AniDB_Episode ep = repAniEps.GetByEpisodeID(xref.EpisodeID);
if (ep == null) missingEpisodes = true;
}
if (missingEpisodes)
{
// this will then download the anime etc
CommandRequest_ProcessFile cmd = new CommandRequest_ProcessFile(vl.VideoLocalID, false);
cmd.Save();
continue;
}
}
}
public static void RunImport_ScanFolder(int importFolderID)
{
// get a complete list of files
List<string> fileList = new List<string>();
ImportFolderRepository repFolders = new ImportFolderRepository();
int filesFound = 0, videosFound = 0;
int i = 0;
try
{
ImportFolder fldr = repFolders.GetByID(importFolderID);
if (fldr == null) return;
VideoLocalRepository repVidLocals = new VideoLocalRepository();
// first build a list of files that we already know about, as we don't want to process them again
List<VideoLocal> filesAll = repVidLocals.GetAll();
Dictionary<string, VideoLocal> dictFilesExisting = new Dictionary<string, VideoLocal>();
foreach (VideoLocal vl in filesAll)
{
try
{
dictFilesExisting[vl.FullServerPath] = vl;
}
catch (Exception ex)
{
string msg = string.Format("Error RunImport_ScanFolder XREF: {0} - {1}", vl.ToStringDetailed(), ex.ToString());
logger.Info(msg);
}
}
logger.Debug("ImportFolder: {0} || {1}", fldr.ImportFolderName, fldr.ImportFolderLocation);
Utils.GetFilesForImportFolder(fldr.ImportFolderLocation, ref fileList);
// get a list of all files in the share
foreach (string fileName in fileList)
{
i++;
if (dictFilesExisting.ContainsKey(fileName)) continue;
filesFound++;
logger.Info("Processing File {0}/{1} --- {2}", i, fileList.Count, fileName);
if (!FileHashHelper.IsVideo(fileName)) continue;
videosFound++;
CommandRequest_HashFile cr_hashfile = new CommandRequest_HashFile(fileName, false);
cr_hashfile.Save();
}
logger.Debug("Found {0} new files", filesFound);
logger.Debug("Found {0} videos", videosFound);
}
catch (Exception ex)
{
logger.ErrorException(ex.ToString(), ex);
}
}
public override void ProcessCommand()
{
logger.Info("Processing CommandRequest_SyncMyList");
try
{
// we will always assume that an anime was downloaded via http first
ScheduledUpdateRepository repSched = new ScheduledUpdateRepository();
AniDB_FileRepository repAniFile = new AniDB_FileRepository();
VideoLocalRepository repVidLocals = new VideoLocalRepository();
ScheduledUpdate sched = repSched.GetByUpdateType((int)ScheduledUpdateType.AniDBMyListSync);
if (sched == null)
{
sched = new ScheduledUpdate();
sched.UpdateType = (int)ScheduledUpdateType.AniDBMyListSync;
sched.UpdateDetails = "";
}
else
{
int freqHours = Utils.GetScheduledHours(ServerSettings.AniDB_MyList_UpdateFrequency);
// if we have run this in the last 24 hours and are not forcing it, then exit
TimeSpan tsLastRun = DateTime.Now - sched.LastUpdate;
if (tsLastRun.TotalHours < freqHours)
{
if (!ForceRefresh) return;
}
}
AniDBHTTPCommand_GetMyList cmd = new AniDBHTTPCommand_GetMyList();
cmd.Init(ServerSettings.AniDB_Username, ServerSettings.AniDB_Password);
enHelperActivityType ev = cmd.Process();
if (ev == enHelperActivityType.GotMyListHTTP && cmd.MyListItems.Count > 1)
{
int totalItems = 0;
int watchedItems = 0;
int modifiedItems = 0;
double pct = 0;
// 2. find files locally for the user, which are not recorded on anidb
// and then add them to anidb
Dictionary<int, Raw_AniDB_MyListFile> onlineFiles = new Dictionary<int, Raw_AniDB_MyListFile>();
foreach (Raw_AniDB_MyListFile myitem in cmd.MyListItems)
onlineFiles[myitem.FileID] = myitem;
Dictionary<string, AniDB_File> dictAniFiles = new Dictionary<string, AniDB_File>();
List<AniDB_File> allAniFiles = repAniFile.GetAll();
foreach (AniDB_File anifile in allAniFiles)
dictAniFiles[anifile.Hash] = anifile;
int missingFiles = 0;
foreach (VideoLocal vid in repVidLocals.GetAll())
{
if (!dictAniFiles.ContainsKey(vid.Hash)) continue;
int fileID = dictAniFiles[vid.Hash].FileID;
if (!onlineFiles.ContainsKey(fileID))
{
// means we have found a file in our local collection, which is not recorded online
CommandRequest_AddFileToMyList cmdAddFile = new CommandRequest_AddFileToMyList(vid.Hash);
cmdAddFile.Save();
missingFiles++;
}
}
logger.Info(string.Format("MYLIST Missing Files: {0} Added to queue for inclusion", missingFiles));
JMMUserRepository repUsers = new JMMUserRepository();
List<JMMUser> aniDBUsers = repUsers.GetAniDBUsers();
VideoLocal_UserRepository repVidUsers = new VideoLocal_UserRepository();
CrossRef_File_EpisodeRepository repFileEp = new CrossRef_File_EpisodeRepository();
// 1 . sync mylist items
foreach (Raw_AniDB_MyListFile myitem in cmd.MyListItems)
{
// ignore files mark as deleted by the user
if (myitem.State == (int)AniDBFileStatus.Deleted) continue;
totalItems++;
if (myitem.IsWatched) watchedItems++;
//calculate percentage
pct = (double)totalItems / (double)cmd.MyListItems.Count * (double)100;
string spct = pct.ToString("#0.0");
string hash = string.Empty;
AniDB_File anifile = repAniFile.GetByFileID(myitem.FileID);
if (anifile != null)
hash = anifile.Hash;
else
{
// look for manually linked files
List<CrossRef_File_Episode> xrefs = repFileEp.GetByEpisodeID(myitem.EpisodeID);
foreach (CrossRef_File_Episode xref in xrefs)
{
if (xref.CrossRefSource != (int)CrossRefSource.AniDB)
{
hash = xref.Hash;
break;
}
}
}
if (!string.IsNullOrEmpty(hash))
{
// find the video associated with this record
VideoLocal vl = repVidLocals.GetByHash(hash);
if (vl == null) continue;
foreach (JMMUser juser in aniDBUsers)
{
bool localStatus = false;
int? jmmUserID = null;
// doesn't matter which anidb user we use
jmmUserID = juser.JMMUserID;
VideoLocal_User userRecord = vl.GetUserRecord(juser.JMMUserID);
if (userRecord != null) localStatus = true;
string action = "";
if (localStatus != myitem.IsWatched)
{
if (localStatus == true)
{
// local = watched, anidb = unwatched
if (ServerSettings.AniDB_MyList_ReadUnwatched)
{
modifiedItems++;
if (jmmUserID.HasValue)
vl.ToggleWatchedStatus(myitem.IsWatched, false, myitem.WatchedDate, false, false, jmmUserID.Value, false, true);
action = "Used AniDB Status";
}
}
else
{
// means local is un-watched, and anidb is watched
if (ServerSettings.AniDB_MyList_ReadWatched)
{
modifiedItems++;
if (jmmUserID.HasValue)
vl.ToggleWatchedStatus(true, false, myitem.WatchedDate, false, false, jmmUserID.Value, false, true);
action = "Updated Local record to Watched";
}
}
string msg = string.Format("MYLISTDIFF:: File {0} - Local Status = {1}, AniDB Status = {2} --- {3}",
vl.FullServerPath, localStatus, myitem.IsWatched, action);
logger.Info(msg);
}
}
//string msg = string.Format("MYLIST:: File {0} - Local Status = {1}, AniDB Status = {2} --- {3}",
// vl.FullServerPath, localStatus, myitem.IsWatched, action);
//logger.Info(msg);
}
}
// now update all stats
Importer.UpdateAllStats();
logger.Info("Process MyList: {0} Items, {1} Watched, {2} Modified", totalItems, watchedItems, modifiedItems);
sched.LastUpdate = DateTime.Now;
repSched.Save(sched);
}
}
catch (Exception ex)
{
logger.Error("Error processing CommandRequest_SyncMyList: {0} ", ex.ToString());
return;
}
}