public void RefreshAdminMessages()
{
AdminMessages.Clear();
try
{
AdminMessagesAvailable = false;
if (!ServerSettings.Instance.WebCache.Enabled)
{
return;
}
List <Azure_AdminMessage> msgs = AzureWebAPI.Get_AdminMessages();
if (msgs == null || msgs.Count == 0)
{
AdminMessagesAvailable = false;
return;
}
foreach (Azure_AdminMessage msg in msgs)
{
AdminMessages.Add(msg);
}
AdminMessagesAvailable = true;
}
catch (Exception ex)
{
Utils.ShowErrorMessage(ex);
}
}
public static Azure_UserInfo GetUserInfoData(string vidPlayer = "")
{
try
{
if (string.IsNullOrEmpty(ServerSettings.Instance.AniDb.Username))
{
return(null);
}
Azure_UserInfo uinfo = new Azure_UserInfo
{
DateTimeUpdated = DateTime.Now,
DateTimeUpdatedUTC = 0,
// Optional JMM Desktop data
DashboardType = null,
VideoPlayer = vidPlayer
};
Assembly a = Assembly.GetEntryAssembly();
try
{
if (a != null)
{
uinfo.JMMServerVersion = Utils.GetApplicationVersion(a);
}
}
catch
{
// ignored
}
uinfo.UsernameHash = Utils.GetMd5Hash(ServerSettings.Instance.AniDb.Username);
uinfo.DatabaseType = ServerSettings.Instance.Database.Type;
uinfo.WindowsVersion = Utils.GetOSInfo();
uinfo.TraktEnabled = ServerSettings.Instance.TraktTv.Enabled ? 1 : 0;
uinfo.CountryLocation = string.Empty;
// this field is not actually used
uinfo.LastEpisodeWatchedAsDate = DateTime.Now.AddDays(-5);
uinfo.LocalUserCount = (int)RepoFactory.JMMUser.GetTotalRecordCount();
uinfo.FileCount = RepoFactory.VideoLocal.GetTotalRecordCount();
SVR_AnimeEpisode_User rec = RepoFactory.AnimeEpisode_User.GetLastWatchedEpisode();
uinfo.LastEpisodeWatched = 0;
if (rec != null)
{
uinfo.LastEpisodeWatched = Commons.Utils.AniDB.GetAniDBDateAsSeconds(rec.WatchedDate);
}
return(uinfo);
}
catch (Exception ex)
{
logger.Error(ex, ex.ToString());
return(null);
}
}
private static bool PostData(string type, IDictionary <string, string> extraData)
{
#if DEBUG
return(false);
#endif
#pragma warning disable CS0162 // Unreachable code detected
if (ServerSettings.Instance.GA_OptOutPlzDont)
{
return(false);
}
try
{
using (var client = new HttpClient())
{
var data = new Dictionary <string, string>
{
{ "t", type },
{ "an", "Shoko Server" },
{ "tid", AnalyticsId },
{ "cid", ServerSettings.Instance.GA_ClientId },
{ "v", "1" },
{ "av", Utils.GetApplicationVersion() },
{ "ul", CultureInfo.GetCultureInfo(ServerSettings.Instance.Culture).DisplayName },
{ "aip", "1" }
};
data.AddRange(extraData);
var resp = client.PostAsync($"{Endpoint}/collect", new FormUrlEncodedContent(data))
.ConfigureAwait(false).GetAwaiter().GetResult();
return(true);
}
}
catch (HttpRequestException ex)
{
Logger.Error("There was an error posting to Google Analytics", ex);
return(false);
}
#pragma warning restore CS0162 // Unreachable code detected
}
private async Task FillMissingHashes(SVR_VideoLocal vlocal, CancellationToken token, IProgress <ICommand> progress = null)
{
HashTypes types = 0;
if (string.IsNullOrEmpty(vlocal.CRC32))
{
types |= HashTypes.CRC;
}
if (string.IsNullOrEmpty(vlocal.MD5))
{
types |= HashTypes.MD5;
}
if (string.IsNullOrEmpty(vlocal.SHA1))
{
types |= HashTypes.SHA1;
}
if (types > 0)
{
FillVideoHashes(vlocal);
}
types = 0;
if (string.IsNullOrEmpty(vlocal.CRC32))
{
types |= HashTypes.CRC;
}
if (string.IsNullOrEmpty(vlocal.MD5))
{
types |= HashTypes.MD5;
}
if (string.IsNullOrEmpty(vlocal.SHA1))
{
types |= HashTypes.SHA1;
}
if (types > 0)
{
_hashingState = true;
DateTime start = DateTime.Now;
logger.Trace("Calculating missing {1} hashes for: {0}", File.FullName, types.ToString("F"));
// update the VideoLocal record with the Hash, since cloud support we calculate everything
Hasher h = new Hasher(File, HashAll);
string error = await h.RunAsync(new ChildProgress(20, 60, this, progress), token);
TimeSpan ts = DateTime.Now - start;
logger.Trace("Hashed file in {0:#0.0} seconds --- {1} ({2})", ts.TotalSeconds, File.FullName, Utils.FormatByteSize(vlocal.FileSize));
if (error != null)
{
logger.Error("Unable to add additional hashes missing {1} hashes for: {0} Error {2}", File.FullName, types.ToString("F"), error);
}
else
{
if ((types & HashTypes.CRC) > 0)
{
vlocal.CRC32 = h.Result.GetHash(HashTypes.CRC);
}
if ((types & HashTypes.MD5) > 0)
{
vlocal.MD5 = h.Result.GetHash(HashTypes.MD5);
}
if ((types & HashTypes.SHA1) > 0)
{
vlocal.SHA1 = h.Result.GetHash(HashTypes.SHA1);
}
WebCacheAPI.Instance.AddHash(new List <WebCache_FileHash> {
vlocal.ToHashRequest()
});
}
}
}
public override async Task RunAsync(IProgress <ICommand> progress = null, CancellationToken token = default(CancellationToken))
{
logger.Trace($"Checking File For Hashes: {File.FullName}");
try
{
ReportInit(progress);
// hash and read media info for file
int nshareID;
long filesize = 0;
if (_importFolder.CloudID == null) // Local Access
{
if (!System.IO.File.Exists(File.FullName))
{
ReportError(progress, $"File does not exist: {File.FullName}");
return;
}
int numAttempts = 0;
bool writeAccess = _importFolder.IsDropSource == 1;
// Wait 1 minute before giving up on trying to access the file
// first only do read to not get in something's way
while ((filesize = CanAccessFile(File.FullName, false)) == 0 && (numAttempts < 60))
{
numAttempts++;
Thread.Sleep(1000);
logger.Trace($@"Failed to access, (or filesize is 0) Attempt # {numAttempts}, {File.FullName}");
}
// if we failed to access the file, get ouuta here
if (numAttempts >= 60)
{
ReportError(progress, $"Could not access file: {File.FullName}");
return;
}
// At least 1s between to ensure that size has the chance to change
// TODO make this a setting to allow fine tuning on various configs
// TODO Make this able to be disabled. It adds 1.5s to hashing just waiting for the Linux/NAS use case
int seconds = 8;
int waitTime = seconds * 1000 / 2;
Thread.Sleep(waitTime);
numAttempts = 0;
//For systems with no locking
// TODO make this a setting as well
while (FileModified(File.FullName, seconds, ref filesize, writeAccess) && numAttempts < 60)
{
numAttempts++;
Thread.Sleep(waitTime);
// Only show if it's more than 'seconds' past
if (numAttempts != 0 && numAttempts * 2 % seconds == 0)
{
logger.Warn($@"The modified date is too soon. Waiting to ensure that no processes are writing to it. {numAttempts}/60 {File.FullName}");
}
}
// if we failed to access the file, get ouuta here
if (numAttempts >= 60)
{
ReportError(progress, $"Could not access file: {File.FullName}");
return;
}
}
ReportUpdate(progress, 10);
if (_importFolder.CloudID.HasValue)
{
filesize = File.Size;
}
nshareID = _importFolder.ImportFolderID;
// check if we have already processed this file
SVR_VideoLocal_Place vlocalplace = Repo.Instance.VideoLocal_Place.GetByFilePathAndImportFolderID(_filePath, nshareID);
SVR_VideoLocal vlocal = null;
var filename = Path.GetFileName(_filePath);
if (vlocalplace != null)
{
vlocal = vlocalplace.VideoLocal;
if (vlocal != null)
{
logger.Trace("VideoLocal record found in database: {0}", File.FullName);
// This will only happen with DB corruption, so just clean up the mess.
if (vlocalplace.FullServerPath == null)
{
if (vlocal.Places.Count == 1)
{
Repo.Instance.VideoLocal.Delete(vlocal);
vlocal = null;
}
Repo.Instance.VideoLocal_Place.Delete(vlocalplace);
vlocalplace = null;
}
if (vlocal != null && Force)
{
vlocal.FileSize = filesize;
vlocal.DateTimeUpdated = DateTime.Now;
}
}
}
bool duplicate = false;
SVR_VideoLocal vlocal1 = vlocal;
using (var txn = Repo.Instance.VideoLocal.BeginAddOrUpdate(vlocal1?.VideoLocalID ?? 0, () =>
{
logger.Trace("No existing VideoLocal, creating temporary record");
return(new SVR_VideoLocal
{
DateTimeUpdated = DateTime.Now,
DateTimeCreated = DateTime.Now,
FileSize = filesize,
Hash = string.Empty,
CRC32 = string.Empty,
MD5 = File?.MD5?.ToUpperInvariant() ?? string.Empty,
SHA1 = File?.SHA1?.ToUpperInvariant() ?? string.Empty,
IsIgnored = 0,
IsVariation = 0
});
}))
{
vlocal = txn.Entity;
if (vlocalplace == null)
{
logger.Trace("No existing VideoLocal_Place, creating a new record");
vlocalplace = new SVR_VideoLocal_Place {
FilePath = _filePath, ImportFolderID = nshareID, ImportFolderType = _importFolder.ImportFolderType
};
// Make sure we have an ID
vlocalplace = Repo.Instance.VideoLocal_Place.BeginAdd(vlocalplace).Commit();
}
// check if we need to get a hash this file
// IDEs might warn of possible null. It is set in the lambda above, so it shouldn't ever be null
if (string.IsNullOrEmpty(vlocal.Hash) || Force)
{
logger.Trace("No existing hash in VideoLocal, checking XRefs");
if (!Force)
{
// try getting the hash from the CrossRef
List <CrossRef_File_Episode> crossRefs = Repo.Instance.CrossRef_File_Episode.GetByFileNameAndSize(filename, vlocal.FileSize);
if (crossRefs.Any())
{
vlocal.Hash = crossRefs[0].Hash;
vlocal.HashSource = (int)HashSource.DirectHash;
}
}
// try getting the hash from the LOCAL cache
if (!Force && string.IsNullOrEmpty(vlocal.Hash))
{
Repo.Instance.FileNameHash.FindAndDelete(() =>
{
List <FileNameHash> fnhashes = Repo.Instance.FileNameHash.GetByFileNameAndSize(filename, vlocal.FileSize);
if (fnhashes != null && fnhashes.Count > 1)
{
// if we have more than one record it probably means there is some sort of corruption
// lets delete the local records
return(fnhashes);
}
return(new List <FileNameHash>());
});
// reinit this to check if we erased them
FileNameHash fnhash = Repo.Instance.FileNameHash.GetByFileNameAndSize(filename, vlocal.FileSize).FirstOrDefault();
logger.Trace("Got hash from LOCAL cache: {0} ({1})", File.FullName, fnhash.Hash);
vlocal.Hash = fnhash.Hash;
vlocal.HashSource = (int)HashSource.WebCacheFileName;
}
if (string.IsNullOrEmpty(vlocal.Hash))
{
FillVideoHashes(vlocal);
}
//Cloud and no hash, Nothing to do, except maybe Get the mediainfo....
if (string.IsNullOrEmpty(vlocal.Hash) && _importFolder.CloudID.HasValue)
{
logger.Trace("No Hash found for cloud " + filename + " putting in videolocal table with empty ED2K");
vlocal = txn.Commit(true);
int vlpid = vlocalplace.VideoLocalID;
using (var upd = Repo.Instance.VideoLocal_Place.BeginAddOrUpdate(vlpid))
{
upd.Entity.VideoLocalID = vlocal.VideoLocalID;
vlocalplace = upd.Commit();
}
if (vlocalplace.RefreshMediaInfo(vlocal))
{
txn.Commit(true);
}
ReportFinish(progress);
return;
}
// hash the file
if (string.IsNullOrEmpty(vlocal.Hash) || Force)
{
logger.Info("Hashing File: {0}", File.FullName);
_hashingState = true;
DateTime start = DateTime.Now;
// update the VideoLocal record with the Hash, since cloud support we calculate everything
Hasher h = new Hasher(File, HashAll);
string error = await h.RunAsync(new ChildProgress(20, 60, this, progress), token);
if (error != null)
{
ReportError(progress, error);
return;
}
TimeSpan ts = DateTime.Now - start;
logger.Trace("Hashed file in {0:#0.0} seconds --- {1} ({2})", ts.TotalSeconds, File.FullName, Utils.FormatByteSize(vlocal.FileSize));
vlocal.Hash = h.Result.GetHash(HashTypes.ED2K);
vlocal.CRC32 = h.Result.GetHash(HashTypes.CRC);
vlocal.MD5 = h.Result.GetHash(HashTypes.MD5);
vlocal.SHA1 = h.Result.GetHash(HashTypes.SHA1);
vlocal.HashSource = (int)HashSource.DirectHash;
}
_hashingState = false;
await FillMissingHashes(vlocal, token, progress);
// We should have a hash by now
// before we save it, lets make sure there is not any other record with this hash (possible duplicate file)
// TODO Check this case. I'm not sure how EF handles changing objects that we are working on
SVR_VideoLocal tlocal = Repo.Instance.VideoLocal.GetByHash(vlocal.Hash);
bool changed = false;
if (tlocal != null)
{
logger.Trace("Found existing VideoLocal with hash, merging info from it");
// Aid with hashing cloud. Merge hashes and save, regardless of duplicate file
changed = tlocal.MergeInfoFrom(vlocal);
vlocal = tlocal;
List <SVR_VideoLocal_Place> preps = vlocal.Places.Where(a => a.ImportFolder.CloudID == _importFolder.CloudID && !vlocalplace.FullServerPath.Equals(a.FullServerPath)).ToList();
foreach (var prep in preps)
{
if (prep == null)
{
continue;
}
// clean up, if there is a 'duplicate file' that is invalid, remove it.
if (prep.FullServerPath == null)
{
Repo.Instance.VideoLocal_Place.Delete(prep);
}
else
{
IResult dupFileSystemResult = prep.ImportFolder?.FileSystem?.Resolve(prep.FullServerPath);
if (dupFileSystemResult == null || dupFileSystemResult.Status != NutzCode.CloudFileSystem.Status.Ok)
{
Repo.Instance.VideoLocal_Place.Delete(prep);
}
}
}
var dupPlace = vlocal.Places.FirstOrDefault(a => a.ImportFolder.CloudID == _importFolder.CloudID && !vlocalplace.FullServerPath.Equals(a.FullServerPath));
ReportUpdate(progress, 85);
if (dupPlace != null)
{
logger.Warn("Found Duplicate File");
logger.Warn("---------------------------------------------");
logger.Warn($"New File: {vlocalplace.FullServerPath}");
logger.Warn($"Existing File: {dupPlace.FullServerPath}");
logger.Warn("---------------------------------------------");
// check if we have a record of this in the database, if not create one
List <DuplicateFile> dupFiles = Repo.Instance.DuplicateFile.GetByFilePathsAndImportFolder(vlocalplace.FilePath, dupPlace.FilePath, vlocalplace.ImportFolderID, dupPlace.ImportFolderID);
if (dupFiles.Count == 0)
{
dupFiles = Repo.Instance.DuplicateFile.GetByFilePathsAndImportFolder(dupPlace.FilePath, vlocalplace.FilePath, dupPlace.ImportFolderID, vlocalplace.ImportFolderID);
}
if (dupFiles.Count == 0)
{
DuplicateFile dup = new DuplicateFile
{
DateTimeUpdated = DateTime.Now,
FilePathFile1 = vlocalplace.FilePath,
FilePathFile2 = dupPlace.FilePath,
ImportFolderIDFile1 = vlocalplace.ImportFolderID,
ImportFolderIDFile2 = dupPlace.ImportFolderID,
Hash = vlocal.Hash
};
Repo.Instance.DuplicateFile.BeginAdd(dup).Commit();
}
//Notify duplicate, don't delete
duplicate = true;
}
}
if (!duplicate || changed)
{
vlocal = txn.Commit();
}
}
ReportUpdate(progress, 90);
int vlplid = vlocalplace.VideoLocalID;
using (var upd = Repo.Instance.VideoLocal_Place.BeginAddOrUpdate(vlplid))
{
upd.Entity.VideoLocalID = vlocal.VideoLocalID;
vlocalplace = upd.Commit();
}
}
if (duplicate)
{
Queue.Instance.Add(new CmdServerProcessFile(vlocal.VideoLocalID, false));
ReportFinish(progress);
return;
}
// also save the filename to hash record
// replace the existing records just in case it was corrupt
Repo.Instance.FileNameHash.FindAndDelete(() =>
{
List <FileNameHash> fnhashes = Repo.Instance.FileNameHash.GetByFileNameAndSize(filename, vlocal.FileSize);
if (fnhashes != null && fnhashes.Count > 1)
{
// if we have more than one record it probably means there is some sort of corruption
// lets delete the local records
return(fnhashes);
}
return(new List <FileNameHash>());
});
ReportUpdate(progress, 95);
using (var upd = Repo.Instance.FileNameHash.BeginAddOrUpdate(() => Repo.Instance.FileNameHash.GetByFileNameAndSize(filename, vlocal.FileSize).FirstOrDefault()))
{
upd.Entity.FileName = filename;
upd.Entity.FileSize = vlocal.FileSize;
upd.Entity.Hash = vlocal.Hash;
upd.Entity.DateTimeUpdated = DateTime.Now;
upd.Commit();
}
if (vlocal.Media == null || vlocal.MediaVersion < SVR_VideoLocal.MEDIA_VERSION || vlocal.Duration == 0)
{
int vid = vlocal.VideoLocalID;
using (var upd = Repo.Instance.VideoLocal.BeginAddOrUpdate(vid))
if (vlocalplace.RefreshMediaInfo(upd.Entity))
{
vlocal = upd.Commit(true);
}
}
// now add a command to process the file
Queue.Instance.Add(new CmdServerProcessFile(vlocal.VideoLocalID, false));
ReportFinish(progress);
}
catch (Exception ex)
{
ReportError(progress, $"Error processing ServerHashFile: {File.FullName}\n{ex}", ex);
}
}
