private void FillMissingHashes(SVR_VideoLocal vlocal)
{
bool needcrc32 = string.IsNullOrEmpty(vlocal.CRC32);
bool needmd5 = string.IsNullOrEmpty(vlocal.MD5);
bool needsha1 = string.IsNullOrEmpty(vlocal.SHA1);
if (needcrc32 || needmd5 || needsha1)
{
FillVideoHashes(vlocal);
}
needcrc32 = string.IsNullOrEmpty(vlocal.CRC32);
needmd5 = string.IsNullOrEmpty(vlocal.MD5);
needsha1 = string.IsNullOrEmpty(vlocal.SHA1);
if (needcrc32 || needmd5 || needsha1)
{
ShokoService.CmdProcessorHasher.QueueState = PrettyDescriptionHashing;
DateTime start = DateTime.Now;
List <string> tp = new List <string>();
if (needsha1)
{
tp.Add("SHA1");
}
if (needmd5)
{
tp.Add("MD5");
}
if (needcrc32)
{
tp.Add("CRC32");
}
logger.Trace("Calculating missing {1} hashes for: {0}", FileName, string.Join(",", tp));
// update the VideoLocal record with the Hash, since cloud support we calculate everything
Hashes hashes = FileHashHelper.GetHashInfo(FileName.Replace("/", $"{System.IO.Path.DirectorySeparatorChar}"), true, ShokoServer.OnHashProgress,
needcrc32, needmd5, needsha1);
TimeSpan ts = DateTime.Now - start;
logger.Trace("Hashed file in {0:#0.0} seconds --- {1} ({2})", ts.TotalSeconds,
FileName, Utils.FormatByteSize(vlocal.FileSize));
if (String.IsNullOrEmpty(vlocal.Hash))
{
vlocal.Hash = hashes.ED2K?.ToUpperInvariant();
}
if (needsha1)
{
vlocal.SHA1 = hashes.SHA1?.ToUpperInvariant();
}
if (needmd5)
{
vlocal.MD5 = hashes.MD5?.ToUpperInvariant();
}
if (needcrc32)
{
vlocal.CRC32 = hashes.CRC32?.ToUpperInvariant();
}
AzureWebAPI.Send_FileHash(new List <SVR_VideoLocal> {
vlocal
});
}
}
private VideoLocal_Place ProcessFile_LocalInfo()
{
// hash and read media info for file
int nshareID = -1;
string filePath = "";
Tuple <ImportFolder, string> tup = VideoLocal_PlaceRepository.GetFromFullPath(FileName);
if (tup == null)
{
logger.Error($"Unable to locate file {FileName} inside the import folders");
return(null);
}
ImportFolder folder = tup.Item1;
filePath = tup.Item2;
IFileSystem f = tup.Item1.FileSystem;
if (f == null)
{
logger.Error("Unable to open filesystem for : {0}", FileName);
return(null);
}
long filesize = 0;
if (folder.CloudID == null) // Local Access
{
if (!File.Exists(FileName))
{
logger.Error("File does not exist: {0}", FileName);
return(null);
}
int numAttempts = 0;
// Wait 3 minutes seconds before giving up on trying to access the file
while ((filesize = CanAccessFile(FileName)) == 0 && (numAttempts < 180))
{
numAttempts++;
Thread.Sleep(1000);
Console.WriteLine("Attempt # " + numAttempts.ToString());
}
// if we failed to access the file, get ouuta here
if (numAttempts == 180)
{
logger.Error("Could not access file: " + FileName);
return(null);
}
}
FileSystemResult <IObject> source = f.Resolve(FileName);
if (source == null || !source.IsOk || (!(source.Result is IFile)))
{
logger.Error("Could not access file: " + FileName);
return(null);
}
IFile source_file = (IFile)source.Result;
if (folder.CloudID.HasValue)
{
filesize = source_file.Size;
}
nshareID = folder.ImportFolderID;
// check if we have already processed this file
VideoLocal_Place vlocalplace = RepoFactory.VideoLocalPlace.GetByFilePathAndShareID(filePath, nshareID);
VideoLocal vlocal;
if (vlocalplace != null)
{
vlocal = vlocalplace.VideoLocal;
logger.Trace("VideoLocal record found in database: {0}", vlocal.VideoLocalID);
if (ForceHash)
{
vlocal.FileSize = filesize;
vlocal.DateTimeUpdated = DateTime.Now;
}
}
else
{
logger.Trace("VideoLocal, creating new record");
vlocal = new VideoLocal();
vlocal.DateTimeUpdated = DateTime.Now;
vlocal.DateTimeCreated = vlocal.DateTimeUpdated;
vlocal.FileName = Path.GetFileName(filePath);
vlocal.FileSize = filesize;
vlocal.Hash = string.Empty;
vlocal.CRC32 = string.Empty;
vlocal.MD5 = source_file.MD5.ToUpperInvariant() ?? string.Empty;
vlocal.SHA1 = source_file.SHA1.ToUpperInvariant() ?? string.Empty;
vlocal.IsIgnored = 0;
vlocal.IsVariation = 0;
vlocalplace = new VideoLocal_Place();
vlocalplace.FilePath = filePath;
vlocalplace.ImportFolderID = nshareID;
vlocalplace.ImportFolderType = folder.ImportFolderType;
}
// check if we need to get a hash this file
Hashes hashes = null;
if (string.IsNullOrEmpty(vlocal.Hash) || ForceHash)
{
// try getting the hash from the CrossRef
if (!ForceHash)
{
List <CrossRef_File_Episode> crossRefs = RepoFactory.CrossRef_File_Episode.GetByFileNameAndSize(vlocal.FileName, vlocal.FileSize);
if (crossRefs.Count == 1)
{
vlocal.Hash = crossRefs[0].Hash;
vlocal.HashSource = (int)HashSource.DirectHash;
}
}
// try getting the hash from the LOCAL cache
if (!ForceHash && string.IsNullOrEmpty(vlocal.Hash))
{
List <FileNameHash> fnhashes = RepoFactory.FileNameHash.GetByFileNameAndSize(vlocal.FileName, vlocal.FileSize);
if (fnhashes != null && fnhashes.Count > 1)
{
// if we have more than one record it probably means there is some sort of corruption
// lets delete the local records
foreach (FileNameHash fnh in fnhashes)
{
RepoFactory.FileNameHash.Delete(fnh.FileNameHashID);
}
}
if (fnhashes != null && fnhashes.Count == 1)
{
logger.Trace("Got hash from LOCAL cache: {0} ({1})", FileName, fnhashes[0].Hash);
vlocal.Hash = fnhashes[0].Hash;
vlocal.HashSource = (int)HashSource.WebCacheFileName;
}
}
if (string.IsNullOrEmpty(vlocal.Hash))
{
FillVideoHashes(vlocal);
}
if (string.IsNullOrEmpty(vlocal.Hash) && folder.CloudID.HasValue)
{
//Cloud and no hash, Nothing to do, except maybe Get the mediainfo....
RepoFactory.VideoLocal.Save(vlocal, false);
vlocalplace.VideoLocalID = vlocal.VideoLocalID;
RepoFactory.VideoLocalPlace.Save(vlocalplace);
if (vlocalplace.RefreshMediaInfo())
{
RepoFactory.VideoLocal.Save(vlocalplace.VideoLocal, true);
}
return(vlocalplace);
}
// hash the file
if (string.IsNullOrEmpty(vlocal.Hash) || ForceHash)
{
DateTime start = DateTime.Now;
logger.Trace("Calculating ED2K hashes for: {0}", FileName);
// update the VideoLocal record with the Hash, since cloud support we calculate everything
hashes = FileHashHelper.GetHashInfo(FileName.Replace("/", "\\"), true, MainWindow.OnHashProgress, false, false, false);
TimeSpan ts = DateTime.Now - start;
logger.Trace("Hashed file in {0} seconds --- {1} ({2})", ts.TotalSeconds.ToString("#0.0"), FileName, Utils.FormatByteSize(vlocal.FileSize));
vlocal.Hash = hashes.ed2k?.ToUpperInvariant();
vlocal.CRC32 = hashes.crc32?.ToUpperInvariant();
vlocal.MD5 = hashes.md5?.ToUpperInvariant();
vlocal.SHA1 = hashes.sha1?.ToUpperInvariant();
vlocal.HashSource = (int)HashSource.DirectHash;
FillVideoHashes(vlocal);
bool needcrc32 = string.IsNullOrEmpty(vlocal.CRC32);
bool needmd5 = string.IsNullOrEmpty(vlocal.MD5);
bool needsha1 = string.IsNullOrEmpty(vlocal.SHA1);
if (needcrc32 || needmd5 || needsha1)
{
List <string> tp = new List <string>();
if (needsha1)
{
tp.Add("SHA1");
}
if (needmd5)
{
tp.Add("MD5");
}
if (needcrc32)
{
tp.Add("CRC32");
}
logger.Trace("Calculating missing {1} hashes for: {0}", FileName, string.Join(",", tp));
// update the VideoLocal record with the Hash, since cloud support we calculate everything
hashes = FileHashHelper.GetHashInfo(FileName.Replace("/", "\\"), true, MainWindow.OnHashProgress,
needcrc32, needmd5, needsha1);
ts = DateTime.Now - start;
logger.Trace("Hashed file in {0} seconds --- {1} ({2})", ts.TotalSeconds.ToString("#0.0"),
FileName, Utils.FormatByteSize(vlocal.FileSize));
if (needsha1)
{
vlocal.SHA1 = hashes.sha1?.ToUpperInvariant();
}
if (needmd5)
{
vlocal.MD5 = hashes.md5?.ToUpperInvariant();
}
if (needcrc32)
{
vlocal.CRC32 = hashes.crc32?.ToUpperInvariant();
}
AzureWebAPI.Send_FileHash(new List <VideoLocal> {
vlocal
});
}
}
// We should have a hash by now
// before we save it, lets make sure there is not any other record with this hash (possible duplicate file)
VideoLocal tlocal = RepoFactory.VideoLocal.GetByHash(vlocal.Hash);
bool intercloudfolder = false;
VideoLocal_Place prep = tlocal?.Places.FirstOrDefault(a => a.ImportFolder.CloudID == folder.CloudID && vlocalplace.VideoLocal_Place_ID != a.VideoLocal_Place_ID);
if (prep != null)
{
// delete the VideoLocal record
logger.Warn("Deleting duplicate video file record");
logger.Warn("---------------------------------------------");
logger.Warn($"Keeping record for: {vlocalplace.FullServerPath}");
logger.Warn($"Deleting record for: {prep.FullServerPath}");
logger.Warn("---------------------------------------------");
// check if we have a record of this in the database, if not create one
List <DuplicateFile> dupFiles = RepoFactory.DuplicateFile.GetByFilePathsAndImportFolder(vlocalplace.FilePath,
prep.FilePath,
vlocalplace.ImportFolderID, prep.ImportFolderID);
if (dupFiles.Count == 0)
{
dupFiles = RepoFactory.DuplicateFile.GetByFilePathsAndImportFolder(prep.FilePath, vlocalplace.FilePath, prep.ImportFolderID, vlocalplace.ImportFolderID);
}
if (dupFiles.Count == 0)
{
DuplicateFile dup = new DuplicateFile();
dup.DateTimeUpdated = DateTime.Now;
dup.FilePathFile1 = vlocalplace.FilePath;
dup.FilePathFile2 = prep.FilePath;
dup.ImportFolderIDFile1 = vlocalplace.ImportFolderID;
dup.ImportFolderIDFile2 = prep.ImportFolderID;
dup.Hash = vlocal.Hash;
RepoFactory.DuplicateFile.Save(dup);
}
//Notify duplicate, don't delete
}
else if (tlocal != null)
{
vlocal = tlocal;
intercloudfolder = true;
}
if (!intercloudfolder)
{
RepoFactory.VideoLocal.Save(vlocal, true);
}
vlocalplace.VideoLocalID = vlocal.VideoLocalID;
RepoFactory.VideoLocalPlace.Save(vlocalplace);
if (intercloudfolder)
{
return(vlocalplace);
}
// also save the filename to hash record
// replace the existing records just in case it was corrupt
FileNameHash fnhash = null;
List <FileNameHash> fnhashes2 = RepoFactory.FileNameHash.GetByFileNameAndSize(vlocal.FileName, vlocal.FileSize);
if (fnhashes2 != null && fnhashes2.Count > 1)
{
// if we have more than one record it probably means there is some sort of corruption
// lets delete the local records
foreach (FileNameHash fnh in fnhashes2)
{
RepoFactory.FileNameHash.Delete(fnh.FileNameHashID);
}
}
if (fnhashes2 != null && fnhashes2.Count == 1)
{
fnhash = fnhashes2[0];
}
else
{
fnhash = new FileNameHash();
}
fnhash.FileName = vlocal.FileName;
fnhash.FileSize = vlocal.FileSize;
fnhash.Hash = vlocal.Hash;
fnhash.DateTimeUpdated = DateTime.Now;
RepoFactory.FileNameHash.Save(fnhash);
}
if ((vlocal.Media == null) || vlocal.MediaVersion < VideoLocal.MEDIA_VERSION || vlocal.Duration == 0)
{
if (vlocalplace.RefreshMediaInfo())
{
RepoFactory.VideoLocal.Save(vlocalplace.VideoLocal, true);
}
}
// now add a command to process the file
CommandRequest_ProcessFile cr_procfile = new CommandRequest_ProcessFile(vlocal.VideoLocalID, false);
cr_procfile.Save();
return(vlocalplace);
}