public bool AddHash(WebCache_FileHash hash)
{
return(WrapAuthentication((token) =>
{
cclient.AddHash(token, hash);
}));
}
public static void FillWith(this WebCache_FileHash prov, WebCache_FileHash_Collision origin)
{
prov.CRC32 = origin.CRC32;
prov.ED2K = origin.ED2K;
prov.FileSize = origin.FileSize;
prov.MD5 = origin.MD5;
prov.SHA1 = origin.SHA1;
}
public static WebCache_FileHash ToHashRequest(this SVR_VideoLocal vl)
{
WebCache_FileHash r = new WebCache_FileHash
{
ED2K = vl.Hash,
CRC32 = vl.CRC32,
MD5 = vl.MD5,
SHA1 = vl.SHA1,
FileSize = vl.FileSize
};
return(r);
}
/*
* public static WebCache_CrossRef_AniDB_Other_Request ToRequest(this CrossRef_AniDB_Other c)
* {
* return new WebCache_CrossRef_AniDB_Other_Request
* {
* CrossRef_AniDB_OtherID = c.CrossRef_AniDB_OtherID,
* AnimeID = c.AnimeID,
* CrossRefID = c.CrossRefID,
* CrossRefSource = c.CrossRefSource,
* CrossRefType = c.CrossRefType,
* };
* }
*/
public static WebCache_FileHash ToHashRequest(this AniDB_File anifile)
{
WebCache_FileHash r = new WebCache_FileHash
{
ED2K = anifile.Hash,
CRC32 = anifile.CRC,
MD5 = anifile.MD5,
SHA1 = anifile.SHA1,
FileSize = anifile.FileSize
};
return(r);
}
public async Task <IActionResult> GetHash(string token, int type, string hash, long?size)
{
try
{
SessionInfoWithError s = await VerifyTokenAsync(token);
if (s.Error != null)
{
return(s.Error);
}
hash = hash.ToUpperInvariant();
WebCache_FileHash h = null;
switch ((WebCache_HashType)type)
{
case WebCache_HashType.ED2K:
h = await _db.WebCache_FileHashes.FirstOrDefaultAsync(a => a.ED2K == hash);
break;
case WebCache_HashType.CRC:
if (size == null)
{
return(StatusCode(400, "You must include size when asking for CRC"));
}
h = await _db.WebCache_FileHashes.FirstOrDefaultAsync(a => a.CRC32 == hash && a.FileSize == size.Value);
break;
case WebCache_HashType.MD5:
h = await _db.WebCache_FileHashes.FirstOrDefaultAsync(a => a.MD5 == hash);
break;
case WebCache_HashType.SHA1:
h = await _db.WebCache_FileHashes.FirstOrDefaultAsync(a => a.SHA1 == hash);
break;
}
if (h == null)
{
return(StatusCode(404, "Hash not found"));
}
return(new JsonResult(h));
}
catch (Exception e)
{
_logger.LogError(e, $"GETHASH with Token={token} Hash={hash} Type={(WebCache_HashType) type} Size={size ?? 0}");
return(StatusCode(500));
}
}
public async Task <IActionResult> AddHash(string token, [FromBody] WebCache_FileHash hash)
{
try
{
SessionInfoWithError s = await VerifyTokenAsync(token);
if (s.Error != null)
{
return(s.Error);
}
if (await InternalAddHash(s, hash))
{
await _db.SaveChangesAsync();
}
return(Ok());
}
catch (Exception e)
{
_logger.LogError(e, $"ADDHASH with Token={token} ED2K={hash.ED2K}");
return(StatusCode(500));
}
}
private void FillHashesAgainstWebCache(SVR_VideoLocal v)
{
if (!string.IsNullOrEmpty(v.ED2KHash))
{
WebCache_FileHash ls = WebCacheAPI.Instance.GetHash(WebCache_HashType.ED2K, v.ED2KHash);
if (ls != null)
{
v.SHA1 = ls.SHA1.ToUpperInvariant();
v.CRC32 = ls.CRC32.ToUpperInvariant();
v.MD5 = ls.MD5.ToUpperInvariant();
return;
}
}
if (!string.IsNullOrEmpty(v.SHA1))
{
WebCache_FileHash ls = WebCacheAPI.Instance.GetHash(WebCache_HashType.SHA1, v.SHA1);
if (ls != null)
{
v.ED2KHash = ls.ED2K.ToUpperInvariant();
v.CRC32 = ls.CRC32.ToUpperInvariant();
v.MD5 = ls.MD5.ToUpperInvariant();
return;
}
}
if (!string.IsNullOrEmpty(v.MD5))
{
WebCache_FileHash ls = WebCacheAPI.Instance.GetHash(WebCache_HashType.MD5, v.MD5);
if (ls != null)
{
v.ED2KHash = ls.ED2K.ToUpperInvariant();
v.CRC32 = ls.CRC32.ToUpperInvariant();
v.SHA1 = ls.SHA1.ToUpperInvariant();
}
}
}
private async Task <bool> InternalAddHash(SessionInfoWithError s, WebCache_FileHash hash)
{
bool update = false;
if (string.IsNullOrEmpty(hash.ED2K) || string.IsNullOrEmpty(hash.CRC32) || string.IsNullOrEmpty(hash.MD5) || string.IsNullOrEmpty(hash.SHA1) || hash.FileSize == 0)
{
return(false);
}
hash.ED2K = hash.ED2K.ToUpperInvariant();
hash.CRC32 = hash.CRC32.ToUpperInvariant();
hash.MD5 = hash.MD5.ToUpperInvariant();
hash.SHA1 = hash.SHA1.ToUpperInvariant();
WebCache_FileHash_Info ed2k = await _db.WebCache_FileHashes.FirstOrDefaultAsync(a => a.ED2K == hash.ED2K);
WebCache_FileHash_Info md5 = await _db.WebCache_FileHashes.FirstOrDefaultAsync(a => a.MD5 == hash.MD5);
WebCache_FileHash_Info sha1 = await _db.WebCache_FileHashes.FirstOrDefaultAsync(a => a.SHA1 == hash.SHA1);
WebCache_FileHash_Info orig = new WebCache_FileHash_Info();
orig.FillWith(hash);
orig.AniDBUserId = s.AniDBUserId;
orig.CreationDate = DateTime.UtcNow;
if (ed2k == null && md5 == null && sha1 == null) //Not CRC, we may get a normal collision in CRC
{
_db.Add(orig);
update = true;
}
else
{
List <WebCache_FileHash_Info> collisions = new List <WebCache_FileHash_Info>();
// ReSharper disable PossibleNullReferenceException
if (ed2k.CRC32 != hash.CRC32 || ed2k.FileSize != hash.FileSize || ed2k.SHA1 != hash.SHA1 || ed2k.MD5 != hash.MD5)
{
collisions.Add(ed2k);
}
if (md5.CRC32 != hash.CRC32 || md5.FileSize != hash.FileSize || md5.SHA1 != hash.SHA1 || md5.ED2K != hash.ED2K)
{
if (!collisions.Contains(md5))
{
collisions.Add(md5);
}
}
if (sha1.CRC32 != hash.CRC32 || sha1.FileSize != hash.FileSize || sha1.MD5 != hash.MD5 || sha1.ED2K != hash.ED2K)
{
if (!collisions.Contains(sha1))
{
collisions.Add(sha1);
}
}
// ReSharper restore PossibleNullReferenceException
if (collisions.Count > 0)
{
if (collisions.Any(b => b.CollisionApproved))
{
return(false); //We already have the approved one, so this new one is wrong
}
collisions.Add(orig);
string unique = Guid.NewGuid().ToString().Replace("-", string.Empty);
_db.AddRange(collisions.Select(a => a.ToCollision(unique)));
update = true;
}
}
return(update);
}
public override void Run(IProgress <ICommand> progress = null)
{
try
{
ReportInit(progress);
List <SVR_VideoLocal> allfiles = Repo.Instance.VideoLocal.GetAll().ToList();
List <SVR_VideoLocal> missfiles = allfiles.Where(
a =>
string.IsNullOrEmpty(a.CRC32) || string.IsNullOrEmpty(a.SHA1) ||
string.IsNullOrEmpty(a.MD5) || a.SHA1 == "0000000000000000000000000000000000000000" ||
a.MD5 == "00000000000000000000000000000000")
.ToList();
ReportUpdate(progress, 10);
List <SVR_VideoLocal> withfiles = allfiles.Except(missfiles).ToList();
Dictionary <int, (string ed2k, string crc32, string md5, string sha1)> updates = new Dictionary <int, (string ed2k, string crc32, string md5, string sha1)>();
//Check if we can populate md5,sha and crc from AniDB_Files
List <SVR_VideoLocal> vm = missfiles.ToList();
for (int x = 0; x < vm.Count; x++)
{
double prog = (x + 1) * 60 / (double)missfiles.Count;
SVR_VideoLocal v = vm[x];
PrettyDescription.QueueState = QueueStateEnum.CheckingFile;
PrettyDescription.ExtraParams = new [] { v.Info };
SVR_AniDB_File file = Repo.Instance.AniDB_File.GetByHash(v.ED2KHash);
if (file != null)
{
if (!string.IsNullOrEmpty(file.CRC) && !string.IsNullOrEmpty(file.SHA1) &&
!string.IsNullOrEmpty(file.MD5))
{
updates[v.VideoLocalID] = (file.Hash, file.CRC, file.MD5, file.SHA1);
missfiles.Remove(v);
withfiles.Add(v);
ReportUpdate(progress, 10 + prog);
continue;
}
}
WebCache_FileHash ls = WebCacheAPI.Instance.GetHash(WebCache_HashType.ED2K, v.ED2KHash);
if (ls != null)
{
updates[v.VideoLocalID] = (ls.ED2K.ToUpperInvariant(), ls.CRC32.ToUpperInvariant(), ls.MD5.ToUpperInvariant(), ls.SHA1.ToUpperInvariant());
missfiles.Remove(v);
}
ReportUpdate(progress, 10 + prog);
}
//We need to recalculate the sha1, md5 and crc32 of the missing ones.
List <ICommand> tohash = new List <ICommand>();
foreach (SVR_VideoLocal v in missfiles)
{
try
{
SVR_VideoLocal_Place p = v.GetBestVideoLocalPlace();
if (p != null && p.ImportFolder.CloudID == 0)
{
tohash.Add(new CmdHashFile(p.FullServerPath, true));
}
}
catch
{
//Ignored
}
}
Queue.Instance.AddRange(tohash);
ReportUpdate(progress, 80);
if (updates.Count > 0)
{
using (var upd = Repo.Instance.VideoLocal.BeginBatchUpdate(() => Repo.Instance.VideoLocal.GetMany(updates.Keys)))
{
foreach (SVR_VideoLocal v in upd)
{
(string ed2k, string crc32, string md5, string sha1)t = updates[v.VideoLocalID];
v.Hash = t.ed2k;
v.CRC32 = t.crc32;
v.MD5 = t.md5;
v.SHA1 = t.sha1;
upd.Update(v);
}
upd.Commit();
}
}
ReportUpdate(progress, 90);
//Send the hashes
WebCacheAPI.Instance.AddHashes(withfiles.Select(a => a.ToHashRequest()));
logger.Info("Sync Hashes Complete");
ReportFinish(progress);
}
catch (Exception ex)
{
ReportError(progress, $"Error processing ServerSyncHashes: {ex}", ex);
}
}
