private static void ImportInternal(BackgroundWorker backgroundworker) { FileInfo fileinfo; lock (_rwlock) { if (_rwList.Count == 0) { //ComputeInternal(backgroundworker); return; } fileinfo = _rwList.ElementAt(0); _rwList.RemoveAt(0); } /* * _sb.Clear(); * _sb.Append($"a:{_added}/f:{_found}/b:{_bad}"); * backgroundworker.ReportProgress(0, _sb.ToString()); */ var orgfilename = fileinfo.FullName; if (!File.Exists(orgfilename)) { return; } var imagedata = File.ReadAllBytes(orgfilename); if (imagedata == null || imagedata.Length < 256) { FileHelper.MoveCorruptedFile(orgfilename); _bad++; return; } int year = DateTime.Now.Year; var orgextension = Path.GetExtension(orgfilename); if (orgextension.Equals(AppConsts.DatExtension, StringComparison.OrdinalIgnoreCase) || orgextension.Equals(AppConsts.MzxExtension, StringComparison.OrdinalIgnoreCase)) { year = 0; var password = Path.GetFileNameWithoutExtension(orgfilename); var decrypteddata = orgextension.Equals(AppConsts.DatExtension, StringComparison.OrdinalIgnoreCase) ? EncryptionHelper.DecryptDat(imagedata, password) : EncryptionHelper.Decrypt(imagedata, password); if (decrypteddata != null) { imagedata = decrypteddata; } } var hash = MD5HashHelper.Compute(imagedata); bool found; Img imgfound; lock (_imglock) { found = _hashList.TryGetValue(hash, out imgfound); } if (found) { // we found the same image in a database var filenamefound = FileHelper.NameToFileName(imgfound.Name); if (File.Exists(filenamefound)) { // no reason to add the same image from a heap; we have one FileHelper.DeleteToRecycleBin(orgfilename); if (imgfound.Year == 0 && year != 0) { imgfound.Year = year; } _found++; return; } // found image is gone; delete it Delete(imgfound.Id); } var bitmap = BitmapHelper.ImageDataToBitmap(imagedata); if (bitmap == null) { var badname = Path.GetFileName(orgfilename); var badfilename = $"{AppConsts.PathGb}\\{badname}{AppConsts.CorruptedExtension}"; if (File.Exists(badfilename)) { FileHelper.DeleteToRecycleBin(badfilename); } File.WriteAllBytes(badfilename, imagedata); FileHelper.DeleteToRecycleBin(orgfilename); _bad++; return; } bitmap.Dispose(); // we have to create unique name and a location in Hp folder string newname; string newfilename; var iteration = -1; do { iteration++; newname = FileHelper.HashToName(hash, iteration); newfilename = FileHelper.NameToFileName(newname); } while (File.Exists(newfilename)); //var lv = GetMinLastView(); var lc = GetMinLastCheck(); var id = AllocateId(); var nimg = new Img( id: id, name: newname, hash: hash, vector: Array.Empty <ushort>(), year: year, counter: 0, bestid: 0, sig: 0, bestvdistance: 100f, lastview: new DateTime(2020, 1, 1), lastcheck: lc); Add(nimg); var lastmodified = File.GetLastWriteTime(orgfilename); if (lastmodified > DateTime.Now) { lastmodified = DateTime.Now; } if (!orgfilename.Equals(newfilename, StringComparison.OrdinalIgnoreCase)) { FileHelper.WriteData(newfilename, imagedata); File.SetLastWriteTime(newfilename, lastmodified); FileHelper.DeleteToRecycleBin(orgfilename); } _added++; }
private static void ComputeInternal(BackgroundWorker backgroundworker) { Img img1 = null; Img[] shadowcopy = null; int activescope = 0; int sig = 0; int prevsig = 0; lock (_imglock) { if (_imgList.Count < 2) { backgroundworker.ReportProgress(0, "no images"); return; } shadowcopy = _imgList.Select(e => e.Value).OrderBy(e => e.LastCheck).ToArray(); } activescope = shadowcopy.Count(e => e.Vector.Length != 0); sig = _clusters.Rows; foreach (var img in shadowcopy) { if (img.Sig == 0 && img.BestId == 0 || img.BestId == img.Id || !_imgList.ContainsKey(img.BestId)) { img1 = img; break; } if (img1 == null || img.LastCheck < img1.LastCheck) { img1 = img; } } prevsig = img1.Sig; if (img1.Sig != sig || img1.Vector.Length == 0) { var filename = FileHelper.NameToFileName(img1.Name); var imagedata = FileHelper.ReadData(filename); if (imagedata == null) { Delete(img1.Id); return; } using (var bitmap = BitmapHelper.ImageDataToBitmap(imagedata)) { if (bitmap == null) { Delete(img1.Id); return; } var matrix = BitmapHelper.GetMatrix(imagedata); var descriptors = SiftHelper.GetDescriptors(matrix); var vector = ComputeVector(descriptors, backgroundworker); img1.Vector = vector; } img1.Sig = sig; } var candidates = shadowcopy.Where(e => e.Id != img1.Id && e.Vector.Length != 0).ToArray(); if (candidates.Length > 0) { var bestid = img1.Id; var bestvdistance = 100f; for (var i = 0; i < candidates.Length; i++) { var img2 = candidates[i]; var vdistance = GetDistance(img1.Vector, img2.Vector); if (vdistance < bestvdistance) { bestid = img2.Id; bestvdistance = vdistance; } } if (bestid != img1.BestId) { _sb.Clear(); _sb.Append($"(n:{candidates.Length}/c:{_clusters.Rows}) a:{_added}/f:{_found}/b:{_bad} [{img1.Id}:{prevsig}] {img1.BestVDistance:F2} \u2192 {bestvdistance:F2}"); backgroundworker.ReportProgress(0, _sb.ToString()); img1.BestId = bestid; img1.Counter = 0; } if (img1.BestVDistance != bestvdistance) { img1.BestVDistance = bestvdistance; } } img1.LastCheck = DateTime.Now; }
public static void RandomLe(int max, IProgress <string> progress) { lock (_rwlock) { lock (_imglock) { var imgcount = _imgList.Count; var diff = imgcount - _importLimit; if (diff > 0) { return; } DecreaseImportLimit(); } using (var random = new CryptoRandom()) { _rwList.Clear(); for (var i = 0; i < max; i++) { if (progress != null) { progress.Report($"importing {AppConsts.PathLe} ({i})..."); } var path = AppConsts.PathLe; do { var directoryInfo = new DirectoryInfo(path); var ds = directoryInfo.GetDirectories("*.*", SearchOption.TopDirectoryOnly).ToArray(); if (ds.Length > 0) { var rindex = random.Next(0, ds.Length - 1); path = ds[rindex].FullName; } else { var fs = directoryInfo.GetFiles("*.*", SearchOption.TopDirectoryOnly).ToArray(); if (fs.Length > 0) { var rindex = random.Next(0, fs.Length - 1); var filename = fs[rindex].FullName; var imagedata = File.ReadAllBytes(filename); using (var bitmap = BitmapHelper.ImageDataToBitmap(imagedata)) { if (bitmap == null) { progress.Report($"Bad {filename}"); return; } } _rwList.Add(fs[rindex]); break; } else { Directory.Delete(directoryInfo.FullName); break; } } }while (true); } progress.Report($"RandomLe import done!"); } } }