public void UpdatesReferencesOnDuplicatedItem() { var fakeSite = new Sitecore.FakeDb.Sites.FakeSiteContext( new Sitecore.Collections.StringDictionary { { "name", "website" }, { "database", "master" }, { "content", "master" }, { "rootPath", "/sitecore/content" } }); using (new Sitecore.Sites.SiteContextSwitcher(fakeSite)) using (GetFakeDb()) { var db = Sitecore.Context.Database; var page1 = db.GetItem("/sitecore/content/home/page1"); var processor = new DuplicateItem(new TreeReferenceUpdater()); processor.Execute(new ClientPipelineArgs() { Parameters = new NameValueCollection() { { "database", "master" }, { "id", page1.ID.ToString() }, { "name", "page2" } }, }); var page2 = db.GetItem("/sitecore/content/home/page2"); db.GetItem(page2.Fields["Link"].Value).Parent.ID.Should().Be(page2.ID); } }
private List <Image>?GetVideoThumbnail(DuplicateItem videoFile, List <float> positions) { var ffMpeg = new FFmpegWrapper.FFmpegWrapper(); var images = new List <Image>(); try { for (var i = 0; i < positions.Count; i++) { var b = ffMpeg.GetVideoThumbnail(videoFile.Path, Convert.ToSingle(videoFile.Duration.TotalSeconds * positionList[i]), false); if (b == null || b.Length == 0) { return(null); } using var byteStream = new MemoryStream(b); var bitmapImage = Image.FromStream(byteStream); images.Add(bitmapImage); } } catch (FFmpegWrapper.FFMpegException ex) { Logger.Instance.Info($"FFMpegException, file: {videoFile.Path}, reason: {ex.Message}"); return(null); } catch (Exception ex) { Logger.Instance.Info($"Exception, file: {videoFile.Path}, reason: {ex.Message}, stacktrace {ex.StackTrace}"); return(null); } return(images); }
public ArrayList GetDuplicatesRoots() { //TODO: fish out title if places title is empty. if mbkm title is empty too .. try fetching from the other duplicates var reader = ExecuteQuery( "select mbkm.title as bkm_title, moz_places.title as places_title, url, fk, dupe_nr FROM (" + "SELECT title, fk, count(*) AS dupe_nr FROM moz_bookmarks WHERE fk NOT NULL GROUP BY fk" + ") mbkm INNER JOIN moz_places ON fk=moz_places.id WHERE dupe_nr > 1"); var ret = new ArrayList(); while (reader.Read()) { var di = new DuplicateItem { Title = reader["places_title"].ToString(), PathString = reader["url"].ToString(), Fk = (long)reader["fk"], DupeNr = (long)reader["dupe_nr"], IsRoot = true }; ret.Add(di); } _conn.Close(); return(ret); }
/// <summary> /// Invokes the pathSelectionEvent /// </summary> /// <param name="di">The selected item.</param> protected virtual void OnPathSelectionEvent(DuplicateItem di) { if (PathSelectionEvent != null) { PathSelectionEvent(di); } }
public DuplicateItemViewModel(DuplicateItem item) { ItemInfo = item; ItemInfo.ThumbnailsUpdated += () => { Thumbnail = ImageUtils.JoinImages(ItemInfo.ImageList); OnPropertyChanged(nameof(Thumbnail)); }; }
public DuplicateItemVM(DuplicateItem item) { ItemInfo = item; ItemInfo.ThumbnailsUpdated += () => { Thumbnail = ImageUtils.JoinImages(ItemInfo.ImageList) !; this.RaisePropertyChanged(nameof(Thumbnail)); }; }
public void Emit(LogEvent logEvent) { var messageTemplateText = logEvent.MessageTemplate.Text; var key = Hash(logEvent.Level.ToString() + "-" + messageTemplateText); if (!_memoryCache.TryGetValue <DuplicateItem>(key, out DuplicateItem duplicateItem)) { duplicateItem = new DuplicateItem(messageTemplateText); _memoryCache.Set(key, duplicateItem, _blockTimeSpan); duplicateItem.IncrementDuplicateCount(); _logEventSink.Emit(logEvent); } }
private void dataGridView1_SelectionChanged(object sender, EventArgs e) { if (dataGridView1.SelectedRows.Count == 0) { return; } int index = dataGridView1.SelectedRows[0].Index, pos = 0; DuplicateItem item = items[index]; foreach (DuplicateItem item2 in items) { if (item == item2) { item2.Similarity = "---"; } else { int dist = DHash.Distance(item.image.hash, item2.image.hash); item2.Similarity = $"{DHash.ToSimilarity(dist)}%"; } if (!inInit) { bindingList.ResetItem(pos); ++pos; } } if (pictureBox1.Image != null) { pictureBox1.Image.Dispose(); } try { pictureBox1.Image = ImageLoader.Load(item.image.path); } catch (Exception) { pictureBox1.Image = null; } }
private static List <Image>?GetImageThumbnail(DuplicateItem videoFile, int count) { var images = new List <Image>(); for (var i = 0; i < count; i++) { Image bitmapImage; try { bitmapImage = Image.FromFile(videoFile.Path); } catch (Exception ex) { Logger.Instance.Info($"Exception, file: {videoFile.Path}, reason: {ex.Message}, stacktrace {ex.StackTrace}"); return(null); } //Fill some missing data now when we have the information videoFile.FrameSize = $"{bitmapImage.Width}x{bitmapImage.Height}"; videoFile.FrameSizeInt = bitmapImage.Width + bitmapImage.Height; double resizeFactor = 1; if (bitmapImage.Width > 100 || bitmapImage.Height > 100) { double widthFactor = Convert.ToDouble(bitmapImage.Width) / 100; double heightFactor = Convert.ToDouble(bitmapImage.Height) / 100; resizeFactor = Math.Max(widthFactor, heightFactor); } int width = Convert.ToInt32(bitmapImage.Width / resizeFactor); int height = Convert.ToInt32(bitmapImage.Height / resizeFactor); var newImage = new Bitmap(width, height); using (var g = Graphics.FromImage(newImage)) { g.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.HighQualityBicubic; g.DrawImage(bitmapImage, 0, 0, newImage.Width, newImage.Height); } bitmapImage.Dispose(); images.Add(newImage); } return(images); }
public ArrayList GetDuplicatesChildren(DuplicateItem uniqueItem) { var reader = ExecuteQuery("select * from moz_bookmarks where fk = " + uniqueItem.Fk); var ret = new ArrayList(); while (reader.Read()) { var di = new DuplicateItem { Title = reader["title"].ToString(), Parent = (long)reader["parent"], Id = (long)reader["id"], LastModified = (long)reader["lastModified"], DateAdded = (long)reader["dateAdded"] }; di.PathNodes = GetPath(di.Parent); di.IsRoot = false; ret.Add(di); } _conn.Close(); return(ret); }
static void Main(string[] args) { bool _verbose = args.Contains("-v") || args.Contains("--verbose"); bool _debug = args.Contains("-d") || args.Contains("--debug"); Stopwatch sw = new Stopwatch(); ConsoleColor _defaultColor = Console.ForegroundColor; rerun: sw.Restart(); Console.WriteLine("Please specify the minimum count as duplicates (default is 2):"); if (_verbose) { Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine($" eg. this will be used to return the items that match and exceed a count of \"x\" items."); Console.ForegroundColor = _defaultColor; } int _minCount = 2; string?_minCountGiven = Console.ReadLine(); if (!String.IsNullOrEmpty(_minCountGiven) && int.TryParse(_minCountGiven, out _minCount)) { Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine($" successfully changed minimum count for duplicate validations to {_minCount}"); Console.ForegroundColor = _defaultColor; } if (_debug) { Console.ForegroundColor = ConsoleColor.DarkGray; Console.WriteLine($" debug: mincount set to {_minCount}"); Console.ForegroundColor = _defaultColor; } Console.WriteLine("Please specify the directory where the files are:"); string _folder = Console.ReadLine(); string folder = System.IO.Directory.GetCurrentDirectory(); if (!String.IsNullOrEmpty(_folder) && System.IO.Directory.Exists(_folder)) { folder = _folder; } if (_debug) { Console.ForegroundColor = ConsoleColor.DarkGray; Console.WriteLine($" debug: folder is set to {folder}"); Console.ForegroundColor = _defaultColor; } Console.WriteLine("Please specify the file extensions of files you want inspected:"); string _extension = Console.ReadLine(); if (String.IsNullOrEmpty(_extension)) { _extension = $"*"; } if (_debug) { Console.ForegroundColor = ConsoleColor.DarkGray; Console.WriteLine($" debug: files with the {_extension} extension will be analysed"); Console.ForegroundColor = _defaultColor; } Console.WriteLine("Please specify the content that will be looked for (key, etc):"); string _contains = Console.ReadLine(); if (_debug) { Console.ForegroundColor = ConsoleColor.DarkGray; Console.WriteLine($" debug: lines containing {_contains} will be analysed"); Console.ForegroundColor = _defaultColor; } Console.WriteLine($" analysing files for duplicates in {folder} ..."); List <String> files = new List <string>(); List <DuplicateItem> _duplicates = new List <DuplicateItem>(); int totalFiles = 0; int totalDuplicates = 0; foreach (var filepath in System.IO.Directory.GetFiles(folder, $"*.{_extension}", System.IO.SearchOption.AllDirectories)) { if (_debug) { Console.ForegroundColor = ConsoleColor.DarkGray; Console.WriteLine($" debug: analysing file \"{System.IO.Path.GetFileName(filepath)}\""); Console.ForegroundColor = _defaultColor; } totalFiles++; int _linenumber = 0; string line; System.IO.StreamReader file = new System.IO.StreamReader(filepath); while ((line = file.ReadLine()) != null) { _linenumber++; if (line.Contains(_contains)) { //string _key = $"{System.Math.Abs(filepath.GetHashCode())}-{System.Math.Abs(_contains.GetHashCode())}"; string _key = $"{System.Math.Abs(line.GetHashCode())}"; var _duplicate = _duplicates.Find(x => x.Key == _key) ?? null; if (_duplicate == null) { _duplicate = new DuplicateItem(); _duplicate.Count = 1; _duplicate.LineItems = new List <LineItem>() { new LineItem() { FilePath = $"{filepath.Trim()}", LineNumber = _linenumber } }; _duplicate.Key = _key; _duplicate.Value = line.Trim(); _duplicates.Add(_duplicate); } else { _duplicate.LineItems.Add(new LineItem() { FilePath = $"{filepath.Trim()}", LineNumber = _linenumber }); _duplicate.Count++; } totalDuplicates++; } } } Console.WriteLine($""); Console.WriteLine($"Here are all the files with duplicate {_contains}"); foreach (var _duplicate in _duplicates.FindAll(x => x.Count >= _minCount)) { Console.Write($"Found "); Console.ForegroundColor = ConsoleColor.Red; Console.Write($"{_duplicate.Count} duplicates"); Console.ForegroundColor = _defaultColor; Console.Write($" of "); Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine($"{_duplicate.Value}"); Console.ForegroundColor = _defaultColor; foreach (var line in _duplicate.LineItems) { Console.Write($" in: {line.FilePath} "); Console.ForegroundColor = ConsoleColor.Cyan; Console.WriteLine($"(line {line.LineNumber})"); Console.ForegroundColor = _defaultColor; } } Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine($"Would you like to export these results? (Y)es or (N)o, default is (N)o."); ConsoleKey _keyExport = Console.ReadKey(true).Key; if (_keyExport == ConsoleKey.Y) { Console.ForegroundColor = ConsoleColor.DarkYellow; Console.WriteLine($"Results:"); Console.ForegroundColor = _defaultColor; var _outputFile = $"duplicatefinder-{DateTime.Now.ToString("yyyyMMdd")}"; var _outputJson = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(folder), $"{_outputFile}.json"); var _json = JsonConvert.SerializeObject(_duplicates); Console.WriteLine($" creating json output to {_outputJson} ..."); System.IO.File.WriteAllText(_outputJson, _json); var _outputYaml = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(folder), $"{_outputFile}.yaml"); var serializer = new YamlDotNet.Serialization.Serializer(); //serializer.Serialize(Console.Out, _duplicates); Console.WriteLine($" creating yaml output to {_outputYaml} ..."); serializer.Serialize(System.IO.File.CreateText(_outputYaml), _duplicates); } Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine($" analysis of {totalDuplicates} duplicates in {totalFiles} files completed in {sw.Elapsed}"); Console.ForegroundColor = _defaultColor; Console.WriteLine($"-------------------------------------------------------------------------------------"); sw.Reset(); goto rerun; }
/// <summary> /// Path Selection Event Handler for DuplicatesView /// </summary> /// <param name="di">Duplicate item to be selected in the main views.</param> private void DuplicatesView_pathSelectionEvent(DuplicateItem di) { SelectContentItem(di.PathNodes, di.Id); }
public static void Start() { try { sb.AppendLine(string.Format("开始扫描 {0}", DateTime.Now.ToLongTimeString())); //var drivers = Environment.GetLogicalDrives().Skip(1).ToList(); List <FileInfo> fi = new List <FileInfo>(); List <Match> temp = new List <Match>(); foreach (var driver in folders) { sb.AppendLine(string.Format("添加扫描驱动器: {0}", driver)); Console.WriteLine("Processing " + driver); if (!string.IsNullOrEmpty(FileUtility.GetFilesRecursive(driver, formats, excludes, fi, 100))) { sb.AppendLine("获取文件失败"); } } var avs = JavDataBaseManager.GetAllAV(); var prefix = FileUtility.GetPrefix(avs); Console.WriteLine("Fi -> " + fi.Count); Console.WriteLine("AV -> " + avs.Count + " Prefix -> " + prefix.Count); sb.AppendLine(string.Format("符合条件文件: {0}, 总共AV: {1}", fi.Count, avs.Count)); foreach (var file in fi) { var scan = new Scan { FileName = file.Name.Trim().ToUpper(), Location = file.DirectoryName.Trim().ToLower(), Size = FileSize.GetAutoSizeString(file.Length, 2) }; var possibleIDs = FileUtility.GetPossibleID(scan, prefix); possibleIDs = possibleIDs.OrderByDescending(x => x.Length).Take(1).ToList(); sb.AppendLine(string.Format("文件{0}可能的Match有{1}", file, possibleIDs.Count)); Console.WriteLine("PossibleIDs -> " + possibleIDs.Count); AddTemp(scan, possibleIDs, temp); } Console.WriteLine("Temp -> " + temp.Count); sb.AppendLine(string.Format("一共找到{0}个Match", temp.Count)); ScanDataBaseManager.ClearMatch(); foreach (var m in temp) { Console.WriteLine(string.Format("Insert {0}\\{1}", m.Location, m.Name)); sb.AppendLine(string.Format("在库中添加Match -> {0}", m.Location)); ScanDataBaseManager.SaveMatch(m); } sb.AppendLine("更新数据库状态"); ScanDataBaseManager.InsertFinish(); var duplicateItemList = new List <DuplicateItem>(); var tempDic = temp.GroupBy(x => x.AvID.ToLower()).ToDictionary(x => x.Key, y => y.ToList()); foreach (var item in tempDic) { var tempItem = new DuplicateItem(); tempItem.AvId = item.Key; tempItem.Matches = item.Value; duplicateItemList.Add(tempItem); } var jsonRoot = "C:/AvLog/"; var jsonStr = JsonConvert.SerializeObject(duplicateItemList).Replace("\\", "\\\\"); var jsonFile = "ScanJson" + DateTime.Now.ToString("yyyy-MM-dd-hh-mm-ss") + ".json"; LogHelper.WriteLog(jsonFile, jsonStr); EmailHelper.SendEmail("ScanJson", "详情见附件", new[] { "*****@*****.**" }, new[] { jsonRoot + jsonFile }); } catch (Exception e) { sb.AppendLine(e.ToString()); } finally { sb.AppendLine(string.Format("扫描结束 {0}", DateTime.Now.ToLongTimeString())); //var root = "C:/AvLog/"; //var file = "ScanAndMatch" + DateTime.Now.ToString("yyyy-MM-dd-hh-mm-ss") + "-log.txt"; //LogHelper.WriteLog(file, sb.ToString()); //EmailHelper.SendEmail("ScanAndMatchLog", "详情见附件", new[] { "*****@*****.**" }, new[] { root + file }); } }
private void InternalSearch(CancellationToken cancelToken, PauseTokenSource pauseTokenSource) { ElapsedTimer.Start(); var startTime = DateTime.Now; processedFiles = 0; var lastProgressUpdate = DateTime.MinValue; var progressUpdateItvl = TimeSpan.FromMilliseconds(300); void IncrementProgress(Action <TimeSpan> fn) { Interlocked.Increment(ref processedFiles); var pushUpdate = processedFiles == ScanProgressMaxValue || lastProgressUpdate + progressUpdateItvl < DateTime.Now; if (!pushUpdate) { return; } lastProgressUpdate = DateTime.Now; var timeRemaining = TimeSpan.FromTicks(DateTime.Now.Subtract(startTime).Ticks * (ScanProgressMaxValue - (processedFiles + 1)) / (processedFiles + 1)); fn(timeRemaining); } try { var st = Stopwatch.StartNew(); var reScanList = ScanFileList.Where(vf => (vf.mediaInfo == null && !vf.IsImage) || vf.grayBytes == null).ToList(); ScanProgressMaxValue = reScanList.Count; Parallel.For(0, reScanList.Count, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, CancellationToken = cancelToken }, i => { while (pauseTokenSource.IsPaused) { Thread.Sleep(50); } var entry = reScanList[i]; if (entry.mediaInfo == null && !entry.IsImage) { var ffProbe = new FFProbeWrapper.FFProbeWrapper(); var info = ffProbe.GetMediaInfo(entry.Path); if (info == null) { return; } entry.mediaInfo = info; } if (entry.grayBytes == null) { entry.grayBytes = entry.IsImage ? GetImageAsBitmaps(entry, positionList.Count) : GetVideoThumbnailAsBitmaps(entry, positionList); if (entry.grayBytes == null) { return; } } //report progress IncrementProgress(remaining => Progress?.Invoke(this, new OwnScanProgress { CurrentPosition = processedFiles, CurrentFile = entry.Path, Elapsed = ElapsedTimer.Elapsed, Remaining = remaining })); }); st.Stop(); Logger.Instance.Info(string.Format(Properties.Resources.ThumbnailsFinished, st.Elapsed, processedFiles)); processedFiles = 0; ScanProgressMaxValue = ScanFileList.Count; st.Restart(); startTime = DateTime.Now; var percentageDifference = 1.0f - Settings.Percent / 100f; Parallel.For(0, ScanFileList.Count, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, CancellationToken = cancelToken }, i => { while (pauseTokenSource.IsPaused) { Thread.Sleep(50); } foreach (var itm in ScanFileList) { if (itm == ScanFileList[i] || itm.IsImage && !ScanFileList[i].IsImage) { continue; } if (itm.grayBytes == null || itm.grayBytes.Count == 0) { continue; } if (ScanFileList[i].grayBytes == null || ScanFileList[i].grayBytes.Count == 0) { continue; } if (itm.grayBytes.Count != ScanFileList[i].grayBytes.Count) { continue; } var duplicateCounter = 0; var percent = new float[itm.grayBytes.Count]; for (var j = 0; j < itm.grayBytes.Count; j++) { percent[j] = ExtensionMethods.PercentageDifference2(itm.grayBytes[j], ScanFileList[i].grayBytes[j]); if (percent[j] < percentageDifference) { duplicateCounter++; } else { break; } } if (duplicateCounter != itm.grayBytes.Count) { continue; } lock (AddDuplicateLock) { var firstInList = false; var secondInList = false; var groupId = Guid.NewGuid(); foreach (var v in Duplicates) { if (v.Path == itm.Path) { groupId = v.GroupId; firstInList = true; } else if (v.Path == ScanFileList[i].Path) { secondInList = true; } } if (!firstInList) { var origDup = new DuplicateItem(itm, percent.Average()) { GroupId = groupId }; var origImages = itm.IsImage ? GetImageThumbnail(origDup, positionList.Count) : GetVideoThumbnail(origDup, positionList); if (origImages == null) { continue; } origDup.Thumbnail = origImages; Duplicates.Add(origDup); } if (!secondInList) { var dup = new DuplicateItem(ScanFileList[i], percent.Average()) { GroupId = groupId }; var images = ScanFileList[i].IsImage ? GetImageThumbnail(dup, positionList.Count) : GetVideoThumbnail(dup, positionList); if (images == null) { continue; } dup.Thumbnail = images; Duplicates.Add(dup); } } //we found a matching source then duplicate was added no need to go deeper break; } //report progress IncrementProgress(remaining => Progress?.Invoke(this, new OwnScanProgress { CurrentPosition = processedFiles, CurrentFile = ScanFileList[i].Path, Elapsed = ElapsedTimer.Elapsed, Remaining = remaining })); }); st.Stop(); Logger.Instance.Info(string.Format(Properties.Resources.DuplicatesCheckFinishedIn, st.Elapsed)); } catch (OperationCanceledException) { Logger.Instance.Info(Properties.Resources.CancellationExceptionCaught); } }
private void btApply_Click(object sender, EventArgs e) { try { if (items.All(x => x.action == "remove")) { throw new Exception("All images set to remove!"); } List <string> replaces = new List <string>(); for (int i = 0; i < items.Count; ++i) { DuplicateItem item = items[i]; if (item.action.StartsWith("replace")) { int index = int.Parse(item.action.Substring("replace".Length)); if (replaces.Contains(item.action)) { throw new Exception($"{item.Filename} - Replace with '{items[index]}' already used!"); } if (index == i) { throw new Exception($"{item.Filename} - Can't replace with itself!"); } if (items[index].action != "keep") { throw new Exception($"{item.Filename} - Can't replace with '{items[index]}', image not kept!"); } if (items[index].image.score == 0) { throw new Exception($"{item.Filename} - Can't replace with '{items[index]}', image not scored - use remove!"); } } else if (item.action == "remove" && item.image.score != 0) { throw new Exception($"{item.Filename} - Can't remove scored image, must replace."); } } pictureBox1.Image?.Dispose(); pictureBox1.Image = null; for (int i = 0; i < items.Count; ++i) { DuplicateItem item = items[i]; if (item.action.StartsWith("replace")) { int index = int.Parse(item.action.Substring("replace".Length)); int imgIndex = db.sortedImages.IndexOf(item.image); Image replacement = items[index].image; replacement.score = item.image.score; db.sortedImages[imgIndex] = replacement; db.imagesDict.Remove(item.image.Filename); db.duplicates.RemoveAll(x => x.image1 == item.image || x.image2 == item.image); Utility.SafeDelete(this, item.image.path); ++replaced; } else if (item.action == "remove") { db.newImages.Remove(item.image); db.imagesDict.Remove(item.Filename); db.duplicates.RemoveAll(x => x.image1 == item.image || x.image2 == item.image); Utility.SafeDelete(this, item.image.path); ++removed; } } DialogResult = DialogResult.OK; Close(); } catch (Exception ex) { MessageBox.Show(ex.Message); } }