static void ProcessEqualClustering(string[] args) { var iss = new ImageEqualSimilarity(); Console.Instance.WriteLine("이미지들을 해싱하는 중..."); using (var progressBar = new Console.ConsoleProgressBar()) { var files = Directory.GetFiles(args[0]).Where(x => x.EndsWith(".gif") || x.EndsWith(".png") || x.EndsWith(".jpg") || x.EndsWith(".jpeg") || x.EndsWith(".webp") || x.EndsWith(".bmp")); int counts = files.Count(); int complete = 0; Parallel.ForEach(files, x => { iss.AppendImage(x); progressBar.SetProgress(Interlocked.Increment(ref complete) / (float)counts * 100); }); } var clustered = iss.Clustering(); clustered.RemoveAll(x => x.Count == 1); clustered.Sort((x, y) => y.Count.CompareTo(x.Count)); Console.Instance.WriteLine(clustered); }
static void ProcessCollectArticles(string[] args) { var rstarts = Convert.ToInt32(args[1]); var starts = Convert.ToInt32(args[1]); var ends = Convert.ToInt32(args[2]); bool is_minorg = !DCGalleryList.Instance.GalleryIds.Contains(args[0]); var result = new DCInsideGalleryModel(); var articles = new List <DCInsidePageArticle>(); using (var progressBar = new Console.ConsoleProgressBar()) { for (; starts <= ends; starts++) { var url = ""; if (is_minorg) { url = $"https://gall.dcinside.com/mgallery/board/lists/?id={args[0]}&page={starts}"; } else { url = $"https://gall.dcinside.com/board/lists/?id={args[0]}&page={starts}"; } Console.Instance.WriteLine($"Download URL: {url}"); var html = NetTools.DownloadString(url); DCInsideGallery gall = null; if (is_minorg) { gall = DCInsideUtils.ParseMinorGallery(html); } else { gall = DCInsideUtils.ParseGallery(html); } if (is_minorg && (gall.articles == null || gall.articles.Length == 0)) { gall = DCInsideUtils.ParseGallery(html); } articles.AddRange(gall.articles); progressBar.SetProgress((((ends - rstarts + 1) - (ends - starts)) / (float)(ends - rstarts + 1)) * 100); } var overlap = new HashSet <string>(); var articles_trim = new List <DCInsidePageArticle>(); foreach (var article in articles) { if (!overlap.Contains(article.no)) { articles_trim.Add(article); overlap.Add(article.no); } } articles_trim.Sort((x, y) => y.no.ToInt().CompareTo(x.no.ToInt())); result.is_minor_gallery = is_minorg; result.gallery_id = args[0]; result.articles = articles_trim; File.WriteAllText($"list-{args[0]}-{DateTime.Now.Ticks}.txt", JsonConvert.SerializeObject(result)); var bbb = MessagePackSerializer.Serialize(result); using (FileStream fsStream = new FileStream($"list-{args[0]}-{DateTime.Now.Ticks}-index.txt", FileMode.Create)) using (BinaryWriter sw = new BinaryWriter(fsStream)) { sw.Write(bbb); } } }