public DCGalleryList() { if (!File.Exists("ng_list.json")) { GalleryList = DCInsideUtils.GetGalleryList(); File.WriteAllText("ng_list.json", JsonConvert.SerializeObject(GalleryList)); } else { GalleryList = JsonConvert.DeserializeObject <SortedDictionary <string, string> >(File.ReadAllText("ng_list.json")); } if (!File.Exists("mg_list.json")) { MinorGalleryList = DCInsideUtils.GetMinorGalleryList(); File.WriteAllText("mg_list.json", JsonConvert.SerializeObject(MinorGalleryList)); } else { MinorGalleryList = JsonConvert.DeserializeObject <SortedDictionary <string, string> >(File.ReadAllText("mg_list.json")); } GalleryIds = new HashSet <string>(); MinorGalleryIds = new HashSet <string>(); GalleryList.ToList().ForEach(x => GalleryIds.Add(x.Value)); MinorGalleryList.ToList().ForEach(x => MinorGalleryIds.Add(x.Value)); Lists = new List <string>(); GalleryList.ToList().ForEach(x => Lists.Add($"{x.Key} 갤러리 ({x.Value})")); MinorGalleryList.ToList().ForEach(x => Lists.Add($"{x.Key} 마이너 갤러리 ({x.Value})")); Lists.Sort(); }
private void ThumbnailItem_Loaded(object sender, RoutedEventArgs e) { if (loaded) { return; } loaded = true; switch (Article.type) { case "icon_recomimg": case "icon_pic": Icon.Kind = MaterialDesignThemes.Wpf.PackIconKind.Image; break; default: Icon.Kind = MaterialDesignThemes.Wpf.PackIconKind.Pen; break; } if (Article.type == "icon_pic" || Article.type == "icon_recomimg") { Task.Run(() => { var html = NetTools.DownloadString(URL); if (html == null || html == "") { Extends.Post(() => { Icon.Kind = MaterialDesignThemes.Wpf.PackIconKind.TrashCanOutline; }); return; } downloaded_article = DCInsideUtils.ParseBoardView(html); if (downloaded_article.ImagesLink.Count == 0) { return; } temp_file = TemporaryFiles.UseNew(); NetTools.DownloadFile(downloaded_article.ImagesLink[0], temp_file); Extends.Post(() => AnimationBehavior.SetSourceUri(Image, new Uri(temp_file))); }); } }
public CreateNewProgress(string id, string name) { InitializeComponent(); Message.Text = $"{name}의 목록을 가져오는 중 입니다..."; Task.Run(() => { string url; if (DCGalleryList.Instance.MinorGalleryIds.Contains(id)) { url = $"https://gall.dcinside.com/mgallery/board/lists?id={id}"; } else { url = $"https://gall.dcinside.com/board/lists?id={id}"; } var node = NetTools.DownloadString(url).ToHtmlNode().SelectSingleNode("//a[@class='page_end']"); var page_end = 10; if (node != null) { page_end = node.GetAttributeValue("href", "").Split('=').Last().ToInt(); } var is_minor = DCGalleryList.Instance.MinorGalleryIds.Contains(id); var articles = new List <DCInsidePageArticle>(); for (int i = 1; i < page_end + 10; i++) { if (is_minor) { url = $"https://gall.dcinside.com/mgallery/board/lists/?id={id}&page={i}"; } else { url = $"https://gall.dcinside.com/board/lists/?id={id}&page={i}"; } var html = NetTools.DownloadString(url); DCInsideGallery gall; if (is_minor) { gall = DCInsideUtils.ParseMinorGallery(html); } else { gall = DCInsideUtils.ParseGallery(html); } if (is_minor && (gall.articles == null || gall.articles.Length == 0)) { gall = DCInsideUtils.ParseGallery(html); } if (gall.articles.Length == 0) { break; } articles.AddRange(gall.articles); Extends.Post(() => Message2.Text = $"작업 중...[{i}/{page_end + 10} | {(100.0 * i / (page_end + 10)).ToString("#0.00")}%]"); } var overlap = new HashSet <string>(); var articles_trim = new List <DCInsidePageArticle>(); foreach (var article in articles) { if (!overlap.Contains(article.no)) { articles_trim.Add(article); overlap.Add(article.no); } } articles_trim.Sort((x, y) => y.no.ToInt().CompareTo(x.no.ToInt())); var result = new DCInsideGalleryModel(); result.is_minor_gallery = is_minor; result.gallery_id = id; result.gallery_name = name; result.articles = articles_trim; var bbb = MessagePackSerializer.Serialize(result); using (FileStream fsStream = new FileStream($"list-{id}-{DateTime.Now.Ticks}.txt", FileMode.Create)) using (BinaryWriter sw = new BinaryWriter(fsStream)) { sw.Write(bbb); } Extends.Post(() => MainWindow.Instance.CloseDialog()); }); }
private void start() { int ps; if (!int.TryParse(textBox3.Text, out ps)) { append("숫자만 입력해주세요."); return; } int pe; if (!int.TryParse(textBox4.Text, out pe)) { append("숫자만 입력해주세요."); return; } var invalid = new string(Path.GetInvalidFileNameChars()) + new string(Path.GetInvalidPathChars()); var sp = Path.Combine(Directory.GetCurrentDirectory(), "Archive", $"몬무스 갤러리 (monmusu)"); var id = "monmusu"; Directory.CreateDirectory(sp); var starts = ps; status("진행중...[0/" + (pe - ps + 1).ToString("#,#") + "]"); bool real_cookie_receive = false; try { for (; ps <= pe; ps++) { string url; if (true) { url = $"https://gall.dcinside.com/mgallery/board/view/?id={id}&no={ps}"; } else { url = $"https://gall.dcinside.com/board/view/?id={id}&no={ps}"; } Logger.Instance.Push("Downloading String... = " + url); var task = NetTask.MakeDefault(url); task.Cookie = COOKIES; bool y = real_cookie_receive; if (real_cookie_receive == false) { task.HeaderReceive = (ef) => { append("헤더받음: " + ef); var xx = ef.Split('\n').First(x => x.Contains("Set-Cookie")).Replace("Set-Cookie: ", "").Trim(); COOKIES = "PHPSESSID=" + SESS; COOKIES += "; PHPSESSKEY=" + xx.Split(new[] { "PHPSESSKEY=" }, StringSplitOptions.None)[1].Split(';')[0].Trim(); COOKIES += "; block_alert_monmusu=1"; real_cookie_receive = true; append("쿠키 변경됨: " + COOKIES); }; } var html = NetTools.DownloadString(task); if (y == false) { append("PS1"); ps--; continue; } Logger.Instance.Push("Downloaded String = " + url); if (string.IsNullOrEmpty(html)) { append("실패: " + url); Logger.Instance.Push("Fail: " + url); goto NEXT; } if (html.Length < 1000 && html.Contains("해당 마이너 갤러리는 운영원칙 위반으로 접근이 제한되었습니다.")) { append("실패: 접근 거부, 접근 가능한 아이디로 재시도하시기 바랍니다."); break; } if (html.Contains("해당 마이너 갤러리는 운영원칙 위반(사유: 누드패치, 성행위 패치, 음란성 게시물 공지 등록 및 정리 안됨) 으로 접근이 제한되었습니다.")) { //goto F**K; return; } var info = DCInsideUtils.ParseBoardView(html, true); Logger.Instance.Push("Parse: " + url); // 해당 마이너 갤러리는 운영원칙 위반으로 접근이 제한되었습니다.\n마이너 갤러리 메인으로 돌아갑니다. var ttitle = $"{info.Title}"; foreach (char c in invalid) { ttitle = ttitle.Replace(c.ToString(), ""); } Logger.Instance.Push("ttile: " + ttitle); File.WriteAllText(Path.Combine(sp, $"[{ps}]-body-{ttitle}.json"), JsonConvert.SerializeObject(info, Formatting.Indented)); Logger.Instance.Push("file-body: " + $"[{ps}]-body-{ttitle}.json"); int com; if (int.TryParse(info.CommentCount.Replace(",", ""), out com) && com > 0) { try { var comments = DCInsideUtils.GetAllComments(id, ps.ToString()).Result; File.WriteAllText(Path.Combine(sp, $"[{ps}]-comments-{ttitle}.json"), JsonConvert.SerializeObject(comments, Formatting.Indented)); Logger.Instance.Push("file-comment: " + $"[{ps}]-comments-{ttitle}.json"); } catch { } } NEXT: var ss = TimeSpan.FromMilliseconds(720 * (pe - ps)); var yy = ""; if (ss.Days > 0) { yy += ss.Days.ToString() + "일 "; } if (ss.Days > 0 || ss.Hours > 0) { yy += ss.Hours.ToString() + "시간 "; } if (ss.Days > 0 || ss.Hours > 0 || ss.Minutes > 0) { yy += ss.Minutes.ToString() + "분 "; } if (ss.Days > 0 || ss.Hours > 0 || ss.Minutes > 0 || ss.Seconds > 0) { yy += ss.Seconds.ToString() + "초 남음"; } status("진행중...[" + (ps - starts + 1).ToString() + "/" + (pe - starts + 1).ToString("#,#") + "] 남은시간: " + yy); Logger.Instance.Push("next: " + url + $" || {ps}/{pe}/{starts}"); Thread.Sleep(700); } status("완료"); } catch (Exception e) { append("실패: " + e.Message + "\r\n" + e.StackTrace); } }
private void loop(object param) { int latest_index = DCGalleryAnalyzer.Instance.Articles[0].no.ToInt(); var rand = new Random(); var dict = new Dictionary <int, ThumbnailItem>(); int min_sleep = 2000; int rand_max_sleep = 3000; int rand_sleep; int loop_count = 0; while (true) { var url = ""; if (DCGalleryAnalyzer.Instance.Model.is_minor_gallery) { url = $"https://gall.dcinside.com/mgallery/board/lists/?id={DCGalleryAnalyzer.Instance.Model.gallery_id}&page={1}"; } else { url = $"https://gall.dcinside.com/board/lists/?id={DCGalleryAnalyzer.Instance.Model.gallery_id}&page={1}"; } var html = NetTools.DownloadString(url); DCInsideGallery gall = null; if (DCGalleryAnalyzer.Instance.Model.is_minor_gallery) { gall = DCInsideUtils.ParseMinorGallery(html); } else { gall = DCInsideUtils.ParseGallery(html); } if (DCGalleryAnalyzer.Instance.Model.is_minor_gallery && (gall.articles == null || gall.articles.Length == 0)) { gall = DCInsideUtils.ParseGallery(html); } var articles = gall.articles.ToList(); articles.Sort((x, y) => y.no.ToInt().CompareTo(x.no.ToInt())); // Update foreach (var article in articles) { if (dict.ContainsKey(article.no.ToInt())) { Extends.Post(() => dict[article.no.ToInt()].Update(article)); } } if (latest_index < articles[0].no.ToInt()) { int i = 0; while (latest_index < articles[i].no.ToInt()) { i++; } // If article regeneration delay is too short ... if (i >= 3 && loop_count >= 2) { rand_max_sleep = 1000; min_sleep = 500; } // Create while (--i >= 0) { Extends.Post(() => { var ti = new ThumbnailItem(articles[i], true); dict.Add(articles[i].no.ToInt(), ti); SignalPannel.Children.Insert(0, ti); }); Thread.Sleep(300); } latest_index = articles[0].no.ToInt(); } Extends.Post(() => { while (SignalPannel.Children.Count > 10) { dict.Remove((SignalPannel.Children[SignalPannel.Children.Count - 1] as ThumbnailItem).Article.no.ToInt()); SignalPannel.Children.RemoveAt(SignalPannel.Children.Count - 1); } }); rand_sleep = rand.Next(500, rand_max_sleep); Thread.Sleep(min_sleep + rand_sleep); loop_count++; } }
public SyncProgress() { InitializeComponent(); Message.Text = $"{DCGalleryAnalyzer.Instance.Model.gallery_name}의 목록을 가져오는 중 입니다..."; var name = DCGalleryAnalyzer.Instance.Model.gallery_name; var id = DCGalleryAnalyzer.Instance.Model.gallery_id; var is_minor = DCGalleryAnalyzer.Instance.Model.is_minor_gallery; Task.Run(() => { string url; if (is_minor) { url = $"https://gall.dcinside.com/mgallery/board/lists?id={id}"; } else { url = $"https://gall.dcinside.com/board/lists?id={id}"; } var html_board = NetTools.DownloadString(url); DCInsideGallery gall_board; if (is_minor) { gall_board = DCInsideUtils.ParseMinorGallery(html_board); } else { gall_board = DCInsideUtils.ParseGallery(html_board); } if (is_minor && (gall_board.articles == null || gall_board.articles.Length == 0)) { gall_board = DCInsideUtils.ParseGallery(html_board); } var ll = gall_board.articles.ToList(); ll.Sort((x, y) => y.no.ToInt().CompareTo(x.no.ToInt())); var latest = ll[0].no.ToInt(); var last = DCGalleryAnalyzer.Instance.Articles[0].no.ToInt(); var page_end = (latest - last) / 50; var articles = new List <DCInsidePageArticle>(); for (int i = 1; i < page_end + 3; i++) { if (is_minor) { url = $"https://gall.dcinside.com/mgallery/board/lists/?id={id}&page={i}"; } else { url = $"https://gall.dcinside.com/board/lists/?id={id}&page={i}"; } var html = NetTools.DownloadString(url); DCInsideGallery gall; if (is_minor) { gall = DCInsideUtils.ParseMinorGallery(html); } else { gall = DCInsideUtils.ParseGallery(html); } if (is_minor && (gall.articles == null || gall.articles.Length == 0)) { gall = DCInsideUtils.ParseGallery(html); } if (gall.articles.Length == 0) { break; } articles.AddRange(gall.articles); Extends.Post(() => Message2.Text = $"동기화 중...[{i}/{page_end + 3} | {(100.0 * i / (page_end + 3)).ToString("#0.00")}%]"); } articles.AddRange(DCGalleryAnalyzer.Instance.Articles); var overlap = new HashSet <string>(); var articles_trim = new List <DCInsidePageArticle>(); foreach (var article in articles) { if (!overlap.Contains(article.no)) { articles_trim.Add(article); overlap.Add(article.no); } } articles_trim.Sort((x, y) => y.no.ToInt().CompareTo(x.no.ToInt())); DCGalleryAnalyzer.Instance.Model.articles = articles_trim; DCGalleryAnalyzer.Instance.Save(); Extends.Post(() => MainWindow.Instance.CloseDialog()); }); }
private void start() { int ps; if (!int.TryParse(textBox3.Text, out ps)) { append("숫자만 입력해주세요."); return; } int pe; if (!int.TryParse(textBox4.Text, out pe)) { append("숫자만 입력해주세요."); return; } var id = "monmusu"; var starts = ps; status("진행중...[0/" + (pe - ps + 1).ToString("#,#") + "]"); bool real_cookie_receive = false; var articles = new List <DCInsidePageArticle>(); try { for (; ps <= pe; ps++) { string url; if (true) { url = $"https://gall.dcinside.com/mgallery/board/lists/?id={id}&page={ps}"; } else { url = $"https://gall.dcinside.com/board/lists/?id={id}&page={ps}"; } Logger.Instance.Push("Downloading String... = " + url); var task = NetTask.MakeDefault(url); task.Cookie = COOKIES; bool y = real_cookie_receive; if (real_cookie_receive == false) { task.HeaderReceive = (ef) => { append("헤더받음: " + ef); var xx = ef.Split('\n').First(x => x.Contains("Set-Cookie")).Replace("Set-Cookie: ", "").Trim(); COOKIES = "PHPSESSID=" + SESS; COOKIES += "; PHPSESSKEY=" + xx.Split(new[] { "PHPSESSKEY=" }, StringSplitOptions.None)[1].Split(';')[0].Trim(); COOKIES += $"; block_alert_{id}=1"; real_cookie_receive = true; append("쿠키 변경됨: " + COOKIES); }; } var html = NetTools.DownloadString(task); if (y == false) { append("PS1"); ps--; continue; } Logger.Instance.Push("Downloaded String = " + url); if (string.IsNullOrEmpty(html)) { append("실패: " + url); Logger.Instance.Push("Fail: " + url); goto NEXT; } if (html.Length < 1000 && html.Contains("해당 마이너 갤러리는 운영원칙 위반으로 접근이 제한되었습니다.")) { append("실패: 접근 거부, 접근 가능한 아이디로 재시도하시기 바랍니다."); break; } if (html.Contains("해당 마이너 갤러리는 운영원칙 위반(사유: 누드패치, 성행위 패치, 음란성 게시물 공지 등록 및 정리 안됨) 으로 접근이 제한되었습니다.")) { //goto F**K; return; } DCInsideGallery gall; if (true) { gall = DCInsideUtils.ParseMinorGallery(html); } else { gall = DCInsideUtils.ParseGallery(html); } if (true && (gall.articles == null || gall.articles.Length == 0)) { gall = DCInsideUtils.ParseGallery(html); } if (gall.articles.Length == 0) { break; } articles.AddRange(gall.articles); Logger.Instance.Push("Parse: " + url); // 해당 마이너 갤러리는 운영원칙 위반으로 접근이 제한되었습니다.\n마이너 갤러리 메인으로 돌아갑니다. NEXT: var ss = TimeSpan.FromMilliseconds(720 * (pe - ps)); var yy = ""; if (ss.Days > 0) { yy += ss.Days.ToString() + "일 "; } if (ss.Days > 0 || ss.Hours > 0) { yy += ss.Hours.ToString() + "시간 "; } if (ss.Days > 0 || ss.Hours > 0 || ss.Minutes > 0) { yy += ss.Minutes.ToString() + "분 "; } if (ss.Days > 0 || ss.Hours > 0 || ss.Minutes > 0 || ss.Seconds > 0) { yy += ss.Seconds.ToString() + "초 남음"; } status("진행중...[" + (ps - starts + 1).ToString() + "/" + (pe - starts + 1).ToString("#,#") + "] 남은시간: " + yy); Logger.Instance.Push("next: " + url + $" || {ps}/{pe}/{starts}"); Thread.Sleep(700); } DCGalleryAnalyzer.Instance.Articles.AddRange(articles); DCGalleryAnalyzer.Instance.Save(); status("완료"); } catch (Exception e) { append("실패: " + e.Message + "\r\n" + e.StackTrace); } }