public CreateNewProgress(string id, string name)
{
InitializeComponent();
Message.Text = $"{name}의 목록을 가져오는 중 입니다...";
Task.Run(() =>
{
string url;
if (DCGalleryList.Instance.MinorGalleryIds.Contains(id))
{
url = $"https://gall.dcinside.com/mgallery/board/lists?id={id}";
}
else
{
url = $"https://gall.dcinside.com/board/lists?id={id}";
}
var node = NetTools.DownloadString(url).ToHtmlNode().SelectSingleNode("//a[@class='page_end']");
var page_end = 10;
if (node != null)
{
page_end = node.GetAttributeValue("href", "").Split('=').Last().ToInt();
}
var is_minor = DCGalleryList.Instance.MinorGalleryIds.Contains(id);
var articles = new List <DCInsidePageArticle>();
for (int i = 1; i < page_end + 10; i++)
{
if (is_minor)
{
url = $"https://gall.dcinside.com/mgallery/board/lists/?id={id}&page={i}";
}
else
{
url = $"https://gall.dcinside.com/board/lists/?id={id}&page={i}";
}
var html = NetTools.DownloadString(url);
DCInsideGallery gall;
if (is_minor)
{
gall = DCInsideUtils.ParseMinorGallery(html);
}
else
{
gall = DCInsideUtils.ParseGallery(html);
}
if (is_minor && (gall.articles == null || gall.articles.Length == 0))
{
gall = DCInsideUtils.ParseGallery(html);
}
if (gall.articles.Length == 0)
{
break;
}
articles.AddRange(gall.articles);
Extends.Post(() => Message2.Text = $"작업 중...[{i}/{page_end + 10} | {(100.0 * i / (page_end + 10)).ToString("#0.00")}%]");
}
var overlap = new HashSet <string>();
var articles_trim = new List <DCInsidePageArticle>();
foreach (var article in articles)
{
if (!overlap.Contains(article.no))
{
articles_trim.Add(article);
overlap.Add(article.no);
}
}
articles_trim.Sort((x, y) => y.no.ToInt().CompareTo(x.no.ToInt()));
var result = new DCInsideGalleryModel();
result.is_minor_gallery = is_minor;
result.gallery_id = id;
result.gallery_name = name;
result.articles = articles_trim;
var bbb = MessagePackSerializer.Serialize(result);
using (FileStream fsStream = new FileStream($"list-{id}-{DateTime.Now.Ticks}.txt", FileMode.Create))
using (BinaryWriter sw = new BinaryWriter(fsStream))
{
sw.Write(bbb);
}
Extends.Post(() => MainWindow.Instance.CloseDialog());
});
}
public SyncProgress()
{
InitializeComponent();
Message.Text = $"{DCGalleryAnalyzer.Instance.Model.gallery_name}의 목록을 가져오는 중 입니다...";
var name = DCGalleryAnalyzer.Instance.Model.gallery_name;
var id = DCGalleryAnalyzer.Instance.Model.gallery_id;
var is_minor = DCGalleryAnalyzer.Instance.Model.is_minor_gallery;
Task.Run(() =>
{
string url;
if (is_minor)
{
url = $"https://gall.dcinside.com/mgallery/board/lists?id={id}";
}
else
{
url = $"https://gall.dcinside.com/board/lists?id={id}";
}
var html_board = NetTools.DownloadString(url);
DCInsideGallery gall_board;
if (is_minor)
{
gall_board = DCInsideUtils.ParseMinorGallery(html_board);
}
else
{
gall_board = DCInsideUtils.ParseGallery(html_board);
}
if (is_minor && (gall_board.articles == null || gall_board.articles.Length == 0))
{
gall_board = DCInsideUtils.ParseGallery(html_board);
}
var ll = gall_board.articles.ToList();
ll.Sort((x, y) => y.no.ToInt().CompareTo(x.no.ToInt()));
var latest = ll[0].no.ToInt();
var last = DCGalleryAnalyzer.Instance.Articles[0].no.ToInt();
var page_end = (latest - last) / 50;
var articles = new List <DCInsidePageArticle>();
for (int i = 1; i < page_end + 3; i++)
{
if (is_minor)
{
url = $"https://gall.dcinside.com/mgallery/board/lists/?id={id}&page={i}";
}
else
{
url = $"https://gall.dcinside.com/board/lists/?id={id}&page={i}";
}
var html = NetTools.DownloadString(url);
DCInsideGallery gall;
if (is_minor)
{
gall = DCInsideUtils.ParseMinorGallery(html);
}
else
{
gall = DCInsideUtils.ParseGallery(html);
}
if (is_minor && (gall.articles == null || gall.articles.Length == 0))
{
gall = DCInsideUtils.ParseGallery(html);
}
if (gall.articles.Length == 0)
{
break;
}
articles.AddRange(gall.articles);
Extends.Post(() => Message2.Text = $"동기화 중...[{i}/{page_end + 3} | {(100.0 * i / (page_end + 3)).ToString("#0.00")}%]");
}
articles.AddRange(DCGalleryAnalyzer.Instance.Articles);
var overlap = new HashSet <string>();
var articles_trim = new List <DCInsidePageArticle>();
foreach (var article in articles)
{
if (!overlap.Contains(article.no))
{
articles_trim.Add(article);
overlap.Add(article.no);
}
}
articles_trim.Sort((x, y) => y.no.ToInt().CompareTo(x.no.ToInt()));
DCGalleryAnalyzer.Instance.Model.articles = articles_trim;
DCGalleryAnalyzer.Instance.Save();
Extends.Post(() => MainWindow.Instance.CloseDialog());
});
}
private void loop(object param)
{
int latest_index = DCGalleryAnalyzer.Instance.Articles[0].no.ToInt();
var rand = new Random();
var dict = new Dictionary <int, ThumbnailItem>();
int min_sleep = 2000;
int rand_max_sleep = 3000;
int rand_sleep;
int loop_count = 0;
while (true)
{
var url = "";
if (DCGalleryAnalyzer.Instance.Model.is_minor_gallery)
{
url = $"https://gall.dcinside.com/mgallery/board/lists/?id={DCGalleryAnalyzer.Instance.Model.gallery_id}&page={1}";
}
else
{
url = $"https://gall.dcinside.com/board/lists/?id={DCGalleryAnalyzer.Instance.Model.gallery_id}&page={1}";
}
var html = NetTools.DownloadString(url);
DCInsideGallery gall = null;
if (DCGalleryAnalyzer.Instance.Model.is_minor_gallery)
{
gall = DCInsideUtils.ParseMinorGallery(html);
}
else
{
gall = DCInsideUtils.ParseGallery(html);
}
if (DCGalleryAnalyzer.Instance.Model.is_minor_gallery &&
(gall.articles == null || gall.articles.Length == 0))
{
gall = DCInsideUtils.ParseGallery(html);
}
var articles = gall.articles.ToList();
articles.Sort((x, y) => y.no.ToInt().CompareTo(x.no.ToInt()));
// Update
foreach (var article in articles)
{
if (dict.ContainsKey(article.no.ToInt()))
{
Extends.Post(() => dict[article.no.ToInt()].Update(article));
}
}
if (latest_index < articles[0].no.ToInt())
{
int i = 0;
while (latest_index < articles[i].no.ToInt())
{
i++;
}
// If article regeneration delay is too short ...
if (i >= 3 && loop_count >= 2)
{
rand_max_sleep = 1000;
min_sleep = 500;
}
// Create
while (--i >= 0)
{
Extends.Post(() =>
{
var ti = new ThumbnailItem(articles[i], true);
dict.Add(articles[i].no.ToInt(), ti);
SignalPannel.Children.Insert(0, ti);
});
Thread.Sleep(300);
}
latest_index = articles[0].no.ToInt();
}
Extends.Post(() =>
{
while (SignalPannel.Children.Count > 10)
{
dict.Remove((SignalPannel.Children[SignalPannel.Children.Count - 1] as ThumbnailItem).Article.no.ToInt());
SignalPannel.Children.RemoveAt(SignalPannel.Children.Count - 1);
}
});
rand_sleep = rand.Next(500, rand_max_sleep);
Thread.Sleep(min_sleep + rand_sleep);
loop_count++;
}
}
private void start()
{
int ps;
if (!int.TryParse(textBox3.Text, out ps))
{
append("숫자만 입력해주세요.");
return;
}
int pe;
if (!int.TryParse(textBox4.Text, out pe))
{
append("숫자만 입력해주세요.");
return;
}
var id = "monmusu";
var starts = ps;
status("진행중...[0/" + (pe - ps + 1).ToString("#,#") + "]");
bool real_cookie_receive = false;
var articles = new List <DCInsidePageArticle>();
try
{
for (; ps <= pe; ps++)
{
string url;
if (true)
{
url = $"https://gall.dcinside.com/mgallery/board/lists/?id={id}&page={ps}";
}
else
{
url = $"https://gall.dcinside.com/board/lists/?id={id}&page={ps}";
}
Logger.Instance.Push("Downloading String... = " + url);
var task = NetTask.MakeDefault(url);
task.Cookie = COOKIES;
bool y = real_cookie_receive;
if (real_cookie_receive == false)
{
task.HeaderReceive = (ef) =>
{
append("헤더받음: " + ef);
var xx = ef.Split('\n').First(x => x.Contains("Set-Cookie")).Replace("Set-Cookie: ", "").Trim();
COOKIES = "PHPSESSID=" + SESS;
COOKIES += "; PHPSESSKEY=" + xx.Split(new[] { "PHPSESSKEY=" }, StringSplitOptions.None)[1].Split(';')[0].Trim();
COOKIES += $"; block_alert_{id}=1";
real_cookie_receive = true;
append("쿠키 변경됨: " + COOKIES);
};
}
var html = NetTools.DownloadString(task);
if (y == false)
{
append("PS1");
ps--;
continue;
}
Logger.Instance.Push("Downloaded String = " + url);
if (string.IsNullOrEmpty(html))
{
append("실패: " + url);
Logger.Instance.Push("Fail: " + url);
goto NEXT;
}
if (html.Length < 1000 && html.Contains("해당 마이너 갤러리는 운영원칙 위반으로 접근이 제한되었습니다."))
{
append("실패: 접근 거부, 접근 가능한 아이디로 재시도하시기 바랍니다.");
break;
}
if (html.Contains("해당 마이너 갤러리는 운영원칙 위반(사유: 누드패치, 성행위 패치, 음란성 게시물 공지 등록 및 정리 안됨) 으로 접근이 제한되었습니다."))
{
//goto F**K;
return;
}
DCInsideGallery gall;
if (true)
{
gall = DCInsideUtils.ParseMinorGallery(html);
}
else
{
gall = DCInsideUtils.ParseGallery(html);
}
if (true && (gall.articles == null || gall.articles.Length == 0))
{
gall = DCInsideUtils.ParseGallery(html);
}
if (gall.articles.Length == 0)
{
break;
}
articles.AddRange(gall.articles);
Logger.Instance.Push("Parse: " + url);
// 해당 마이너 갤러리는 운영원칙 위반으로 접근이 제한되었습니다.\n마이너 갤러리 메인으로 돌아갑니다.
NEXT:
var ss = TimeSpan.FromMilliseconds(720 * (pe - ps));
var yy = "";
if (ss.Days > 0)
{
yy += ss.Days.ToString() + "일 ";
}
if (ss.Days > 0 || ss.Hours > 0)
{
yy += ss.Hours.ToString() + "시간 ";
}
if (ss.Days > 0 || ss.Hours > 0 || ss.Minutes > 0)
{
yy += ss.Minutes.ToString() + "분 ";
}
if (ss.Days > 0 || ss.Hours > 0 || ss.Minutes > 0 || ss.Seconds > 0)
{
yy += ss.Seconds.ToString() + "초 남음";
}
status("진행중...[" + (ps - starts + 1).ToString() + "/" + (pe - starts + 1).ToString("#,#") + "] 남은시간: " + yy);
Logger.Instance.Push("next: " + url + $" || {ps}/{pe}/{starts}");
Thread.Sleep(700);
}
DCGalleryAnalyzer.Instance.Articles.AddRange(articles);
DCGalleryAnalyzer.Instance.Save();
status("완료");
}
catch (Exception e)
{
append("실패: " + e.Message + "\r\n" + e.StackTrace);
}
}