public static async Task <SortedDictionary <string, DCGallery> > GetGalleryListAsync()
{
SortedDictionary <string, DCGallery> dic = new SortedDictionary <string, DCGallery>();
string src = await Common.GetUrlTextAsync(gallery_list_url);
var parse = new List <Match>();
parse.AddRange(Regex.Matches(src, gallery_list1).Cast <Match>().ToList());
parse.AddRange(Regex.Matches(src, gallery_list2).Cast <Match>().ToList());
foreach (var match in parse)
{
DCGallery dcg = new DCGallery();
dcg.identification = match.Groups[1].Value;
dcg.name = match.Groups[2].Value.Trim();
if (!string.IsNullOrEmpty(dcg.name))
{
if (dcg.name[0] == '-')
{
dcg.name = dcg.name.Remove(0, 1).Trim();
}
if (!dic.ContainsKey(dcg.name))
{
dic.Add(dcg.name, dcg);
}
}
}
return(dic);
}
static void ProcessCollectArticles(string[] args, bool rem)
{
var rstarts = Convert.ToInt32(args[1]);
var starts = Convert.ToInt32(args[1]);
var ends = Convert.ToInt32(args[2]);
LoadGalleryList();
bool is_minorg = minor_galleries.ContainsValue(args[0]);
var result = new DCGalleryModel();
var articles = new List <DCPageArticle>();
using (var progressBar = new Console.ConsoleProgressBar())
{
for (; starts <= ends; starts++)
{
X:
try
{
var url = "";
if (is_minorg)
{
url = $"https://gall.dcinside.com/mgallery/board/lists/?id={args[0]}&page={starts}";
}
else
{
url = $"https://gall.dcinside.com/board/lists/?id={args[0]}&page={starts}";
}
if (rem)
{
url += "&exception_mode=recommend";
}
Console.Instance.WriteLine($"Download URL: {url}");
var client = NetCommon.GetDefaultClient();
if (PHPSESSID != "")
{
client.Headers.Add(System.Net.HttpRequestHeader.Cookie, "PHPSESSID=" + PHPSESSID);
}
var html = client.DownloadString(url);
DCGallery gall = null;
if (is_minorg)
{
gall = DCParser.ParseMinorGallery(html);
}
else
{
gall = DCParser.ParseGallery(html);
}
articles.AddRange(gall.articles);
}
catch { goto X; }
progressBar.SetProgress((((ends - rstarts + 1) - (ends - starts)) / (float)(ends - rstarts + 1)) * 100);
}
result.is_minor_gallery = is_minorg;
result.gallery_id = args[0];
result.articles = articles.ToArray();
File.WriteAllText($"list-{args[0]}-{DateTime.Now.Ticks}.txt", JsonConvert.SerializeObject(result));
}
}
static void ProcessFullParse(string[] args, bool rem, string[] title, string[] nick,
string[] ip, string[] id, bool login, bool fix)
{
using (var progressBar = new Console.ConsoleProgressBar())
{
Console.Instance.WriteLine("Parse gallery...");
var rstarts = Convert.ToInt32(args[1]);
var starts = Convert.ToInt32(args[1]);
var ends = Convert.ToInt32(args[2]);
LoadGalleryList();
bool is_minorg = minor_galleries.ContainsValue(args[0]);
var articles = new List <DCPageArticle>();
//
// Parse Gallery
//
for (; starts <= ends; starts++)
{
var url = "";
if (is_minorg)
{
url = $"https://gall.dcinside.com/mgallery/board/lists/?id={args[0]}&page={starts}";
}
else
{
url = $"https://gall.dcinside.com/board/lists/?id={args[0]}&page={starts}";
}
if (rem)
{
url += "&exception_mode=recommend";
}
var client = NetCommon.GetDefaultClient();
if (PHPSESSID != "")
{
client.Headers.Add(System.Net.HttpRequestHeader.Cookie, "PHPSESSID=" + PHPSESSID);
}
var html = client.DownloadString(url);
DCGallery gall = null;
if (is_minorg)
{
gall = DCParser.ParseMinorGallery(html);
}
else
{
gall = DCParser.ParseGallery(html);
}
foreach (var article in gall.articles)
{
if (title != null && !article.title.Contains(title[0]))
{
continue;
}
if (nick != null && !article.nick.Contains(nick[0]))
{
continue;
}
if (ip != null && !article.ip.Contains(ip[0]))
{
continue;
}
if (id != null && !article.uid.Contains(id[0]))
{
continue;
}
if (login && !article.islogined)
{
continue;
}
if (fix && !article.isfixed)
{
continue;
}
//Console.Instance.Write(Monitor.SerializeObject(article));
//Console.Instance.WriteLine(",");
articles.Add(article);
}
progressBar.SetProgress((((ends - rstarts + 1) - (ends - starts)) / (float)(ends - rstarts + 1)) * 100);
Thread.Sleep(3000);
}
Console.Instance.WriteLine("Parse articles...");
int acnt = 0;
var _articles = new List <DCArticle>();
//
// Parse Articles
//
foreach (var article in articles)
{
var url = "";
if (is_minorg)
{
url = $"https://gall.dcinside.com/mgallery/board/view/?id={args[0]}&no={article.no}";
}
else
{
url = $"https://gall.dcinside.com/board/view/?id={args[0]}&no={article.no}";
}
var client = NetCommon.GetDefaultClient();
if (PHPSESSID != "")
{
client.Headers.Add(System.Net.HttpRequestHeader.Cookie, "PHPSESSID=" + PHPSESSID);
}
var html = client.DownloadString(url);
var _article = DCParser.ParseBoardView(html, is_minorg);
_articles.Add(_article);
acnt++;
progressBar.SetProgress(((acnt) / (float)(articles.Count)) * 100);
Thread.Sleep(3000);
}
var info = new List <Tuple <DCArticle, List <DCComment> > >();
if (ESNO == "")
{
ESNO = _articles[0].ESNO;
}
int ccnt = 0;
//
// Parse Comments
//
foreach (var article in _articles)
{
var cc = new List <DCComment>();
var comments = DCCommon.GetComments(new DCArticle {
OriginalGalleryName = article.OriginalGalleryName, Id = article.Id, ESNO = article.ESNO
}, "1");
Thread.Sleep(2000);
cc.Add(comments);
int tcount = comments.total_cnt;
int count = 100;
for (int i = 2; count < tcount; count += 100)
{
comments = DCCommon.GetComments(new DCArticle {
OriginalGalleryName = args[0], Id = args[1], ESNO = ESNO
}, i.ToString());
if (comments.comment_cnt == 0)
{
break;
}
count += comments.comment_cnt;
cc.Add(comments);
Thread.Sleep(2000);
}
info.Add(new Tuple <DCArticle, List <DCComment> >(article, cc));
ccnt++;
progressBar.SetProgress(((ccnt) / (float)(_articles.Count)) * 100);
}
var result = new Tuple <List <DCPageArticle>, List <Tuple <DCArticle, List <DCComment> > > >(articles, info);
File.WriteAllText($"dc-{args[0]}-{args[1]}-{args[2]}-{DateTime.Now.Ticks}.txt", Monitor.SerializeObject(result));
}
}
static void ProcessParseGallery(string[] args, bool rem, string[] title, string[] nick,
string[] ip, string[] id, bool login, bool fix)
{
var starts = Convert.ToInt32(args[1]);
var ends = Convert.ToInt32(args[2]);
LoadGalleryList();
bool is_minorg = minor_galleries.ContainsValue(args[0]);
for (; starts <= ends; starts++)
{
var url = "";
if (is_minorg)
{
url = $"https://gall.dcinside.com/mgallery/board/lists/?id={args[0]}&page={starts}";
}
else
{
url = $"https://gall.dcinside.com/board/lists/?id={args[0]}&page={starts}";
}
if (rem)
{
url += "&exception_mode=recommend";
}
var client = NetCommon.GetDefaultClient();
if (PHPSESSID != "")
{
client.Headers.Add(System.Net.HttpRequestHeader.Cookie, "PHPSESSID=" + PHPSESSID);
}
var html = client.DownloadString(url);
DCGallery gall = null;
if (is_minorg)
{
gall = DCParser.ParseMinorGallery(html);
}
else
{
gall = DCParser.ParseGallery(html);
}
foreach (var article in gall.articles)
{
if (title != null && !article.title.Contains(title[0]))
{
continue;
}
if (nick != null && !article.nick.Contains(nick[0]))
{
continue;
}
if (ip != null && !article.ip.Contains(ip[0]))
{
continue;
}
if (id != null && !article.uid.Contains(id[0]))
{
continue;
}
if (login && !article.islogined)
{
continue;
}
if (fix && !article.isfixed)
{
continue;
}
Console.Instance.Write(Monitor.SerializeObject(article));
Console.Instance.WriteLine(",");
}
}
}
