public void Test5(string url)
{
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
var task = NetTask.MakeDefault(url);
task.Encoding = Encoding.GetEncoding(51949);
var html = NetTools.DownloadString(task);
var c1 = DepartmentExtractor.ExtractStyle5(html, "");
Assert.IsTrue(c1.Count > 0, $"{c1.Count}");
}
static void Main(string[] args)
{
var task = NetTask.MakeDefault("https://github.com/project-violet/database/releases/download/rd2020.06.07/ex-hentai-archive.json");
task.DownloadString = true;
task.StartCallback = () => Console.WriteLine("Download Start!");
task.SizeCallback = (size) => Console.WriteLine("Total File Size: " + size);
task.DownloadCallback = (size) => Console.WriteLine("Receive: " + size);
task.Filename = "test";
var dq = new DownloadQueue.DownloadQueue();
dq.DownloadFile(task);
}
/// <summary>
/// Try login to pixiv
/// </summary>
/// <param name="username"></param>
/// <param name="password"></param>
/// <returns></returns>
public static bool Auth(string username, string password)
{
if (!string.IsNullOrEmpty(AccessToken))
{
return(true);
}
if (string.IsNullOrEmpty(username) || string.IsNullOrEmpty(password))
{
return(false);
}
Logs.Instance.Push("[PixivAPI] Try auth...");
const string client_id = "MOBrBDS8blbauoSck0ZfDbtuzpyT";
const string client_secret = "lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj";
const string hash_secret = "28c1fdd170a5204386cb1313c7077b34f83e4aaf4aa829ce78c231e05b0bae2c";
var local_time = DateTime.Now.ToString("s");
var task = NetTask.MakeDefault("https://oauth.secure.pixiv.net/auth/token");
task.UserAgent = "PixivAndroidApp/5.0.64 (Android 6.0)";
task.Referer = "http://www.pixiv.net/";
task.Headers = new Dictionary <string, string>();
task.Headers.Add("X-Client-Time", local_time);
task.Headers.Add("X-Client-Hash", (local_time + hash_secret).GetHashMD5().ToLower());
task.Query = new Dictionary <string, string>();
task.Query.Add("grant_type", "password");
task.Query.Add("username", username);
task.Query.Add("password", password);
task.Query.Add("get_secure_url", "1");
task.Query.Add("client_id", client_id);
task.Query.Add("client_secret", client_secret);
var token = NetTools.DownloadString(task);
if (token == null)
{
Logs.Instance.PushError("[PixivAPI] Auth fail. Try again.");
return(false);
}
AccessToken = JObject.Parse(token)["response"]["access_token"].ToString();
Logs.Instance.Push("[PixivAPI] Success auth - " + AccessToken);
return(true);
}
public static DCComment GetComments(DCGallery g, DCPageArticle article, string page)
{
var wc = NetTask.MakeDefault("https://gall.dcinside.com/board/comment/");
wc.Headers = new Dictionary <string, string>();
wc.Headers.Add("X-Requested-With", "XMLHttpRequest");
wc.Query = new Dictionary <string, string>();
wc.Query.Add("id", g.id);
wc.Query.Add("no", article.no);
wc.Query.Add("cmt_id", g.id);
wc.Query.Add("cmt_no", article.no);
wc.Query.Add("e_s_n_o", g.esno);
wc.Query.Add("comment_page", page);
return(JsonConvert.DeserializeObject <DCComment>(NetTools.DownloadStringAsync(wc).Result));
}
static void ProcessDownloadHtml(string[] args, bool DisableAutoRedirection)
{
var task = NetTask.MakeDefault(args[0]);
task.AutoRedirection = !DisableAutoRedirection;
var html = NetTools.DownloadStringAsync(task).Result;
if (html != null)
{
System.Console.WriteLine(html);
}
else
{
System.Console.WriteLine("Internal error! Please, check log.txt file.");
}
}
public override void RunOnPass(ref NetTask content)
{
if (FreeProxy.Instance.Proxy == null || FreeProxy.Instance.Proxy.Count == 0)
{
Logs.Instance.Push("[Free Proxy Pass] Free Proxy is not initialized!");
//
// Raise Critical Error
//
throw new Exception("[Free Proxy Pass] Free Proxy is not initialized!");
}
var proxy = FreeProxy.Instance.Proxy[new Random().Next(FreeProxy.Instance.Proxy.Count)];
content.Proxy = new WebProxy(proxy.IP, proxy.Port);
}
public static async Task <DCInsideComment> GetComments(string gall_id, string article_id, string page)
{
var nt = NetTask.MakeDefault("https://gall.dcinside.com/board/comment/");
nt.Headers = new Dictionary <string, string>()
{
{ "X-Requested-With", "XMLHttpRequest" }
};
nt.Query = new Dictionary <string, string>()
{
{ "id", gall_id },
{ "no", article_id },
{ "cmt_id", gall_id },
{ "cmt_no", article_id },
{ "e_s_n_o", DCInsideManager.ESNO },
{ "comment_page", page }
};
return(JsonConvert.DeserializeObject <DCInsideComment>(await NetTools.DownloadStringAsync(nt)));
}
/// <summary>
/// Get ugoira
/// </summary>
/// <param name="authorId"></param>
/// <returns></returns>
public static async Task <Ugoira> GetUgoiraAsync(string illust_id)
{
var url = "https://app-api.pixiv.net/v1/ugoira/metadata";
var param = new Dictionary <string, string>
{
{ "illust_id", illust_id }
};
var task = NetTask.MakeDefault(url + "?" + string.Join("&", param.ToList().Select(x => $"{x.Key}={x.Value}")));
task.Referer = "http://spapi.pixiv.net/";
task.UserAgent = "PixivIOSApp/5.8.0";
task.Headers = new Dictionary <string, string>();
task.Headers.Add("Authorization", "Bearer " + AccessToken);
var result = await NetTools.DownloadStringAsync(task);
return(JToken.Parse(result)["ugoira_metadata"].ToObject <Ugoira>());
}
static void download_data(string url, string filename)
{
Logs.Instance.Push($"Download {filename}...");
var task = NetTask.MakeDefault(url);
SingleFileProgressBar pb = null;
long tsz = 0;
task.SizeCallback = (sz) =>
{
Console.Write("Downloading ... ");
pb = new SingleFileProgressBar();
pb.Report(sz, 0);
tsz = sz;
};
task.Filename = filename;
task.DownloadCallback = (sz) => pb.Report(tsz, sz);
NetTools.DownloadFile(task);
pb.Dispose();
Console.WriteLine("Complete!");
}
public static async Task <List <UsersWork> > GetWorksAsync(long illustId)
{
var url = "https://public-api.secure.pixiv.net/v1/works/" + illustId.ToString() + ".json";
var param = new Dictionary <string, string>
{
{ "profile_image_sizes", "px_170x170,px_50x50" },
{ "image_sizes", "px_128x128,small,medium,large,px_480mw" },
{ "include_stats", "true" },
};
var task = NetTask.MakeDefault(url + "?" + string.Join("&", param.ToList().Select(x => $"{x.Key}={x.Value}")));
task.Referer = "http://spapi.pixiv.net/";
task.UserAgent = "PixivIOSApp/5.8.0";
task.Headers = new Dictionary <string, string>();
task.Headers.Add("Authorization", "Bearer " + AccessToken);
var result = await NetTools.DownloadStringAsync(task);
return(JToken.Parse(result).SelectToken("response").ToObject <List <UsersWork> >());
}
public override (List <NetTask>, ExtractedInfo) Extract(string url, IExtractorOption option = null)
{
if (option == null)
{
option = RecommendOption(url);
}
var html = NetTools.DownloadString(url);
var match = ValidUrl.Match(url).Groups;
var node = html.ToHtmlNode();
var title = node.SelectSingleNode("/html[1]/body[1]/div[1]/div[3]/div[1]/h1[1]").InnerText.Trim();
var genre = node.SelectSingleNode("/html[1]/body[1]/div[1]/div[3]/div[2]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/ul[1]/li[1]/div[1]/div[2]/div[2]/h3[1]/a[1]").InnerText.Trim();
var artist = node.SelectSingleNode("/html[1]/body[1]/div[1]/div[3]/div[2]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/ul[1]/li[1]/div[1]/div[2]/div[3]/h3[1]/a[1]").InnerText.Trim();
var sub_urls = new List <string>();
var sub_titles = new List <string>();
foreach (var episode in node.SelectNodes("/html[1]/body[1]/div[1]/div[3]/div[2]/div[1]/div[1]/div[2]/div[1]/div[1]/div"))
{
var tag_a = episode.SelectSingleNode("./div[2]/h2[1]/a[1]");
sub_urls.Add(tag_a.GetAttributeValue("href", ""));
sub_titles.Add(tag_a.InnerText.Trim());
}
option.SimpleInfoCallback?.Invoke(title);
option.ThumbnailCallback?.Invoke(NetTask.MakeDefault(
match["host"].Value + node.SelectSingleNode("/html[1]/body[1]/div[1]/div[3]/div[2]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/ul[1]/li[1]/div[1]/div[1]/a[1]/img[1]").GetAttributeValue("src", "")));
option.ProgressMax?.Invoke(sub_urls.Count);
var sub_htmls = NetTools.DownloadStrings(sub_urls, "", () =>
{
option.PostStatus?.Invoke(1);
});
var result = new List <NetTask>();
for (int i = 0; i < sub_urls.Count; i++)
{
var snode = sub_htmls[i].ToHtmlNode();
int count = 1;
foreach (var img in snode.SelectNodes("/html[1]/body[1]/div[1]/div[3]/div[2]/div[1]/div[2]/ul[1]//li/div[1]/img[1]"))
{
var img_src = img.GetAttributeValue("data-src", "");
if (string.IsNullOrWhiteSpace(img_src))
{
img_src = img.GetAttributeValue("src", "");
}
var task = NetTask.MakeDefault(HttpUtility.HtmlDecode(img_src));
task.SaveFile = true;
task.Filename = count.ToString("000") + ".jpg";
task.Format = new ExtractorFileNameFormat
{
Title = title,
Episode = sub_titles[i],
FilenameWithoutExtension = count.ToString("000"),
Extension = Path.GetExtension(task.Filename).Replace(".", ""),
};
result.Add(task);
count++;
}
}
result.ForEach(task => task.Format.Extractor = GetType().Name.Replace("Extractor", ""));
return(result, new ExtractedInfo {
Type = ExtractedInfo.ExtractedType.WorksComic
});
}
public override (List <NetTask>, ExtractedInfo) Extract(string url, IExtractorOption option = null)
{
if (option == null)
{
option = RecommendOption(url);
}
var limit = int.MaxValue;
if ((option as InstagramExtractorOption).LimitPosts != null)
{
limit = (option as InstagramExtractorOption).LimitPosts[0].ToInt();
}
var html = NetTools.DownloadString(url);
var user = InstaApi.get_user(option as InstagramExtractorOption, html);
var urls = new List <string>();
urls.AddRange(user.FirstPost.DisplayUrls);
option.PostStatus?.Invoke(user.FirstPost.PostCount);
option.SimpleInfoCallback?.Invoke($"{user.FullName} ({user.UserName})");
var count = 0;
var pp = user.FirstPost;
while (pp.HasNext)
{
if (count >= limit)
{
break;
}
var posts = InstaApi.query_next(option as InstagramExtractorOption, InstaApi.posts_query_hash(), user.UserId, "50", pp.EndCursor);
urls.AddRange(posts.DisplayUrls);
option.PostStatus?.Invoke(posts.PostCount);
count += 50;
pp = posts;
}
var result = new List <NetTask>();
foreach (var surl in urls)
{
var task = NetTask.MakeDefault(surl);
task.SaveFile = true;
var fn = surl.Split('?')[0].Split('/').Last();
task.Filename = fn;
task.Format = new ExtractorFileNameFormat
{
FilenameWithoutExtension = Path.GetFileNameWithoutExtension(fn),
Extension = Path.GetExtension(fn).Replace(".", ""),
User = user.FullName,
Account = user.UserName
};
result.Add(task);
}
result.ForEach(task => task.Format.Extractor = GetType().Name.Replace("Extractor", ""));
return(result, new ExtractedInfo {
Type = ExtractedInfo.ExtractedType.UserArtist
});
}
private void ugoira2webp(NetTask task)
{
}
public override (List <NetTask>, ExtractedInfo) Extract(string url, IExtractorOption option = null)
{
var match = ValidUrl.Match(url).Groups;
if (option == null)
{
option = new GelbooruExtractorOption {
Type = ExtractorType.Images
}
}
;
var tags = match[1].Value;
var result = new List <NetTask>();
var page = 0;
if ((option as GelbooruExtractorOption).StartPage != null)
{
page = (option as GelbooruExtractorOption).StartPage[0].ToInt();
}
var end_page = int.MaxValue;
if ((option as GelbooruExtractorOption).EndPage != null)
{
end_page = (option as GelbooruExtractorOption).EndPage[0].ToInt();
}
option.SimpleInfoCallback?.Invoke($"{HttpUtility.UrlDecode(tags)}");
var post_thumbnail = false;
while (true)
{
var durl = "https://gelbooru.com/index.php?page=dapi&s=post&q=index&limit=100&tags=" + tags + "&pid=" + page.ToString();
option.PageReadCallback?.Invoke(durl);
var data = NetTools.DownloadString(durl);
var document = new HtmlDocument();
document.LoadHtml(data);
var nodes = document.DocumentNode.SelectNodes("/posts[1]/post");
if (nodes == null || nodes.Count == 0)
{
break;
}
foreach (var node in nodes)
{
var imgurl = node.GetAttributeValue("file_url", "");
var task = NetTask.MakeDefault(imgurl);
task.SaveFile = true;
task.Filename = imgurl.Split('/').Last();
task.Format = new ExtractorFileNameFormat
{
Search = HttpUtility.UrlDecode(tags),
FilenameWithoutExtension = Path.GetFileNameWithoutExtension(imgurl.Split('/').Last()),
Extension = Path.GetExtension(imgurl.Split('/').Last()).Replace(".", "")
};
result.Add(task);
}
if (!post_thumbnail)
{
option.ThumbnailCallback?.Invoke(result[0]);
post_thumbnail = true;
}
option.PostStatus?.Invoke(nodes.Count);
page += 1;
if (page > end_page)
{
break;
}
}
result.ForEach(task => task.Format.Extractor = GetType().Name.Replace("Extractor", ""));
return(result, new ExtractedInfo {
Type = ExtractedInfo.ExtractedType.Search
});
}
}
public override (List <NetTask>, ExtractedInfo) Extract(string url, IExtractorOption option = null)
{
if (option == null)
{
option = RecommendOption(url);
}
var match = ValidUrl.Match(url).Groups;
var host = "https://" + match["host"];
var html = NetTools.DownloadString(url);
var node = html.ToHtmlNode();
var title = node.SelectSingleNode("/html[1]/body[1]/div[2]/div[1]/div[1]/div[3]/div[1]/table[2]/tbody[1]/tr[2]/td[1]/table[1]/tr[1]/td[1]").InnerText;
var sub_datas = node.SelectNodes("/html[1]/body[1]/div[2]/div[1]/div[1]/div[3]/div[1]/div[1]/form[1]/table[1]//tr/td[2]");
option.SimpleInfoCallback?.Invoke($"{title}");
var sub_urls = new List <string>();
var sub_titles = new List <string>();
foreach (var sub_data in sub_datas)
{
sub_urls.Add(host + sub_data.GetAttributeValue("data-role", ""));
sub_titles.Add(sub_data.InnerText.Trim());
}
var htmls = NetTools.DownloadStrings(sub_urls);
var result = new List <NetTask>();
for (int i = 0; i < htmls.Count; i++)
{
var base64encoded = Regex.Match(htmls[i], "var toon_img = '(.*)'").Groups[1].Value;
string rhtml;
Strings.TryParseBase64(base64encoded, out rhtml);
var snode = rhtml.ToHtmlNode();
int count = 1;
foreach (var img in snode.SelectNodes("/img"))
{
var task = NetTask.MakeDefault(host + img.GetAttributeValue("src", ""));
task.SaveFile = true;
task.Filename = task.Url.Split('/').Last();
task.Format = new ExtractorFileNameFormat
{
Title = title,
Episode = sub_titles[i],
FilenameWithoutExtension = count.ToString("000"),
Extension = Path.GetExtension(task.Filename).Replace(".", "")
};
result.Add(task);
count++;
}
}
result.ForEach(task => task.Format.Extractor = GetType().Name.Replace("Extractor", ""));
return(result, new ExtractedInfo {
Type = ExtractedInfo.ExtractedType.WorksComic
});
}
public override (List <NetTask>, ExtractedInfo) Extract(string url, IExtractorOption option = null)
{
if (option == null)
{
option = RecommendOption(url);
}
var html = NetTools.DownloadString(url);
var match = ValidUrl.Match(url).Groups;
var document = new HtmlDocument();
document.LoadHtml(html);
var node = document.DocumentNode;
if (option.Type == ExtractorType.EpisodeImages)
{
var images = get_board_images(html);
var title = node.SelectSingleNode("/html[1]/head[1]/title[1]").InnerText;
var result = new List <NetTask>();
int count = 1;
foreach (var img in images)
{
var task = NetTask.MakeDefault(img);
task.SaveFile = true;
task.Filename = count.ToString("000") + Path.GetExtension(img.Split('/').Last());
task.Format = new ExtractorFileNameFormat
{
Episode = title,
FilenameWithoutExtension = count.ToString("000"),
Extension = Path.GetExtension(task.Filename).Replace(".", "")
};
result.Add(task);
count++;
}
result.ForEach(task => task.Format.Extractor = GetType().Name.Replace("Extractor", ""));
return(result, null);
}
else if (option.Type == ExtractorType.Works)
{
var title = node.SelectSingleNode("/html[1]/body[1]/div[1]/div[3]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[2]/div[1]/div[1]").InnerText;
var sub_urls = new List <string>();
var sub_titles = new List <string>();
option.SimpleInfoCallback?.Invoke($"{title}");
option.ThumbnailCallback?.Invoke(NetTask.MakeDefault(
Regex.Match(node.SelectSingleNode("/html[1]/body[1]/div[1]/div[3]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]").GetAttributeValue("style", ""), @"(https?://.*?)\)").Groups[1].Value));
foreach (var item in node.SelectNodes("/html[1]/body[1]/div[1]/div[3]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[2]/div[2]/div[1]/div[1]/div"))
{
sub_urls.Add(match["host"] + item.SelectSingleNode("./a[1]").GetAttributeValue("href", ""));
sub_titles.Add(item.SelectSingleNode("./a[1]/div[1]").MyText());
}
option.ProgressMax?.Invoke(sub_urls.Count);
var htmls = NetTools.DownloadStrings(sub_urls, "PHPSESSID=" + Externals.ManamoaPHPSESSID, () =>
{
option.PostStatus?.Invoke(1);
});
var result = new List <NetTask>();
for (int i = 0; i < sub_urls.Count; i++)
{
try
{
var images = get_board_images(htmls[i]);
int count = 1;
foreach (var img in images)
{
var task = NetTask.MakeDefault(img);
task.SaveFile = true;
task.Filename = count.ToString("000") + Path.GetExtension(img.Split('/').Last());
task.Format = new ExtractorFileNameFormat
{
Title = title,
Episode = sub_titles[i],
FilenameWithoutExtension = count.ToString("000"),
Extension = Path.GetExtension(task.Filename).Replace(".", "")
};
result.Add(task);
count++;
}
}
catch (Exception e)
{
;
}
}
result.ForEach(task => task.Format.Extractor = GetType().Name.Replace("Extractor", ""));
return(result, new ExtractedInfo {
Type = ExtractedInfo.ExtractedType.WorksComic
});
}
return(null, null);
}
public override (List <NetTask>, ExtractedInfo) Extract(string url, IExtractorOption option = null)
{
var match = ValidUrl.Match(url).Groups;
if (option == null)
{
option = RecommendOption(url);
}
var mtask = NetTask.MakeDefault(url);
mtask.Encoding = Encoding.GetEncoding(51949);
var html = NetTools.DownloadString(mtask);
var node = html.ToHtmlNode();
var result = new List <NetTask>();
var xcode = match["xcode"].Value;
if (match["menu"].Value == "shopbrand" || match["menu"].Value == "bestseller")
{
var filtering_filename = new string[]
{
"HN_Copyright2.jpg",
"next_product.gif",
"prev_product.gif",
"btn_h8_spin_dw.gif",
"btn_h8_spin_up.gif",
"Review.jpg",
"shoppingguide2.jpg",
"sizetip-2.jpg"
};
var gallery = node.SelectSingleNode("/html[1]/head[1]/title[1]").InnerText.Trim();
option.SimpleInfoCallback?.Invoke(gallery);
var last_page_node = node.SelectSingleNode("/html[1]/body[1]/div[3]/div[3]/div[1]/div[2]/div[3]/div[1]/div[5]/ol[1]/li[@class='last']/a");
var last_page = 1;
if (last_page_node != null)
{
last_page = node.SelectSingleNode("/html[1]/body[1]/div[3]/div[3]/div[1]/div[2]/div[3]/div[1]/div[5]/ol[1]/li[@class='last']/a").GetAttributeValue("href", "").Split('=').Last().ToInt();
}
var page_urls = Enumerable.Range(1, last_page).Select(page => $"{url}&page={page}").ToList();
var htmls = NetTools.DownloadStrings(page_urls);
var sub_urls = new List <string>();
foreach (var shtml in htmls)
{
var snode = shtml.ToHtmlNode();
sub_urls.AddRange(snode.SelectNodes("/html[1]/body[1]/div[3]/div[3]/div[1]/div[2]/div[3]/div[1]/div[5]/table[1]/tbody[1]//a").Select(x => "http://www.hn-hn.co.kr" + x.GetAttributeValue("href", "")));
}
option.ProgressMax?.Invoke(sub_urls.Count);
var sub_htmls = new List <string>();
foreach (var surl in sub_urls)
{
var task = NetTask.MakeDefault(surl);
task.Encoding = Encoding.GetEncoding(51949);
sub_htmls.Add(NetTools.DownloadString(task));
option.PostStatus?.Invoke(1);
}
foreach (var shtml in sub_htmls)
{
var snode = shtml.ToHtmlNode();
var title = snode.SelectSingleNode("/html[1]/body[1]/div[3]/div[3]/div[1]/div[2]/div[1]/div[2]/div[1]/form[1]/div[1]/div[1]/h3[1]").InnerText.Trim();
var thumbnail = "http://www.hn-hn.co.kr" + snode.SelectSingleNode("/html[1]/body[1]/div[3]/div[3]/div[1]/div[2]/div[1]/div[2]/div[1]/div[3]/div[1]/a[1]/img[1]").GetAttributeValue("src", "").Split('?')[0];
var imgs = snode.SelectNodes("/html[1]/body[1]/div[3]/div[3]/div[1]/div[2]/div[1]/div[2]//img").Select(img =>
{
if (img.GetAttributeValue("src", "").StartsWith("http"))
{
return(img.GetAttributeValue("src", ""));
}
else
{
return("http://www.hn-hn.co.kr" + img.GetAttributeValue("src", "").Split('?')[0]);
}
}).ToList();
foreach (var img in imgs)
{
var task = NetTask.MakeDefault(img);
task.SaveFile = true;
task.Filename = img.Split('/').Last();
if (filtering_filename.Contains(task.Filename))
{
continue;
}
task.Format = new ExtractorFileNameFormat
{
Gallery = gallery,
Title = title,
FilenameWithoutExtension = Path.GetFileNameWithoutExtension(task.Filename),
Extension = Path.GetExtension(task.Filename).Replace(".", "")
};
result.Add(task);
}
}
option.ThumbnailCallback?.Invoke(result[0]);
}
result.ForEach(task => task.Format.Extractor = GetType().Name.Replace("Extractor", ""));
return(result, new ExtractedInfo {
Type = ExtractedInfo.ExtractedType.Search
});
}
public override (List <NetTask>, ExtractedInfo) Extract(string url, IExtractorOption option = null)
{
if (option == null)
{
option = RecommendOption(url);
}
var match = ValidUrl.Match(url).Groups;
var limit = int.MaxValue;
if ((option as TwitterExtractorOption).LimitPosts != null)
{
limit = (option as TwitterExtractorOption).LimitPosts[0].ToInt();
}
if (match["id"].Value == "hashtag")
{
#if DEBUG && false
var html = NetTools.DownloadString(url);
var search = HttpUtility.UrlDecode(match["search"].Value);
var position = Regex.Match(html, @"data-max-position""(.*?)""").Groups[1].Value;
var document = new HtmlDocument();
document.LoadHtml(html);
var node = document.DocumentNode.SelectSingleNode("/html[1]/body[1]/div[1]/div[2]/div[1]/div[2]/div[1]/div[1]/div[2]/div[1]/div[1]/div[1]/div[1]/div[2]/ol[1]");
var tweets = node.SelectNodes("./li[@data-item-type='tweet']");
var urls = new List <string>();
foreach (var tweet in tweets)
{
urls.AddRange(parse_tweet_hashtag(option as TwitterExtractorOption, tweet));
}
while (true)
{
try
{
var next = seach_query(option as TwitterExtractorOption, search, position);
position = JToken.Parse(next)["min_position"].ToString();
var html2 = JToken.Parse(next)["items_html"].ToString();
var document2 = new HtmlDocument();
document2.LoadHtml(html2);
var tweets2 = node.SelectNodes("./li[@data-item-type='tweet']");
foreach (var tweet in tweets2)
{
urls.AddRange(parse_tweet_hashtag(option as TwitterExtractorOption, tweet));
}
}
catch
{
break;
}
}
var result = new List <NetTask>();
foreach (var surl in urls)
{
var task = NetTask.MakeDefault(surl);
task.SaveFile = true;
var fn = surl.Split('/').Last();
task.Filename = fn;
task.Format = new ExtractorFileNameFormat
{
FilenameWithoutExtension = Path.GetFileNameWithoutExtension(fn),
Extension = Path.GetExtension(fn).Replace(".", ""),
User = search
};
result.Add(task);
}
return(new Tuple <List <NetTask>, object>(result, null));
#endif
throw new ExtractorException("'hashtag' is not support yet!");
}
else
{
var name = match["id"].Value;
var html = NetTools.DownloadString($"https://twitter.com/{name}/media");
var min_position = Regex.Match(html, @"data-min-position=""(.*?)""").Groups[1].Value;
var node = html.ToHtmlNode();
var tweets = node.SelectNodes("./html[1]/body[1]/div[1]/div[2]/div[1]/div[2]/div[1]/div[1]/div[2]/div[1]/div[2]/div[2]/div[1]/div[2]/ol[1]/li[@data-item-type='tweet']");
var urls = new List <string>();
var user = node.SelectSingleNode("/html[1]/body[1]/div[1]/div[2]/div[1]/div[2]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/h1[1]/a[1]").InnerText;
var videos = new List <(string, List <string>)>();
var post_count = tweets.Count;
var last_url_count = 0;
option.SimpleInfoCallback?.Invoke($"{user} ({name})");
foreach (var tweet in tweets)
{
urls.AddRange(parse_tweet_hashtag(option as TwitterExtractorOption, tweet, videos));
}
while (post_count < limit)
{
var next = profile_query(option as TwitterExtractorOption, name, min_position);
var html2 = JToken.Parse(next)["items_html"].ToString();
var tweets2 = html2.ToHtmlNode().SelectNodes("./li[@data-item-type='tweet']");
if (tweets2 == null)
{
break;
}
foreach (var tweet in tweets2)
{
urls.AddRange(parse_tweet_hashtag(option as TwitterExtractorOption, tweet, videos));
}
option.PostStatus?.Invoke(urls.Count - last_url_count);
last_url_count = urls.Count;
post_count += tweets2.Count;
min_position = JToken.Parse(next)["min_position"].ToString();
if (!(bool)JToken.Parse(next)["has_more_items"])
{
break;
}
Thread.Sleep(3000);
}
var result = new List <NetTask>();
foreach (var surl in urls)
{
var task = NetTask.MakeDefault(surl);
task.SaveFile = true;
var fn = surl.Split('/').Last();
task.Filename = fn;
task.Format = new ExtractorFileNameFormat
{
FilenameWithoutExtension = Path.GetFileNameWithoutExtension(fn),
Extension = Path.GetExtension(fn).Replace(".", ""),
Account = name,
User = user,
};
result.Add(task);
}
foreach (var video in videos)
{
var count = 0;
foreach (var ts in video.Item2)
{
var task = NetTask.MakeDefault(ts);
task.SaveFile = true;
var fn = ts.Split('/').Last();
task.Filename = fn;
task.Format = new ExtractorFileNameFormat
{
FilenameWithoutExtension = video.Item1 + "/" + count++.ToString("000"),
Extension = Path.GetExtension(fn).Replace(".", ""),
Account = name,
User = user,
};
result.Add(task);
}
}
result.ForEach(task => task.Format.Extractor = GetType().Name.Replace("Extractor", ""));
return(result, new ExtractedInfo {
Type = ExtractedInfo.ExtractedType.UserArtist
});
}
}
public abstract void Run(NetTask task);
public override (List <NetTask>, ExtractedInfo) Extract(string url, IExtractorOption option = null)
{
var html = NetTools.DownloadString(NetTask.MakeDefault(url, cookie: cookies[0]));
var data = EHentaiExtractor.ParseArticleData(html, @"https://exhentai.org/.*?(?=\))");
var pages = EHentaiExtractor.GetPagesUri(html);
var image_urls = new List <string>();
option.SimpleInfoCallback?.Invoke($"{data.Title}");
if (option == null)
{
option = RecommendOption(url);
}
if (option.ExtractInformation)
{
return(null, null /*data*/);
}
//
// Extract Image Url-Url
//
image_urls.AddRange(EHentaiExtractor.GetImagesUri(html));
for (int i = 1; i < pages.Length; i++)
{
(option as EHentaiExtractorOption).PageReadCallback?.Invoke(pages[i]);
var page = NetTools.DownloadString(NetTask.MakeDefault(pages[i], cookie: cookies[0]));
image_urls.AddRange(EHentaiExtractor.GetImagesUri(page));
}
//
// Extract Image Url
//
var result = new NetTask[image_urls.Count];
var artist = "N/A";
var group = "N/A";
var series = "N/A";
if (data.artist != null && data.artist.Length > 0)
{
artist = data.artist[0];
}
if (data.group != null && data.group.Length > 0)
{
group = data.group[0];
}
if (data.parody != null && data.parody.Length > 0)
{
series = data.parody[0];
}
if (artist == "N/A" && group != "N/A")
{
artist = group;
}
for (int i = 0; i < image_urls.Count; i++)
{
var html2 = NetTools.DownloadString(NetTask.MakeDefault(image_urls[i], cookies[0]));
var durl = EHentaiExtractor.GetImagesAddress(html2);
var task = NetTask.MakeDefault(durl, cookies[0]);
task.SaveFile = true;
task.Filename = durl.Split('/').Last();
task.Format = new ExtractorFileNameFormat
{
Title = data.Title,
FilenameWithoutExtension = Path.GetFileNameWithoutExtension(task.Filename),
Extension = Path.GetExtension(task.Filename).Replace(".", ""),
OriginalTitle = data.SubTitle,
Artist = artist,
Group = group,
Series = series
};
result[i] = task;
if (i == 0)
{
option.ThumbnailCallback?.Invoke(task);
}
}
var result_list = result.ToList();
result_list.ForEach(task => task.Format.Extractor = GetType().Name.Replace("Extractor", ""));
return(result_list, new ExtractedInfo {
Type = ExtractedInfo.ExtractedType.WorksComic
});
}
public override (List <NetTask>, ExtractedInfo) Extract(string url, IExtractorOption option = null)
{
var match = ValidUrl.Match(url).Groups;
if (option == null)
{
option = RecommendOption(url);
}
var html = NetTools.DownloadString(url);
var node = html.ToHtmlNode();
var result = new List <NetTask>();
var id = match["id"].Value;
if (match["type"].Value == "list")
{
var title = node.SelectSingleNode("/html[1]/head[1]/title[1]").InnerText.Replace("아뜨랑스 - ", "").Trim();
var sub_urls = get_first_urls(html);
var page = 1;
var prev_count = 0;
option.SimpleInfoCallback?.Invoke(title);
do
{
prev_count = sub_urls.Count;
var task = NetTask.MakeDefault("https://attrangs.co.kr/shop/list_ajax.php");
task.Query = make_next_list_dict(id, page++);
sub_urls.UnionWith(get_first_urls(NetTools.DownloadString(task)));
} while (prev_count != sub_urls.Count);
option.ProgressMax?.Invoke(sub_urls.Count);
var sub_htmls = new List <string>();
var rand = new Random();
foreach (var surl in sub_urls)
{
sub_htmls.Add(NetTools.DownloadString(surl));
option.PostStatus?.Invoke(1);
// Kuipernet Handling
Thread.Sleep(rand.Next(3, 7) * 100);
}
foreach (var shtml in sub_htmls)
{
var view = new AttrangsViewParser(shtml);
foreach (var img in view.Images)
{
var task = NetTask.MakeDefault(img);
task.SaveFile = true;
task.Filename = img.Split('/').Last();
task.Format = new ExtractorFileNameFormat
{
Gallery = title,
Title = view.Title,
FilenameWithoutExtension = Path.GetFileNameWithoutExtension(task.Filename),
Extension = Path.GetExtension(task.Filename).Replace(".", "")
};
result.Add(task);
}
}
option.ThumbnailCallback?.Invoke(result[0]);
}
result.ForEach(task => task.Format.Extractor = GetType().Name.Replace("Extractor", ""));
return(result, new ExtractedInfo {
Type = ExtractedInfo.ExtractedType.Search
});
}
public void ClickVipBtn()
{
NetTask netTask = new NetTask();
GameRoot.AddTips("测试用按钮");
}
public override (List <NetTask>, ExtractedInfo) Extract(string url, IExtractorOption option = null)
{
if (option == null)
{
option = new DCInsideExtractorOption {
Type = ExtractorType.Images
}
}
;
if ((option as DCInsideExtractorOption).OnlyRecommend)
{
url += "&exception_mode=recommend";
}
var match = ValidUrl.Match(url).Groups;
var result = new List <NetTask>();
var html = NetTools.DownloadString(url);
if (html == null)
{
return(result, null);
}
if (match[1].Value == "gall")
{
try
{
//
// Parse article
//
if (match[3].Value == "view")
{
var article = ParseBoardView(html, match[2].Value != "");
if (option.Type == ExtractorType.Images && option.ExtractInformation == false)
{
if (article.ImagesLink == null || article.ImagesLink.Count == 0)
{
throw new Exception("Nothing to download!");
}
option.SimpleInfoCallback?.Invoke($"{article.Title}");
for (int i = 0; i < article.ImagesLink.Count; i++)
{
var task = NetTask.MakeDefault(article.ImagesLink[i]);
task.Filename = article.FilesName[i];
task.SaveFile = true;
task.Referer = url;
task.Format = new ExtractorFileNameFormat
{
Id = article.Id,
Gallery = article.GalleryName,
Title = article.Title,
FilenameWithoutExtension = (i + 1).ToString("000"),
Extension = Path.GetExtension(article.FilesName[i]).Replace(".", ""),
};
result.Add(task);
}
result.ForEach(task => task.Format.Extractor = GetType().Name.Replace("Extractor", ""));
return(result, null /*article*/);
}
else if (option.Type == ExtractorType.ArticleInformation || option.ExtractInformation == true)
{
return(null, null /*article*/);
}
else if (option.Type == ExtractorType.Comments)
{
var cc = new List <DCComment>();
var comments = GetComments(article, "1");
cc.Add(comments);
//
// To avoid server blocks
//
Thread.Sleep(2000);
int tcount = comments.total_cnt;
int count = 100;
for (int i = 2; count < tcount; count += 100)
{
comments = GetComments(article, i.ToString());
if (comments.comment_cnt == 0)
{
break;
}
count += comments.comment_cnt;
cc.Add(comments);
Thread.Sleep(2000);
}
return(null, null /*GetComments(article, "0")*/);
}
else
{
throw new Exception("You cannot do that with this URL. " + url);
}
}
//
// Parse Articles List
//
else if (match[3].Value == "lists")
{
DCGallery gallery;
if (match[2].Value == "")
{
gallery = ParseGallery(html);
}
else
{
gallery = ParseMinorGallery(html);
}
if (option.Type == ExtractorType.GalleryInformation || option.ExtractInformation == true)
{
return(null, null /*gallery*/);
}
else
{
throw new Exception("You cannot do that with this URL." + url);
}
}
}
catch (Exception e)
{
Log.Logs.Instance.PushError("[DCInsideExtractor] Extract error - " + option.Type.ToString() + " - " + e.Message + "\r\n" + e.StackTrace);
}
}
else
{
// Not support mobile page.
throw new ExtractorException("[DCInside Extractor] Not support mobile page yet.");
}
result.ForEach(task => task.Format.Extractor = GetType().Name.Replace("Extractor", ""));
return(result, new ExtractedInfo {
Type = ExtractedInfo.ExtractedType.Community
});
}
public static async Task LoopInternal()
{
// Inha Univ Article
{
var range = Enumerable.Range(Convert.ToInt32(ExtractManager.InhaUnivArticles.Last().Link.Split('/')[6]) + 1, 5).ToList();
var htmls = NetTools.DownloadStrings(range.Select(x => $"https://www.inha.ac.kr/bbs/kr/8/{x}/artclView.do").ToList());
for (int i = 0; i < htmls.Count; i++)
{
try
{
var cc = InhaUnivExtractor.Parse(htmls[i]);
cc.Link = $"https://www.inha.ac.kr/bbs/kr/8/{range[i]}/artclView.do";
ExtractManager.InhaUnivArticles.Add(cc);
ExtractManager.InhaUnivDB.Add(cc);
Log.Logs.Instance.Push($"[Loop] New item is added. - IUA - {cc.Title}");
if (EnableServer)
{
await BotManager.Instance.Notice(cc.ToString(), "MSG-MAIN");
}
}
catch { }
}
}
// Department Notices
{
// Lazy downloading
foreach (var department in DepartmentList.Lists)
{
try
{
if (department.Item3 == "")
{
continue;
}
var task = NetTask.MakeDefault(department.Item3);
if (department.Item2 == "s5")
{
task.Encoding = Encoding.GetEncoding(51949);
}
var html = NetTools.DownloadString(task);
List <DepartmentDBModel> cc = null;
if (department.Item2 == "s1")
{
cc = DepartmentExtractor.ExtractStyle1(html, department.Item1);
}
else if (department.Item2 == "s2")
{
cc = DepartmentExtractor.ExtractStyle2(html, department.Item1);
}
else if (department.Item2 == "s3")
{
cc = DepartmentExtractor.ExtractStyle3(html, department.Item1);
}
else if (department.Item2 == "s4")
{
cc = DepartmentExtractor.ExtractStyle4(html, department.Item1);
}
else if (department.Item2 == "s5")
{
cc = DepartmentExtractor.ExtractStyle5(html, department.Item1);
}
// get cse latest
var mm = new HashSet <int>();
ExtractManager.DepartmentArticles.Where(x => x.Department == department.Item1).ToList().ForEach(x => mm.Add(Convert.ToInt32(x.Number)));
int starts = 0;
for (starts = cc.Count - 1; starts >= 0; starts--)
{
if (mm.Contains(Convert.ToInt32(cc[starts].Number)))
{
break;
}
}
for (int i = starts + 1; i < cc.Count; i++)
{
ExtractManager.DepartmentArticles.Add(cc[i]);
ExtractManager.DepartmentDB.Add(cc[i]);
Log.Logs.Instance.Push($"[Loop] New item is added. - DN - {cc[i].Title}");
if (EnableServer)
{
await BotManager.Instance.Notice(cc[i].ToString(), "MSG-" + department.Item1);
}
}
}
catch (Exception e)
{
Log.Logs.Instance.PushError("[Loop] '" + department.Item1 + "' " + e.Message + "\r\n" + e.StackTrace);
}
}
}
Log.Logs.Instance.Push("[Loop] Cycle " + Count.ToString());
Count++;
}
public override void Run(NetTask task)
{
ugoira2gif(task);
}
public DownloadElement(string url)
{
InitializeComponent();
DownloadInfo.DownloadStarts = DateTime.Now;
Info.Text = url;
int hitomi_id = 0;
if (int.TryParse(url, out hitomi_id))
{
url = "https://hitomi.la/galleries/" + url + ".html";
}
var dbm = new DownloadDBModel();
dbm.Url = url;
dbm.StartsTime = DownloadInfo.DownloadStarts;
dbm.State = DownloadDBState.Downloading;
DownloadDBManager.Instance.Add(dbm);
Commands.SetTap(Body, new Command(async() =>
{
await(Application.Current.MainPage as MainPage).NaviInstance.PushAsync(new DownloadInfoPage(dbm));
}));
if (!url.StartsWith("http://") && !url.StartsWith("https://"))
{
Status.Text = "옳바른 URL이 아닙니다!";
Status.TextColor = Color.Red;
Spinner.IsVisible = false;
dbm.State = DownloadDBState.ErrorOccured;
DownloadDBManager.Instance.Add(dbm);
return;
}
SetupFavicon(url);
Task.Run(() =>
{
var extractor = ExtractorManager.Instance.GetExtractor(url);
if (extractor == null)
{
Device.BeginInvokeOnMainThread(() =>
{
Status.Text = "적절한 다운로드작업을 찾을 수 없습니다!";
Status.TextColor = Color.Red;
Spinner.IsVisible = false;
});
dbm.State = DownloadDBState.ErrorOccured;
DownloadDBManager.Instance.Update(dbm);
return;
}
if (extractor.IsForbidden)
{
Device.BeginInvokeOnMainThread(() =>
{
Status.Text = "정책상 금지된 작업입니다.";
Status.TextColor = Color.Red;
Spinner.IsVisible = false;
});
dbm.State = DownloadDBState.Forbidden;
DownloadDBManager.Instance.Update(dbm);
return;
}
WAIT_ANOTHER_TASKS:
if (DownloadAvailable == 4)
{
Device.BeginInvokeOnMainThread(() =>
{
Status.Text = $"다른 작업이 끝나길 기다리는 중 입니다...";
});
while (DownloadAvailable >= 4)
{
Thread.Sleep(1000);
}
}
if (Interlocked.Increment(ref DownloadAvailable) > 4)
{
goto WAIT_ANOTHER_TASKS;
}
Device.BeginInvokeOnMainThread(() =>
{
Info.Text = extractor.GetType().Name.Replace("Extractor", "") + " (" + "/" + string.Join("/", url.Split('/').Skip(3)) + ")";
dbm.ShortInfo = Info.Text;
DownloadDBManager.Instance.Update(dbm);
Status.Text = "다운로드 정보를 추출 중 입니다...";
});
var option = extractor.RecommendOption(url);
long extracting_progress_max = 0;
long extracting_cumulative_count = 0;
option.ProgressMax = (count) =>
{
extracting_progress_max = count;
Device.BeginInvokeOnMainThread(() =>
{
ProgressProgressText.IsVisible = false;
Progress.IsVisible = true;
});
};
option.PostStatus = (count) =>
{
var val = Interlocked.Add(ref extracting_cumulative_count, count);
Device.BeginInvokeOnMainThread(() =>
{
if (extracting_progress_max != 0)
{
Progress.Progress = val / (double)extracting_progress_max;
Status.Text = $"추출중...[{val}/{extracting_progress_max}]";
}
else
{
Status.Text = $"추출중...[{val}개 항목 추출됨]";
}
});
};
option.SimpleInfoCallback = (info) =>
{
Device.BeginInvokeOnMainThread(() =>
{
Info.Text = $"{info}";
dbm.ShortInfo = info;
DownloadDBManager.Instance.Update(dbm);
});
};
option.ThumbnailCallback = (thumbnail) =>
{
Task.Run(async() =>
{
var ttask = NetTask.MakeDefault(thumbnail.Url);
ttask.Priority = new NetPriority {
Type = NetPriorityType.Trivial
};
ttask.Filename = Path.Combine(AppProvider.ApplicationPath, (url + "*thumbnail" + dbm.Id).GetHashMD5() + Path.GetExtension(thumbnail.Filename));
ttask.Headers = thumbnail.Headers;
ttask.Referer = thumbnail.Referer;
ttask.Cookie = thumbnail.Cookie;
ttask.Accept = thumbnail.Accept;
ttask.UserAgent = thumbnail.UserAgent;
dbm.ThumbnailCahce = ttask.Filename;
await NetTools.DownloadFileAsync(ttask);
DownloadDBManager.Instance.Update(dbm);
Device.BeginInvokeOnMainThread(() =>
{
Thumbnail.HeightRequest = Height - 8;
Thumbnail.IsVisible = true;
Thumbnail.Source = ttask.Filename;
});
});
};
(List <NetTask>, ExtractedInfo)tasks;
try
{
tasks = extractor.Extract(url, option);
}
catch (Exception e)
{
Logs.Instance.PushError(e.Message);
Logs.Instance.PushError(e.StackTrace);
Device.BeginInvokeOnMainThread(() =>
{
ProgressProgressText.IsVisible = true;
ProgressProgressText.Text = "";
ProgressText.Text = "";
Progress.IsVisible = false;
Spinner.IsVisible = false;
Status.Text = "추출 작업 중 오류가 발생했습니다 :(\n" + e.Message;
Status.TextColor = Color.Red;
});
Interlocked.Decrement(ref DownloadAvailable);
dbm.State = DownloadDBState.ErrorOccured;
DownloadDBManager.Instance.Update(dbm);
return;
}
if (tasks.Item1 == null)
{
Device.BeginInvokeOnMainThread(() =>
{
ProgressProgressText.IsVisible = true;
ProgressProgressText.Text = "";
ProgressText.Text = "";
Progress.IsVisible = false;
Spinner.IsVisible = false;
Status.Text = "다운로드할 내용이 없습니다 :(";
});
Interlocked.Decrement(ref DownloadAvailable);
dbm.State = DownloadDBState.ErrorOccured;
DownloadDBManager.Instance.Update(dbm);
return;
}
if (tasks.Item2 != null)
{
ExtractedInfo = tasks.Item2;
CacheManager.Instance.Append(url + "*info" + dbm.Id, ExtractedInfo);
dbm.InfoCache = url + "*info" + dbm.Id;
DownloadDBManager.Instance.Update(dbm);
}
var format = extractor.RecommendFormat(option);
Device.BeginInvokeOnMainThread(() =>
{
Spinner.IsVisible = false;
Status.Text = "다운로드 중...";
ProgressProgressText.IsVisible = false;
Progress.IsVisible = true;
});
int download_count = 0;
long download_bytes = 0;
long download_1s = 0;
int task_count = AppProvider.Scheduler.LatestPriority != null ?
AppProvider.Scheduler.LatestPriority.TaskPriority : 0;
int post_process_count = 0;
int post_process_progress = 0;
bool canceled = false;
if (tasks.Item1.Count > 0)
{
var hash_set = new HashSet <string>();
tasks.Item1.ForEach(x => hash_set.Add(Path.GetDirectoryName(Path.Combine("/", x.Format.Formatting(format)))));
if (hash_set.Count == 1)
{
dbm.Directory = Path.GetDirectoryName(Path.Combine(Settings.Instance.Model.SuperPath, tasks.Item1[0].Format.Formatting(format)));
}
else
{
dbm.Directory = Path.GetDirectoryName(Path.GetDirectoryName(Path.Combine(Settings.Instance.Model.SuperPath, tasks.Item1[0].Format.Formatting(format))));
}
dbm.CountOfFiles = tasks.Item1.Count;
DownloadDBManager.Instance.Update(dbm);
}
tasks.Item1.ForEach(task => {
task.Priority.TaskPriority = task_count++;
task.Filename = Path.Combine(Settings.Instance.Model.SuperPath, task.Format.Formatting(format));
if (!Directory.Exists(Path.GetDirectoryName(task.Filename)))
{
Directory.CreateDirectory(Path.GetDirectoryName(task.Filename));
}
task.DownloadCallback = (sz) =>
{
Interlocked.Add(ref download_1s, sz);
Interlocked.Add(ref download_bytes, sz);
};
task.CompleteCallback = () =>
{
var cur = Interlocked.Increment(ref download_count);
Device.BeginInvokeOnMainThread(() =>
{
Progress.Progress = cur / (double)tasks.Item1.Count;
});
};
task.CancleCallback = () =>
{
if (!canceled)
{
dbm.State = DownloadDBState.Aborted;
DownloadDBManager.Instance.Update(dbm);
}
canceled = true;
};
task.Cancel = CancelSource.Token;
if (task.PostProcess != null)
{
task.StartPostprocessorCallback = () =>
{
Interlocked.Increment(ref post_process_count);
};
task.PostProcess.CompletePostprocessor = (index) =>
{
Interlocked.Increment(ref post_process_progress);
};
}
AppProvider.Scheduler.Add(task);
});
while (tasks.Item1.Count != download_count && !canceled)
{
Thread.Sleep(1000);
Device.BeginInvokeOnMainThread(() =>
{
Status.Text = $"[{download_count}/{tasks.Item1.Count}] ({convert_bytes2string(download_1s)}/S {convert_bytes2string(download_bytes)})";
Interlocked.Exchange(ref download_1s, 0);
});
}
Interlocked.Decrement(ref DownloadAvailable);
dbm.State = DownloadDBState.Downloaded;
dbm.EndsTime = DateTime.Now;
dbm.SizeOfContents = download_bytes;
DownloadDBManager.Instance.Update(dbm);
while (post_process_progress != post_process_count && !canceled)
{
Device.BeginInvokeOnMainThread(() =>
{
Status.Text = $"후처리 작업 중...[{post_process_progress}/{post_process_count}]";
Progress.Progress = post_process_progress / (double)post_process_count;
});
Thread.Sleep(1000);
}
if (!canceled)
{
Device.BeginInvokeOnMainThread(() =>
{
Status.Text = "다운로드 완료";
ProgressProgressText.IsVisible = true;
ProgressProgressText.Text = "";
ProgressText.Text = "";
Progress.IsVisible = false;
Plugin.XSnack.CrossXSnack.Current.ShowMessage(Info.Text + " 항목의 다운로드가 완료되었습니다.");
});
}
else
{
Device.BeginInvokeOnMainThread(() =>
{
Status.Text = "다운로드 취소됨";
ProgressProgressText.IsVisible = true;
ProgressProgressText.Text = "";
ProgressText.Text = "";
Progress.IsVisible = false;
Plugin.XSnack.CrossXSnack.Current.ShowMessage(Info.Text + " 항목의 다운로드가 취소되었습니다.");
});
}
DownloadInfo.DownloadEnds = DateTime.Now;
});
}
public override (List <NetTask>, ExtractedInfo) Extract(string url, IExtractorOption option = null)
{
var match = ValidUrl.Match(url).Groups;
if (option == null)
{
option = RecommendOption(url);
}
if (option.Type == ExtractorType.Images)
{
var sinfo = new ExtractedInfo.WorksComic();
var imgs_url = $"https://ltn.hitomi.la/galleries/{match["id"].Value}.js";
option.PageReadCallback?.Invoke($"https://ltn.hitomi.la/galleryblock/{match["id"]}.html");
option.PageReadCallback?.Invoke(url);
option.PageReadCallback?.Invoke(imgs_url);
var urls = new List <string> {
$"https://ltn.hitomi.la/galleryblock/{match["id"]}.html",
imgs_url
};
var strings = NetTools.DownloadStrings(urls);
if (string.IsNullOrEmpty(strings[0]) || string.IsNullOrEmpty(strings[1]))
{
return(null, null);
}
var data1 = ParseGalleryBlock(strings[0]);
var imgs = strings[1];
var string2 = NetTools.DownloadString($"https://hitomi.la{data1.Magic}");
if (string.IsNullOrEmpty(string2))
{
return(null, null);
}
var data2 = ParseGallery(string2);
option.SimpleInfoCallback?.Invoke($"[{match["id"].Value}] {data1.Title}");
// download.js
var number_of_frontends = 3;
var subdomain = Convert.ToChar(97 + (Convert.ToInt32(match["id"].Value.Last()) % number_of_frontends));
if (match["id"].Value.Last() == '0')
{
subdomain = 'a';
}
var vv = JToken.Parse(imgs.Substring(imgs.IndexOf('=') + 1))["files"];
var arr = (JArray)vv;
//var arr = JArray.Parse(imgs.Substring(imgs.IndexOf('[')));
var img_urls = new List <string>();
foreach (var obj in arr)
{
var hash = obj.Value <string>("hash");
if (obj.Value <int>("haswebp") == 0 || hash == null)
{
img_urls.Add($"https://{subdomain}a.hitomi.la/galleries/{match["id"].Value}/{obj.Value<string>("name")}");
}
else if (hash == "")
{
img_urls.Add($"https://{subdomain}a.hitomi.la/webp/{obj.Value<string>("name")}.webp");
}
else if (hash.Length < 3)
{
img_urls.Add($"https://{subdomain}a.hitomi.la/webp/{hash}.webp");
}
else
{
var postfix = hash.Substring(hash.Length - 3);
img_urls.Add($"https://{subdomain}a.hitomi.la/webp/{postfix[2]}/{postfix[0]}{postfix[1]}/{hash}.webp");
}
}
var result = new List <NetTask>();
var ordering = 1;
foreach (var img in img_urls)
{
var filename = Path.GetFileNameWithoutExtension(img.Split('/').Last());
if (!(option as HitomiExtractorOption).RealFilename)
{
filename = ordering++.ToString("000");
}
var task = NetTask.MakeDefault(img);
task.SaveFile = true;
task.Filename = img.Split('/').Last();
task.Format = new ExtractorFileNameFormat
{
Title = data1.Title,
Id = match["id"].Value,
Language = data1.Language,
UploadDate = data1.Posted,
FilenameWithoutExtension = filename,
Extension = Path.GetExtension(img.Split('/').Last()).Replace(".", "")
};
if (data1.artist != null)
{
task.Format.Artist = data1.artist[0];
}
else
{
task.Format.Artist = "NA";
}
if (data1.parody != null)
{
task.Format.Series = data1.parody[0];
}
else
{
task.Format.Series = "NA";
}
if (data2.group != null)
{
task.Format.Group = data2.group[0];
}
else
{
task.Format.Group = "NA";
}
if (data2.character != null)
{
task.Format.Character = data2.character[0];
}
else
{
task.Format.Character = "NA";
}
if (task.Format.Artist == "NA" && task.Format.Group != "NA")
{
task.Format.Artist = task.Format.Group;
}
result.Add(task);
}
option.ThumbnailCallback?.Invoke(result[0]);
sinfo.Thumbnail = result[0];
sinfo.URL = url;
sinfo.Title = data1.Title;
sinfo.Author = data1.artist?.ToArray();
sinfo.AuthorGroup = data2.group?.ToArray();
sinfo.ShortInfo = $"[{match["id"].Value}] {data1.Title}";
sinfo.Tags = data1.Tags?.ToArray();
sinfo.Characters = data2.character?.ToArray();
sinfo.Language = data1.Language;
sinfo.Parodies = data1.parody?.ToArray();
result.ForEach(task => task.Format.Extractor = GetType().Name.Replace("Extractor", ""));
return(result, new ExtractedInfo {
Info = sinfo, Type = ExtractedInfo.ExtractedType.WorksComic
});
}
return(null, null);
}
public override (List <NetTask>, ExtractedInfo) Extract(string url, IExtractorOption option = null)
{
if (!PixivAPI.Auth(Settings.Instance.Model.PixivSettings.Id, Settings.Instance.Model.PixivSettings.Password))
{
throw new ExtractorException("Authentication error! Check setting.json/PixivSetting.");
}
var match = ValidUrl.Match(url).Groups;
if (option == null)
{
option = new PixivExtractorOption {
Type = ExtractorType.Works
}
}
;
if (match[2].Value.StartsWith("member") && option.ExtractInformation == false)
{
var user = PixivAPI.GetUsersAsync(match["id"].Value.ToInt()).Result;
var works = PixivAPI.GetUsersWorksAsync(match["id"].Value.ToInt(), 1, 10000000).Result;
option.SimpleInfoCallback?.Invoke($"{user[0].Name} ({user[0].Account})");
option.ThumbnailCallback?.Invoke(NetTask.MakeDefault(user[0].ProfileImageUrls.Px170x170));
var result = new List <NetTask>();
foreach (var work in works)
{
if (work.PageCount > 1)
{
;
}
if (work.Type == null || work.Type == "illustration")
{
var task = NetTask.MakeDefault(work.ImageUrls.Large);
task.Filename = work.ImageUrls.Large.Split('/').Last();
task.SaveFile = true;
task.Referer = url;
task.Format = new ExtractorFileNameFormat
{
Artist = user[0].Name,
Account = user[0].Account,
Id = user[0].Id.Value.ToString(),
FilenameWithoutExtension = Path.GetFileNameWithoutExtension(work.ImageUrls.Large.Split('/').Last()),
Extension = Path.GetExtension(work.ImageUrls.Large.Split('/').Last()).Replace(".", "")
};
result.Add(task);
}
else if (work.Type == "ugoira")
{
var ugoira_data = PixivAPI.GetUgoiraAsync(work.Id.ToString()).Result;
var task = NetTask.MakeDefault(ugoira_data.ZipUrls.Medium);
task.Filename = ugoira_data.ZipUrls.Medium.Split('/').Last();
task.SaveFile = true;
task.Referer = url;
var pptask = new PostprocessorTask();
pptask.Postprocessor = new UgoiraPostprocessor {
Frames = ugoira_data.Frames
};
task.PostProcess = pptask;
task.Format = new ExtractorFileNameFormat
{
Artist = user[0].Name,
Account = user[0].Account,
Id = user[0].Id.Value.ToString(),
FilenameWithoutExtension = Path.GetFileNameWithoutExtension(ugoira_data.ZipUrls.Medium.Split('/').Last()),
Extension = Path.GetExtension(ugoira_data.ZipUrls.Medium.Split('/').Last()).Replace(".", "")
};
result.Add(task);
}
}
result.ForEach(task => task.Format.Extractor = GetType().Name.Replace("Extractor", ""));
return(result, new ExtractedInfo {
Type = ExtractedInfo.ExtractedType.UserArtist
});
}
else if (option.ExtractInformation == true)
{
var user = PixivAPI.GetUsersAsync(match["id"].Value.ToInt()).Result;
return(null, null /*user*/);
}
return(null, null);
}
public override (List <NetTask>, ExtractedInfo) Extract(string url, IExtractorOption option = null)
{
var match = ValidUrl.Match(url).Groups;
if (option == null)
{
option = new DanbooruExtractorOption {
Type = ExtractorType.Images
}
}
;
var tags = match["search"].Value;
var result = new List <NetTask>();
var page = 1;
option.SimpleInfoCallback?.Invoke($"{tags}");
if ((option as DanbooruExtractorOption).StartPage != null)
{
page = (option as DanbooruExtractorOption).StartPage[0].ToInt();
}
var end_page = int.MaxValue;
if ((option as DanbooruExtractorOption).EndPage != null)
{
end_page = (option as DanbooruExtractorOption).EndPage[0].ToInt();
}
var already_thumbnail = false;
while (true)
{
var durl = $"https://danbooru.donmai.us/posts?tags={tags}&page=" + page.ToString();
option.PageReadCallback?.Invoke(durl);
var html = NetTools.DownloadString(durl);
var node = html.ToHtmlNode().SelectNodes("/html[1]/body[1]/div[1]/div[3]/div[1]/section[1]/div[3]/div[1]/article");
if (node == null)
{
break;
}
var ds = new List <string>();
foreach (var sub in node)
{
ds.Add("https://danbooru.donmai.us" + sub.SelectSingleNode("./a").GetAttributeValue("href", ""));
}
var htmls = NetTools.DownloadStrings(ds);
//foreach (var shtml in htmls)
for (int i = 0; i < htmls.Count; i++)
{
var snode = htmls[i].ToHtmlNode();
var img_url = "";
// Just one banner
if (snode.SelectSingleNode("/html[1]/body[1]/div[1]/div[2]/div[1]/section[1]/div[1]/span[1]/a[1]")?.GetAttributeValue("id", "") == "image-resize-link")
{
img_url = snode.SelectSingleNode("/html[1]/body[1]/div[1]/div[2]/div[1]/section[1]/div[1]/span[1]/a[1]").GetAttributeValue("href", "");
}
// Two banner
else if (snode.SelectSingleNode("/html[1]/body[1]/div[1]/div[2]/div[1]/section[1]/div[2]/span[1]/a[1]")?.GetAttributeValue("id", "") == "image-resize-link")
{
img_url = snode.SelectSingleNode("/html[1]/body[1]/div[1]/div[2]/div[1]/section[1]/div[2]/span[1]/a[1]").GetAttributeValue("href", "");
}
// Three or none banner
else if (snode.SelectSingleNode("/html[1]/body[1]/div[1]/div[2]/div[1]/section[1]/section[1]/img[1]") != null)
{
img_url = snode.SelectSingleNode("/html[1]/body[1]/div[1]/div[2]/div[1]/section[1]/section[1]/img[1]").GetAttributeValue("src", "");
}
// Video URL
else if (snode.SelectSingleNode("/html[1]/body[1]/div[1]/div[2]/div[1]/section[1]/section[1]/p[1]/a[1]") != null)
{
if ((option as DanbooruExtractorOption).ExcludeVideo)
{
continue;
}
img_url = snode.SelectSingleNode("/html[1]/body[1]/div[1]/div[2]/div[1]/section[1]/section[1]/p[1]/a[1]").GetAttributeValue("href", "");
}
else
{
// ?
Log.Logs.Instance.PushError("[DanbooruExtractor] Cannot find html format! " + ds[i]);
}
var task = NetTask.MakeDefault(img_url);
task.SaveFile = true;
task.Filename = img_url.Split('/').Last();
task.Format = new ExtractorFileNameFormat
{
Search = tags,
FilenameWithoutExtension = Path.GetFileNameWithoutExtension(task.Filename),
Extension = Path.GetExtension(task.Filename).Replace(".", "")
};
result.Add(task);
}
if (!already_thumbnail)
{
option.ThumbnailCallback?.Invoke(result[0]);
already_thumbnail = true;
}
page += 1;
if (page > end_page)
{
break;
}
}
result.ForEach(task => task.Format.Extractor = GetType().Name.Replace("Extractor", ""));
return(result, new ExtractedInfo {
Type = ExtractedInfo.ExtractedType.UserArtist
});
}
}
private void start()
{
int ps;
if (!int.TryParse(textBox3.Text, out ps))
{
append("숫자만 입력해주세요.");
return;
}
int pe;
if (!int.TryParse(textBox4.Text, out pe))
{
append("숫자만 입력해주세요.");
return;
}
var invalid = new string(Path.GetInvalidFileNameChars()) + new string(Path.GetInvalidPathChars());
var sp = Path.Combine(Directory.GetCurrentDirectory(), "Archive",
$"몬무스 갤러리 (monmusu)");
var id = "monmusu";
Directory.CreateDirectory(sp);
var starts = ps;
status("진행중...[0/" + (pe - ps + 1).ToString("#,#") + "]");
bool real_cookie_receive = false;
try
{
for (; ps <= pe; ps++)
{
string url;
if (true)
{
url = $"https://gall.dcinside.com/mgallery/board/view/?id={id}&no={ps}";
}
else
{
url = $"https://gall.dcinside.com/board/view/?id={id}&no={ps}";
}
Logger.Instance.Push("Downloading String... = " + url);
var task = NetTask.MakeDefault(url);
task.Cookie = COOKIES;
bool y = real_cookie_receive;
if (real_cookie_receive == false)
{
task.HeaderReceive = (ef) =>
{
append("헤더받음: " + ef);
var xx = ef.Split('\n').First(x => x.Contains("Set-Cookie")).Replace("Set-Cookie: ", "").Trim();
COOKIES = "PHPSESSID=" + SESS;
COOKIES += "; PHPSESSKEY=" + xx.Split(new[] { "PHPSESSKEY=" }, StringSplitOptions.None)[1].Split(';')[0].Trim();
COOKIES += "; block_alert_monmusu=1";
real_cookie_receive = true;
append("쿠키 변경됨: " + COOKIES);
};
}
var html = NetTools.DownloadString(task);
if (y == false)
{
append("PS1");
ps--;
continue;
}
Logger.Instance.Push("Downloaded String = " + url);
if (string.IsNullOrEmpty(html))
{
append("실패: " + url);
Logger.Instance.Push("Fail: " + url);
goto NEXT;
}
if (html.Length < 1000 && html.Contains("해당 마이너 갤러리는 운영원칙 위반으로 접근이 제한되었습니다."))
{
append("실패: 접근 거부, 접근 가능한 아이디로 재시도하시기 바랍니다.");
break;
}
if (html.Contains("해당 마이너 갤러리는 운영원칙 위반(사유: 누드패치, 성행위 패치, 음란성 게시물 공지 등록 및 정리 안됨) 으로 접근이 제한되었습니다."))
{
//goto F**K;
return;
}
var info = DCInsideUtils.ParseBoardView(html, true);
Logger.Instance.Push("Parse: " + url);
// 해당 마이너 갤러리는 운영원칙 위반으로 접근이 제한되었습니다.\n마이너 갤러리 메인으로 돌아갑니다.
var ttitle = $"{info.Title}";
foreach (char c in invalid)
{
ttitle = ttitle.Replace(c.ToString(), "");
}
Logger.Instance.Push("ttile: " + ttitle);
File.WriteAllText(Path.Combine(sp, $"[{ps}]-body-{ttitle}.json"), JsonConvert.SerializeObject(info, Formatting.Indented));
Logger.Instance.Push("file-body: " + $"[{ps}]-body-{ttitle}.json");
int com;
if (int.TryParse(info.CommentCount.Replace(",", ""), out com) && com > 0)
{
try
{
var comments = DCInsideUtils.GetAllComments(id, ps.ToString()).Result;
File.WriteAllText(Path.Combine(sp, $"[{ps}]-comments-{ttitle}.json"), JsonConvert.SerializeObject(comments, Formatting.Indented));
Logger.Instance.Push("file-comment: " + $"[{ps}]-comments-{ttitle}.json");
}
catch { }
}
NEXT:
var ss = TimeSpan.FromMilliseconds(720 * (pe - ps));
var yy = "";
if (ss.Days > 0)
{
yy += ss.Days.ToString() + "일 ";
}
if (ss.Days > 0 || ss.Hours > 0)
{
yy += ss.Hours.ToString() + "시간 ";
}
if (ss.Days > 0 || ss.Hours > 0 || ss.Minutes > 0)
{
yy += ss.Minutes.ToString() + "분 ";
}
if (ss.Days > 0 || ss.Hours > 0 || ss.Minutes > 0 || ss.Seconds > 0)
{
yy += ss.Seconds.ToString() + "초 남음";
}
status("진행중...[" + (ps - starts + 1).ToString() + "/" + (pe - starts + 1).ToString("#,#") + "] 남은시간: " + yy);
Logger.Instance.Push("next: " + url + $" || {ps}/{pe}/{starts}");
Thread.Sleep(700);
}
status("완료");
} catch (Exception e)
{
append("실패: " + e.Message + "\r\n" + e.StackTrace);
}
}
