private bool DownloadWorker(HttpClientEx hc, ImageInfomation e, int uriIndex)
{
using (var req = new HttpRequestMessage(HttpMethod.Get, e.ImageUri[uriIndex]))
{
req.Headers.Referrer = this.Uri;
using (var res = hc.SendAsync(req).Exec())
{
if ((int)res.StatusCode / 100 != 2)
{
return(false);
}
e.TempStream.SetLength(0);
using (var fileWriter = new StreamWithNotify(e.TempStream, count => Interlocked.Add(ref this.m_downloaded, count)))
res.Content.CopyToAsync(fileWriter).Wait();
}
e.TempStream.Flush();
e.TempStream.Position = 0;
e.Extension = Signatures.GetExtension(e.TempStream);
if (e.Extension == null)
{
return(false);
}
// 이미지 암호화 푸는 작업
this.m_decryptor.Decrypt(e.TempStream);
}
this.IncrementProgress();
return(true);
}
private async Task <bool> DownloadWorker(HttpClientEx hc, ImageInfomation e, Uri uri, CancellationToken cancellationToken)
{
using (var req = new HttpRequestMessage(HttpMethod.Get, uri))
{
req.Headers.Referrer = this.Uri;
using (var res = await hc.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, cancellationToken))
{
if ((int)res.StatusCode / 100 != 2)
{
return(false);
}
if (res.Headers.Server.FirstOrDefault()?.ToString()?.Contains("ddos-guard") ?? false)
{
return(false);
}
using (var stream = await res.Content.ReadAsStreamAsync())
{
e.TempStream.SetLength(0);
using (var fileWriter = new StreamWithNotify(e.TempStream, count => Interlocked.Add(ref this.m_downloaded, count)))
{
await stream.CopyToAsync(fileWriter, 4096, cancellationToken);
}
}
}
e.TempStream.Flush();
e.TempStream.Position = 0;
e.Extension = Signatures.GetExtension(e.TempStream);
if (e.Extension == null)
{
return(false);
}
// 이미지 암호화 푸는 작업
this.m_decryptor.Decrypt(e.TempStream);
}
this.IncrementProgress();
return(true);
}
private MangaInfomation GetInfomationWorker(HttpClientEx hc, int retries, out HttpStatusCode statusCode)
{
var mangaInfo = new MangaInfomation();
var doc = new HtmlDocument();
string html;
using (var res = hc.GetAsync(this.Uri).Exec())
{
statusCode = res.StatusCode;
if (this.WaitFromHttpStatusCode(retries, statusCode))
{
return(null);
}
html = res.Content.ReadAsStringAsync().Exec();
if (string.IsNullOrWhiteSpace(html))
{
return(null);
}
mangaInfo.NewUri = res.RequestMessage.RequestUri ?? this.Uri;
}
doc.LoadHtml(html);
#region 폴더 이름은 Detail 에서 설정
{
// /bbs/page.php?hid=manga_detail&manga_id=9495
var toonNav = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'toon-nav')]");
if (toonNav == null)
{
return(null);
}
string detailCode = null;
foreach (HtmlNode node in toonNav.SelectNodes(".//a[@href]").ToArray())
{
var href = node.GetAttributeValue("href", "");
if (string.IsNullOrWhiteSpace(href))
{
continue;
}
var code = DaruUriParser.Detail.GetCode(new Uri(this.Uri, href));
if (string.IsNullOrWhiteSpace(code))
{
continue;
}
detailCode = code;
break;
}
var detailEntry = ArchiveManager.GetDetail(detailCode);
if (detailEntry == null)
{
var detailUri = DaruUriParser.Detail.GetUri(detailCode);
DetailPage.DetailInfomation detailInfo = null;
var detailResult = Utility.Retry((retries2) =>
{
using (var res = hc.GetAsync(detailUri).Exec())
{
if (this.WaitFromHttpStatusCode(retries2, res.StatusCode))
{
return(false);
}
var htmlDetail = res.Content.ReadAsStringAsync().Exec();
if (string.IsNullOrWhiteSpace(html))
{
return(false);
}
detailInfo = DetailPage.GetDetailInfomation(hc, detailUri, htmlDetail);
return(detailInfo != null);
}
});
if (!detailResult)
{
return(null);
}
ArchiveManager.UpdateDetail(detailCode, detailInfo.Title, detailInfo.MangaList.Select(e => e.MangaCode).ToArray());
detailEntry = ArchiveManager.GetDetail(detailCode);
if (detailEntry == null)
{
return(null);
}
}
this.Title = detailEntry.Title;
}
#endregion
#region Detail.Title + xx화
{
var titleNode = doc.DocumentNode.SelectSingleNode(".//div[contains(@class, 'toon-title')]");
if (titleNode == null)
{
return(null);
}
foreach (var titleNodeChild in titleNode.ChildNodes.ToArray())
{
if (titleNodeChild.NodeType == HtmlNodeType.Element)
{
titleNodeChild.Remove();
}
}
var title = Utility.ReplaceHtmlTagAndRemoveTab(titleNode.InnerText ?? string.Empty);
if (string.IsNullOrWhiteSpace(title))
{
return(null);
}
if (string.IsNullOrWhiteSpace(title))
{
return(null);
}
this.TitleWithNo = title;
}
#endregion
#region 이미지 정보 가져오는 부분
{
// /js/viewer.b.js?v=90
var chapter = int.Parse(Regex.Match(doc.DocumentNode.InnerHtml, "var *chapter *= *(\\d+)")?.Groups[1].Value ?? "0");
var catchArr = new Func <string, IEnumerable <string> >(e =>
{
var m = Regex.Match(doc.DocumentNode.InnerHtml, $"var {e} *= *((?:[^;\\\\]|\\\\.)*)");
if (!m.Success)
{
return(new string[0]);
}
return(Regex.Matches(m.Groups[1].Value, @"""((?:[^""\\]|\\.)*)""")
.Cast <Match>()
.Select(le => Regex.Replace(le.Groups[1].Value, @"\\(.)", lm => lm.Groups[1].Value)));
});
var cdnList = catchArr("cdn_domains")
.ToArray();
var imgList = catchArr("img_list")
.Select(e => new Uri(this.Uri, e + "?quick"))
.ToArray();
var imgList1 = catchArr("img_list1")
.Select(e => new Uri(this.Uri, e))
.ToArray();
var lst = new HashSet <Uri>(new UriIEqualityComparer());
for (var i = 0; i < imgList.Length; i++)
{
lst.Clear();
var addList = new Action <Uri>(uri =>
{
var host = uri.Host;
if (cdnList?.Length > 0)
{
if (host.Contains("cdntigermask.xyz") || host.Contains("cdnmadmax.xyz") || host.Contains("filecdn.xyz"))
{
lst.Add(new UriBuilder(uri)
{
Host = cdnList[(chapter + 4 * i) % cdnList.Length]
}.Uri);
for (var cdnPeek = 0; cdnPeek < CDNPeekCount; cdnPeek++)
{
lst.Add(new UriBuilder(uri)
{
Host = cdnList[Random.Next(cdnList.Length)]
}.Uri);
lst.Add(new UriBuilder(uri)
{
Host = cdnList[Random.Next(cdnList.Length)]
}.Uri);
lst.Add(new UriBuilder(uri)
{
Host = cdnList[Random.Next(cdnList.Length)]
}.Uri);
}
}
}
lst.Add(imgList[i]);
if (!host.Contains("google.") | !host.Contains("blogspot."))
{
lst.Add(new UriBuilder(uri)
{
Host = "s3." + host
}.Uri);
lst.Add(new UriBuilder(uri)
{
Host = host.Replace("img.", "s3.")
}.Uri);
}
});
if (i < imgList1.Length)
{
addList(imgList1[i]);
}
addList(imgList[i]);
var imageInfo = new ImageInfomation()
{
Index = i + 1,
ImageUri = lst.Distinct().ToArray(),
};
mangaInfo.Images.Add(imageInfo);
}
}
#endregion
this.m_decryptor = new ImageDecryptor(html, mangaInfo.NewUri);
return(mangaInfo);
}
private MangaInfomation GetInfomationWorker(HttpClientEx hc, int retries, out HttpStatusCode statusCode)
{
var mangaInfo = new MangaInfomation();
var doc = new HtmlDocument();
string html;
using (var req = new HttpRequestMessage(HttpMethod.Get, this.Uri))
using (var res = this.CallRequest(hc, req))
{
statusCode = res.StatusCode;
if (this.WaitFromHttpStatusCode(retries, statusCode))
{
return(null);
}
html = res.Content.ReadAsStringAsync().Exec();
if (string.IsNullOrWhiteSpace(html))
{
return(null);
}
mangaInfo.NewUri = res.RequestMessage.RequestUri ?? this.Uri;
}
doc.LoadHtml(html);
#region 폴더 이름은 Detail 에서 설정
{
// /bbs/page.php?hid=manga_detail&manga_id=9495
var toonNav = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'toon-nav')]");
if (toonNav == null)
{
return(null);
}
string detailCode = null;
foreach (HtmlNode node in toonNav.SelectNodes(".//a[@href]").ToArray())
{
var href = node.GetAttributeValue("href", "");
if (string.IsNullOrWhiteSpace(href))
{
continue;
}
var code = DaruUriParser.Detail.GetCode(new Uri(this.Uri, href));
if (string.IsNullOrWhiteSpace(code))
{
continue;
}
detailCode = code;
break;
}
var detailEntry = ArchiveManager.GetDetail(detailCode);
if (detailEntry == null)
{
var detailUri = DaruUriParser.Detail.GetUri(detailCode);
DetailPage.DetailInfomation detailInfo = null;
var detailResult = Utility.Retry((retries2) =>
{
using (var req = new HttpRequestMessage(HttpMethod.Get, detailUri))
using (var res = this.CallRequest(hc, req))
{
if (this.WaitFromHttpStatusCode(retries2, res.StatusCode))
{
return(false);
}
var htmlDetail = res.Content.ReadAsStringAsync().Exec();
if (string.IsNullOrWhiteSpace(html))
{
return(false);
}
detailInfo = DetailPage.GetDetailInfomation(hc, detailUri, htmlDetail);
return(detailInfo != null);
}
});
if (!detailResult)
{
return(null);
}
ArchiveManager.UpdateDetail(detailCode, detailInfo.Title, detailInfo.MangaList.Select(e => e.MangaCode).ToArray());
detailEntry = ArchiveManager.GetDetail(detailCode);
if (detailEntry == null)
{
return(null);
}
}
this.Title = detailEntry.Title;
}
#endregion
#region Detail.Title + xx화
{
var titleNode = doc.DocumentNode.SelectSingleNode(".//div[contains(@class, 'toon-title')]");
if (titleNode == null)
{
return(null);
}
foreach (var titleNodeChild in titleNode.ChildNodes.ToArray())
{
if (titleNodeChild.NodeType == HtmlNodeType.Element)
{
titleNodeChild.Remove();
}
}
var title = Utility.ReplaceHtmlTagAndRemoveTab(titleNode.InnerText ?? string.Empty);
if (string.IsNullOrWhiteSpace(title))
{
return(null);
}
if (string.IsNullOrWhiteSpace(title))
{
return(null);
}
this.TitleWithNo = title;
}
#endregion
#region 이미지 정보 가져오는 부분
{
var imgList = Regex.Matches(Regex.Match(doc.DocumentNode.InnerHtml, "var img_list = [^;]+").Groups[0].Value, "\"([^\"]+)\"")
.Cast <Match>()
.Select(e => e.Groups[1].Value.Replace("\\", ""))
.Select(e => new Uri(this.Uri, e))
.ToArray();
var imgList1 = Regex.Matches(Regex.Match(html, "var img_list1 = [^;]+").Groups[0].Value, "\"([^\"]+)\"")
.Cast <Match>()
.Select(e => e.Groups[1].Value.Replace("\\", ""))
.Select(e => new Uri(this.Uri, e))
.ToArray();
for (var i = 0; i < imgList.Length; i++)
{
var imageInfo = new ImageInfomation()
{
Index = i + 1,
};
if (i < imgList1.Length)
{
imageInfo.ImageUri = new Uri[]
{
new Uri(imgList[i].ToString().Replace("//img.", "//s3.")),
imgList[i],
imgList1[i],
}.Distinct().ToArray();
}
else
{
imageInfo.ImageUri = new Uri[]
{
new Uri(imgList[i].ToString().Replace("//img.", "//s3.")),
imgList[i],
}.Distinct().ToArray();
}
mangaInfo.Images.Add(imageInfo);
}
}
#endregion
this.m_decryptor = new ImageDecryptor(html, mangaInfo.NewUri);
return(mangaInfo);
}
private async Task DownloadImage(ImageInfomation e, HttpClientEx hc, CancellationToken cancellationToken, Action cancel)
{
for (var index = 0; index < e.ImageUri.Length && !cancellationToken.IsCancellationRequested; ++index)
{
lock (CdnScore)
{
Array.Sort(
e.ImageUri,
(a, b) => GetScore(a.Uri.Host).CompareTo(GetScore(b.Uri.Host)) * -1
);
}
for (var i = 0; i < e.ImageUri.Length && !cancellationToken.IsCancellationRequested; ++i)
{
if (e.ImageUri[i].Downloaded)
{
continue;
}
var succ = await Utility.RetryAsync(async (retries) => await this.DownloadWorker(hc, e, e.ImageUri[i].Uri, cancellationToken), 1);
e.ImageUri[i].Downloaded = true;
var h = e.ImageUri[i].Uri.Host;
lock (CdnScore)
{
CdnScore[h] = (CdnScore.TryGetValue(h, out var c) ? c : CdnScoreDefault) + (succ ? 1 : -2);
}
if (succ)
{
return;
}
}
}
if (!this.IgnoreErrorMissingPage)
{
cancel();
}
int GetScore(string host)
{
if (CdnScore.TryGetValue(host, out var v))
{
return(v);
}
while (true)
{
var dot = host.IndexOf('.');
if (dot == -1)
{
break;
}
var dotNext = host.IndexOf('.', dot + 1);
if (dotNext == -1)
{
break;
}
host = host.Substring(dot + 1);
if (CdnScore.TryGetValue(host, out v))
{
return(v);
}
}
return(CdnScoreDefault);
}
}
