public NijieImage ParseImage(NijieImage image, ref NijieMember member, HtmlDocument doc)
{
checkErrorMessage(doc);
var doujinDiv = doc.DocumentNode.SelectSingleNode("//div[@id='dojin_header']");
if (doujinDiv != null)
{
ProcessDoujin(image, doc);
}
else
{
var videoDiv = doc.DocumentNode.SelectSingleNode("//div[@id='gallery']//video");
if (videoDiv != null)
{
image.IsVideo = true;
}
if (member == null)
{
member = ParseMemberFromImage(doc);
}
image.Member = member;
ParseImageLinks(image, doc);
ParseImageTitleAndDescription(image, doc);
ParseImageTags(image, doc);
}
ParseImageExtras(image, doc);
return(image);
}
/// <summary>
/// Parse member from internet based on given member id, mode, and page number.
/// </summary>
/// <param name="memberId"></param>
/// <param name="mode"></param>
/// <param name="page"></param>
/// <returns></returns>
public NijieMember ParseMember(int memberId, MemberMode mode, int page)
{
HtmlDocument doc = null;
try
{
canOperate();
NijieMember member = new NijieMember(memberId, mode, page);
var result = getPage(member.MemberUrl);
var res = result.Item2;
if (Util.IsRedirected(res.ResponseUri.ToString(), member.MemberUrl, ignoreProtocol: true))
{
throw new NijieException(string.Format("Redirected to another page: {0} ==> {1}", member.MemberUrl, res.ResponseUri.ToString()), NijieException.MEMBER_REDIR);
}
doc = result.Item1;
return(ParseMember(doc, member, mode));
}
catch (NijieException)
{
throw;
}
catch (Exception ex)
{
if (doc != null)
{
var filename = String.Format("Dump for Member {0} Mode {1}.html", memberId, mode.ToString());
Log.Debug("Dumping member page to: " + filename);
doc.Save(filename);
}
throw new NijieException(String.Format("Error when processing member: {0} ==> {1}", memberId, ex.Message), ex, NijieException.MEMBER_UNKNOWN_ERROR);
}
}
private void ParseMemberImages(HtmlDocument doc, NijieMember member)
{
var imagesDivs = doc.DocumentNode.SelectNodes("//div[@id='members_dlsite_left']/div");
if (imagesDivs == null)
{
// doujin page
imagesDivs = doc.DocumentNode.SelectNodes("//div[@class='mem-index clearboth']");
}
var imagesDiv = "";
foreach (var item in imagesDivs)
{
if (item.InnerHtml.Contains("class=\"nijie\""))
{
imagesDiv = item.InnerHtml;
break;
}
}
member.Images = ParseImageList(imagesDiv, member.MemberUrl);
foreach (var image in member.Images)
{
image.Member = member;
}
CheckNextPageAvailable(doc, member);
var imageCountElements = doc.DocumentNode.SelectNodes("//p[@class='mem-indent float-left']/em");
member.TotalImages = ParseTotalImageCount(imageCountElements);
}
public void TestVideoParserMethod()
{
NijieMember nullMember = null;
//var nijie = Nijie.GetInstance();
{
// http://nijie.info/view.php?id=256283
var page = UpdateHtmlForm.PATH + "image-video.html";
// http://nijie.info/view_popup.php?id=256283
var ppage = UpdateHtmlForm.PATH + "image-video-popup.html";
Assert.IsTrue(File.Exists(page), "Test file is missing!");
Assert.IsTrue(File.Exists(ppage), "Test file is missing!");
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(File.ReadAllText(page));
HtmlDocument pdoc = new HtmlDocument();
pdoc.LoadHtml(File.ReadAllText(ppage));
var image = new NijieImage(UpdateHtmlForm.VIDEO);
var result = nijie.ParseImage(image, ref nullMember, doc);
Assert.IsTrue(image.ImageId > 0, "Image Id not valid");
Assert.IsTrue(image.IsVideo, "Image is not video");
Assert.IsNotNull(image.BigImageUrl, "Big image url is missing!");
Nijie.ParseImagePopUp(image, pdoc);
}
}
public NijieImage ParseImage(NijieImage image, NijieMember member = null)
{
HtmlDocument doc = null;
try
{
canOperate();
var result = getPage(image.ViewUrl);
PrintCookie("Image Page " + image.ImageId);
doc = result.Item1;
if (Util.IsRedirected(result.Item2.ResponseUri.ToString(), image.ViewUrl, true))
{
Log.Debug(String.Format("Redirection for Image {0}: {1} ==> {2}, possibly locked.", image.ImageId, image.ViewUrl, result.Item2.ResponseUri));
image.IsFriendOnly = true;
return(image);
}
return(ParseImage(image, ref member, doc));
}
catch (NijieException)
{
throw;
}
catch (Exception ex)
{
if (doc != null)
{
var filename = "Dump for Image " + image.ImageId + ".html";
Log.Debug("Dumping image page to: " + filename);
doc.Save(filename);
}
throw new NijieException(String.Format("Error when processing image: {0} ==> {1}", image.ImageId, ex.Message), ex, NijieException.IMAGE_UNKNOWN_ERROR);
}
}
public Tuple <List <NijieMember>, bool> ParseMyMemberBookmark(int page)
{
canOperate();
List <NijieMember> members = new List <NijieMember>();
HtmlDocument doc = null;
bool isNextPageAvailable = false;
try
{
var url = Util.FixUrl(String.Format("//nijie.info/like_my.php?p={0}", page), ROOT_DOMAIN, Properties.Settings.Default.UseHttps);
var result = getPage(url);
doc = result.Item1;
var membersDiv = doc.DocumentNode.SelectNodes("//div[@class='nijie-okini']");
if (membersDiv != null)
{
foreach (var memberDiv in membersDiv)
{
var memberUrl = memberDiv.SelectSingleNode("//div[@class='nijie-okini']//a").Attributes["href"].Value;
var res = re_member.Match(memberUrl);
if (res.Success)
{
var member = new NijieMember(Int32.Parse(res.Groups[1].Value), 0);
member.UserName = memberDiv.SelectSingleNode("//div[@class='nijie-okini']//p[@class='sougo']").InnerText;
member.AvatarUrl = memberDiv.SelectSingleNode("//div[@class='nijie-okini']//a//img").Attributes["src"].Value;
members.Add(member);
}
memberDiv.Remove();
}
}
var nextPageButton = doc.DocumentNode.SelectNodes("//p[@class='page_button']//a");
if (nextPageButton != null)
{
foreach (var item in nextPageButton)
{
if (item.InnerText.StartsWith("次へ"))
{
isNextPageAvailable = true;
break;
}
}
}
}
catch (Exception ex)
{
if (doc != null)
{
var filename = "Dump for My Member Bookmark.html";
Log.Debug("Dumping My Member Bookmark page to: " + filename);
doc.Save(filename);
}
throw new NijieException(String.Format("Error when processing my member bookmark ==> {0}", ex.Message), ex, NijieException.IMAGE_UNKNOWN_ERROR);
}
return(new Tuple <List <NijieMember>, bool>(members, isNextPageAvailable));
}
public void TestMangaParserMethod()
{
NijieMember nullMember = null;
//var nijie = Nijie.GetInstance();
{
{
var page = UpdateHtmlForm.PATH + "image-manga.html";
var ppage = UpdateHtmlForm.PATH + "image-manga-popup.html";
Assert.IsTrue(File.Exists(page), "Test file is missing!");
Assert.IsTrue(File.Exists(ppage), "Test file is missing!");
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(File.ReadAllText(page));
HtmlDocument pdoc = new HtmlDocument();
pdoc.LoadHtml(File.ReadAllText(ppage));
var image = new NijieImage(UpdateHtmlForm.MANGA);
var result = nijie.ParseImage(image, ref nullMember, doc);
Assert.IsTrue(image.ImageId > 0, "Image Id not valid");
Assert.IsTrue(image.IsManga, "Image is not manga");
Assert.AreEqual(5, image.MangaPages.Count, "Manga pages count are different!");
Nijie.ParseImagePopUp(image, pdoc);
Assert.AreEqual(5, image.ImageUrls.Count, "Manga image urls count are different!");
foreach (var item in image.MangaPages)
{
Assert.IsNotNull(item.ImageUrl, "Image url is missing!");
}
}
{
var page = UpdateHtmlForm.PATH + "image-manga-filter.html";
var ppage = UpdateHtmlForm.PATH + "image-manga-popup-filter.html";
Assert.IsTrue(File.Exists(page), "Test file is missing!");
Assert.IsTrue(File.Exists(ppage), "Test file is missing!");
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(File.ReadAllText(page));
HtmlDocument pdoc = new HtmlDocument();
pdoc.LoadHtml(File.ReadAllText(ppage));
var image = new NijieImage(UpdateHtmlForm.MANGA2);
var result = nijie.ParseImage(image, ref nullMember, doc);
Assert.IsTrue(image.ImageId > 0, "Image Id not valid");
Assert.IsTrue(image.IsManga, "Image is not manga");
Assert.AreEqual(4, image.MangaPages.Count, "Manga pages count are different!");
Nijie.ParseImagePopUp(image, pdoc);
Assert.AreEqual(4, image.ImageUrls.Count, "Manga image urls count are different!");
foreach (var item in image.MangaPages)
{
Assert.IsNotNull(item.ImageUrl, "Image url is missing!");
}
}
}
}
private void ParseMemberProfile(HtmlDocument doc, NijieMember member)
{
var profileDiv = doc.DocumentNode.SelectSingleNode("//div[@id='pro']/p/a/img");
if (profileDiv != null)
{
member.UserName = profileDiv.Attributes["alt"].Value;
member.AvatarUrl = profileDiv.Attributes["src"].Value;
}
}
public void GetMember(SynchronizationContext context)
{
try
{
_member = MainWindow.Bot.ParseMember(this.MemberId, this.Mode, this.Page);
this.UserName = _member.UserName;
ImageLoader.LoadImage(_member.AvatarUrl, _member.MemberUrl,
new Action <BitmapImage, string>((image, status) =>
{
this.AvatarImage = null;
this.AvatarImage = image;
_avatarImageStatus = status;
}
));
if (_member.Images != null)
{
Images = new ObservableCollection <NijieImageViewModel>();
foreach (var image in _member.Images)
{
var temp = new NijieImageViewModel(image);
context.Send((x) =>
{
Images.Add(temp);
}, null);
}
this.Status = String.Format("Loaded: {0} images.", _member.Images.Count);
onPropertyChanged("TotalImages");
this.HasError = false;
}
}
catch (NijieException ne)
{
MainWindow.Log.Error(ne.Message, ne);
this.UserName = null;
this.AvatarImage = ViewModelHelper.NoAvatar;
context.Send((x) =>
{
if (Images != null)
{
Images.Clear();
Images = null;
}
}, null);
this.HasError = true;
this.Status = "[Error] " + ne.Message;
}
}
private NijieMember ParseMemberFromImage(HtmlDocument doc)
{
var memberUrl = doc.DocumentNode.SelectSingleNode("//div[@id='pro']/p/a").Attributes["href"].Value;
var split = memberUrl.Split('?');
int memberId = Int32.Parse(split[1].Replace("id=", ""));
NijieMember member = new NijieMember(memberId, 0);
var profileDiv = doc.DocumentNode.SelectSingleNode("//div[@id='pro']/p/a/img");
if (profileDiv != null)
{
member.UserName = profileDiv.Attributes["alt"].Value;
member.AvatarUrl = profileDiv.Attributes["src"].Value;
}
return(member);
}
/// <summary>
/// Parse the image page
/// - Illustration
/// - Manga
/// </summary>
/// <param name="job"></param>
/// <param name="memberPage"></param>
/// <param name="imageTemp"></param>
private void processImage(JobDownloadViewModel job, NijieMember memberPage, NijieImage imageTemp)
{
if (isJobCancelled(job))
{
return;
}
MainWindow.Log.Debug("Processing Image:" + imageTemp.ImageId);
// skip if exists in DB
if (Properties.Settings.Default.SkipIfExistsInDB && !NijieDownloader.Library.Properties.Settings.Default.Overwrite)
{
using (var dao = new NijieContext())
{
var result = NijieImage.IsDownloadedInDB(imageTemp.ImageId);
if (result)
{
job.Message = String.Format("Image {0} already downloaded in DB", imageTemp.ImageId);
job.SkipCount++;
return;
}
}
}
var image = MainWindow.Bot.ParseImage(imageTemp, memberPage);
if (image.IsFriendOnly)
{
// sample: 74587
job.Message = "Image locked!";
return;
}
if (image.IsGoldenMember)
{
job.Message = "Image only for Gold Membership";
return;
}
if (image.IsManga)
{
processManga(job, image);
}
else
{
processIllustration(job, image);
}
}
/// <summary>
/// Parse member from html document.
/// </summary>
/// <param name="doc"></param>
/// <param name="member"></param>
/// <param name="mode"></param>
/// <returns></returns>
public NijieMember ParseMember(HtmlDocument doc, NijieMember member, MemberMode mode)
{
ParseMemberProfile(doc, member);
if (mode == MemberMode.Images || mode == MemberMode.Doujin)
{
ParseMemberImages(doc, member);
}
else
{
ParseMemberBookmark(doc, member);
}
member.Status = String.Format("Completed, found {0} images", member.Images.Count);
return(member);
}
private static void CheckNextPageAvailable(HtmlDocument doc, NijieMember member)
{
// check next page
member.IsNextAvailable = false;
var navButtons = doc.DocumentNode.SelectNodes("//p[@class='page_button']/a");
if (navButtons != null)
{
foreach (var item in navButtons)
{
if (item.InnerText.StartsWith("次へ"))
{
member.IsNextAvailable = true;
break;
}
}
}
}
public void TestImageParserMethod()
{
NijieMember nullMember = null;
//var nijie = Nijie.GetInstance();
{
var page = UpdateHtmlForm.PATH + "image-normal.html";
Assert.IsTrue(File.Exists(page), "Test file is missing!");
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(File.ReadAllText(page));
var image = new NijieImage(UpdateHtmlForm.IMAGE);
var result = nijie.ParseImage(image, ref nullMember, doc);
Assert.IsTrue(image.ImageId > 0, "Image Id not valid");
Assert.IsFalse(image.IsManga, "Image is not big image");
Assert.IsNotNull(image.BigImageUrl, "Big image url is missing!");
}
{
var page = UpdateHtmlForm.PATH + "image-doujin.html";
var ppage = UpdateHtmlForm.PATH + "image-doujin-popup.html";
Assert.IsTrue(File.Exists(page), "Test file is missing!");
Assert.IsTrue(File.Exists(ppage), "Test file is missing!");
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(File.ReadAllText(page));
HtmlDocument pdoc = new HtmlDocument();
pdoc.LoadHtml(File.ReadAllText(ppage));
var image = new NijieImage(UpdateHtmlForm.DOUJIN);
var result = nijie.ParseImage(image, ref nullMember, doc);
Assert.IsTrue(image.ImageId > 0, "Image Id not valid");
Assert.IsTrue(image.IsDoujin, "Image is not doujin");
Assert.AreEqual(6, image.MangaPages.Count, "Doujin pages count are different!");
Nijie.ParseImagePopUp(image, pdoc);
Assert.AreEqual(6, image.ImageUrls.Count, "Doujin image urls count are different!");
foreach (var item in image.MangaPages)
{
Assert.IsNotNull(item.ImageUrl, "Image url is missing!");
}
}
}
public NijieMemberViewModel(NijieMember member)
{
_member = member;
this.MemberId = member.MemberId;
this.Mode = (MemberMode)member.Mode;
}
private void ParseMemberBookmark(HtmlDocument doc, NijieMember member)
{
member.Images = new List <NijieImage>();
var bookmarkedImages = doc.DocumentNode.SelectNodes("//p[@class='nijiedao']");
if (bookmarkedImages == null || bookmarkedImages.Count == 0)
{
return;
}
foreach (var item in bookmarkedImages)
{
var div = new HtmlDocument();
div.LoadHtml(item.InnerHtml);
var imageId = div.DocumentNode.SelectSingleNode("//a").Attributes["href"].Value;
var res = re_image.Match(imageId);
if (res.Success)
{
NijieImage image = new NijieImage(Int32.Parse(res.Groups[1].Value));
image.Referer = member.MemberUrl;
var link = div.DocumentNode.SelectSingleNode("//a");
image.Title = link.Attributes["title"].Value;
var thumb = div.DocumentNode.SelectSingleNode("//a/img");
image.ThumbImageUrl = Util.FixUrl(thumb.Attributes["src"].Value, ROOT_DOMAIN, Properties.Settings.Default.UseHttps);
// check if image is friend only
// img src="//img.nijie.info/pic/common_icon/illust/friends.png"
image.IsFriendOnly = false;
if (image.ThumbImageUrl.EndsWith("friends.png"))
{
image.IsFriendOnly = true;
}
//"//img.nijie.info/pic/common_icon/illust/golden.png"
image.IsGoldenMember = false;
if (image.ThumbImageUrl.EndsWith("golden.png"))
{
image.IsGoldenMember = true;
}
// check manga icon
image.IsManga = false;
var icon = div.DocumentNode.SelectSingleNode("//div[@class='thumbnail-icon']/img");
if (icon != null)
{
if (icon.Attributes["src"].Value.EndsWith("thumbnail_comic.png"))
{
image.IsManga = true;
}
}
// check animation icon
image.IsAnimated = false;
var animeIcon = div.DocumentNode.SelectSingleNode("//div[@class='thumbnail-anime-icon']/img");
if (animeIcon != null)
{
if (animeIcon.Attributes["src"].Value.EndsWith("thumbnail_anime.png"))
{
image.IsAnimated = true;
}
}
image.BookmarkedBy = member;
member.Images.Add(image);
}
item.Remove();
}
CheckNextPageAvailable(doc, member);
var imageCountElements = doc.DocumentNode.SelectNodes("//p[@class='mem-indent float-left']/em");
member.TotalImages = ParseTotalImageCount(imageCountElements);
}
public void TestMemberParserMethod()
{
//var nijie = Nijie.GetInstance();
var member = new NijieMember()
{
MemberId = UpdateHtmlForm.MEMBER_1
};
// test member images
{
// https://nijie.info/members_illust.php?id=29353
var page = UpdateHtmlForm.PATH + "member-images.html";
Assert.IsTrue(File.Exists(page), "Test file is missing!");
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(File.ReadAllText(page));
var result = nijie.ParseMember(doc, member, MemberMode.Images);
Assert.AreEqual("mumei", result.UserName);
Assert.AreEqual(20, result.Images.Count, "Image counts differents");
Assert.AreEqual(20, result.TotalImages, "Image counts differents");
foreach (var image in result.Images)
{
Assert.IsTrue(image.ImageId > 0, "Image Id not valid");
Assert.IsNotNull(image.ThumbImageUrl, "Thumbnail image missing!");
}
}
// test member bookmarked
{
// https://nijie.info/user_like_illust_view.php?id=29353
var page = UpdateHtmlForm.PATH + "member-bookmarked-images.html";
Assert.IsTrue(File.Exists(page), "Test file is missing!");
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(File.ReadAllText(page));
var result = nijie.ParseMember(doc, member, MemberMode.Bookmark);
Assert.AreEqual(43, result.Images.Count, "Image counts differents");
Assert.AreEqual(72, result.TotalImages, "Total Image counts differents");
foreach (var image in result.Images)
{
Assert.IsTrue(image.ImageId > 0, "Image Id not valid");
Assert.IsNotNull(image.ThumbImageUrl, "Thumbnail image missing!");
}
Assert.IsTrue(result.IsNextAvailable, "Next Page should be available");
Assert.AreEqual(72, result.TotalImages, "Different image count");
}
// test member bookmarked - last page
{
// https://nijie.info/user_like_illust_view.php?p=2&id=29353
var page = UpdateHtmlForm.PATH + "member-bookmarked-images-lastpage.html";
Assert.IsTrue(File.Exists(page), "Test file is missing!");
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(File.ReadAllText(page));
var result = nijie.ParseMember(doc, member, MemberMode.Bookmark);
Assert.AreEqual(25, result.Images.Count, "Image counts differents");
foreach (var image in result.Images)
{
Assert.IsTrue(image.ImageId > 0, "Image Id not valid");
Assert.IsNotNull(image.ThumbImageUrl, "Thumbnail image missing!");
}
Assert.IsFalse(result.IsNextAvailable, "Next Page should not be available");
Assert.AreEqual(72, result.TotalImages, "Different image count");
}
// test member doujins
// need to be updated with proper member with doujin
var member3 = new NijieMember()
{
MemberId = UpdateHtmlForm.MEMBER_3
};
{
// https://nijie.info/members_dojin.php?id=251720
var page = UpdateHtmlForm.PATH + "member-doujins.html";
Assert.IsTrue(File.Exists(page), "Test file is missing!");
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(File.ReadAllText(page));
var result = nijie.ParseMember(doc, member3, MemberMode.Doujin);
Assert.AreEqual(30, result.Images.Count, "Image counts differents");
Assert.AreEqual(30, result.TotalImages, "Total Image counts differents");
foreach (var image in result.Images)
{
Assert.IsTrue(image.ImageId > 0, "Image Id not valid");
Assert.IsNotNull(image.ThumbImageUrl, "Thumbnail image missing!");
}
}
}
/// <summary>
/// Process images from member page.
/// </summary>
/// <param name="job"></param>
private void doMemberJob(JobDownloadViewModel job)
{
if (isJobCancelled(job))
{
return;
}
MainWindow.Log.Debug("Running Member Job: " + job.Name);
try
{
job.Message = "Parsing member page";
job.CurrentPage = job.StartPage;
NijieMember memberPage = null;
do
{
memberPage = MainWindow.Bot.ParseMember(job.MemberId, job.MemberMode, job.CurrentPage);
job.TotalImages = memberPage.TotalImages;
if (Properties.Settings.Default.DownloadAvatar)
{
var rootPath = Properties.Settings.Default.RootDirectory;
var avatarFilename = MainWindow.makeFilename(job, new NijieImage()
{
Member = memberPage
}, type: MainWindow.FilenameFormatType.Avatar);
downloadUrl(job, memberPage.AvatarUrl, memberPage.MemberUrl, rootPath + Path.DirectorySeparatorChar + avatarFilename);
}
foreach (var imageTemp in memberPage.Images)
{
if (isJobCancelled(job))
{
return;
}
try
{
if (job.MemberMode == MemberMode.Bookmark)
{
processImage(job, null, imageTemp);
}
else
{
processImage(job, memberPage, imageTemp);
}
}
catch (NijieException ne)
{
if (ne.ErrorCode == NijieException.DOWNLOAD_ERROR)
{
job.Exceptions.Add(ne);
continue;
}
else
{
throw;
}
}
if (job.DownloadCount > job.Limit && job.Limit != 0)
{
job.Message = "Image limit reached: " + job.Limit;
return;
}
}
job.CurrentPage++;
MainWindow.Log.Info("Moving to next page: " + job.CurrentPage);
if (job.CurrentPage > job.EndPage && job.EndPage != 0)
{
job.Message = "Page limit reached: " + job.EndPage;
return;
}
} while (memberPage != null && memberPage.IsNextAvailable);
}
catch (NijieException ne)
{
HandleJobException(job, ne);
MainWindow.Log.Error("Error when processing Member Job: " + job.Name, ne);
}
}
