public NijieImage ParseImage(NijieImage image, NijieMember member = null)
{
HtmlDocument doc = null;
try
{
canOperate();
var result = getPage(image.ViewUrl);
PrintCookie("Image Page " + image.ImageId);
doc = result.Item1;
if (Util.IsRedirected(result.Item2.ResponseUri.ToString(), image.ViewUrl, true))
{
Log.Debug(String.Format("Redirection for Image {0}: {1} ==> {2}, possibly locked.", image.ImageId, image.ViewUrl, result.Item2.ResponseUri));
image.IsFriendOnly = true;
return image;
}
return ParseImage(image, ref member, doc);
}
catch (NijieException)
{
throw;
}
catch (Exception ex)
{
if (doc != null)
{
var filename = "Dump for Image " + image.ImageId + ".html";
Log.Debug("Dumping image page to: " + filename);
doc.Save(filename);
}
throw new NijieException(String.Format("Error when processing image: {0} ==> {1}", image.ImageId, ex.Message), ex, NijieException.IMAGE_UNKNOWN_ERROR);
}
}
/// <summary>
/// Parse member from internet based on given member id, mode, and page number.
/// </summary>
/// <param name="memberId"></param>
/// <param name="mode"></param>
/// <param name="page"></param>
/// <returns></returns>
public NijieMember ParseMember(int memberId, MemberMode mode, int page)
{
HtmlDocument doc = null;
try
{
canOperate();
NijieMember member = new NijieMember(memberId, mode, page);
var result = getPage(member.MemberUrl);
var res = result.Item2;
if (Util.IsRedirected(res.ResponseUri.ToString(), member.MemberUrl, ignoreProtocol: true))
{
throw new NijieException(string.Format("Redirected to another page: {0} ==> {1}", member.MemberUrl, res.ResponseUri.ToString()), NijieException.MEMBER_REDIR);
}
doc = result.Item1;
return ParseMember(doc, member, mode);
}
catch (NijieException)
{
throw;
}
catch (Exception ex)
{
if (doc != null)
{
var filename = String.Format("Dump for Member {0} Mode {1}.html", memberId, mode.ToString());
Log.Debug("Dumping member page to: " + filename);
doc.Save(filename);
}
throw new NijieException(String.Format("Error when processing member: {0} ==> {1}", memberId, ex.Message), ex, NijieException.MEMBER_UNKNOWN_ERROR);
}
}
/// <summary>
/// Parse member from html document.
/// </summary>
/// <param name="doc"></param>
/// <param name="member"></param>
/// <param name="mode"></param>
/// <returns></returns>
public NijieMember ParseMember(HtmlDocument doc, NijieMember member, MemberMode mode)
{
ParseMemberProfile(doc, member);
if (mode == MemberMode.Images || mode == MemberMode.Doujin)
ParseMemberImages(doc, member);
else
ParseMemberBookmark(doc, member);
member.Status = String.Format("Completed, found {0} images", member.Images.Count);
return member;
}
public NijieImage ParseImage(NijieImage image, ref NijieMember member, HtmlDocument doc)
{
checkErrorMessage(doc);
var doujinDiv = doc.DocumentNode.SelectSingleNode("//div[@id='dojin_header']");
if (doujinDiv != null)
{
ProcessDoujin(image, doc);
}
else
{
if (member == null)
{
member = ParseMemberFromImage(doc);
}
image.Member = member;
ParseImageLinks(image, doc);
ParseImageTitleAndDescription(image, doc);
ParseImageTags(image, doc);
}
ParseImageExtras(image, doc);
return image;
}
private NijieMember ParseMemberFromImage(HtmlDocument doc)
{
var memberUrl = doc.DocumentNode.SelectSingleNode("//div[@id='pro']/p/a").Attributes["href"].Value;
var split = memberUrl.Split('?');
int memberId = Int32.Parse(split[1].Replace("id=", ""));
NijieMember member = new NijieMember(memberId, 0);
var profileDiv = doc.DocumentNode.SelectSingleNode("//div[@id='pro']/p/a/img");
if (profileDiv != null)
{
member.UserName = profileDiv.Attributes["alt"].Value;
member.AvatarUrl = profileDiv.Attributes["src"].Value;
}
return member;
}
public NijieMemberViewModel(NijieMember member)
{
_member = member;
this.MemberId = member.MemberId;
this.Mode = (MemberMode)member.Mode;
}
public void GetMember(SynchronizationContext context)
{
try
{
_member = MainWindow.Bot.ParseMember(this.MemberId, this.Mode, this.Page);
this.UserName = _member.UserName;
ImageLoader.LoadImage(_member.AvatarUrl, _member.MemberUrl,
new Action<BitmapImage, string>((image, status) =>
{
this.AvatarImage = null;
this.AvatarImage = image;
_avatarImageStatus = status;
}
));
if (_member.Images != null)
{
Images = new ObservableCollection<NijieImageViewModel>();
foreach (var image in _member.Images)
{
var temp = new NijieImageViewModel(image);
context.Send((x) =>
{
Images.Add(temp);
}, null);
}
this.Status = String.Format("Loaded: {0} images.", _member.Images.Count);
onPropertyChanged("TotalImages");
this.HasError = false;
}
}
catch (NijieException ne)
{
MainWindow.Log.Error(ne.Message, ne);
this.UserName = null;
this.AvatarImage = ViewModelHelper.NoAvatar;
context.Send((x) =>
{
if (Images != null)
{
Images.Clear();
Images = null;
}
}, null);
this.HasError = true;
this.Status = "[Error] " + ne.Message;
}
}
public Tuple<List<NijieMember>, bool> ParseMyMemberBookmark(int page)
{
canOperate();
List<NijieMember> members = new List<NijieMember>();
HtmlDocument doc = null;
bool isNextPageAvailable = false;
try
{
var url = Util.FixUrl(String.Format("//nijie.info/like_my.php?p={0}", page), ROOT_DOMAIN, Properties.Settings.Default.UseHttps);
var result = getPage(url);
doc = result.Item1;
var membersDiv = doc.DocumentNode.SelectNodes("//div[@class='nijie-okini']");
if (membersDiv != null)
{
foreach (var memberDiv in membersDiv)
{
var memberUrl = memberDiv.SelectSingleNode("//div[@class='nijie-okini']//a").Attributes["href"].Value;
var res = re_member.Match(memberUrl);
if (res.Success)
{
var member = new NijieMember(Int32.Parse(res.Groups[1].Value), 0);
member.UserName = memberDiv.SelectSingleNode("//div[@class='nijie-okini']//p[@class='sougo']").InnerText;
member.AvatarUrl = memberDiv.SelectSingleNode("//div[@class='nijie-okini']//a//img").Attributes["src"].Value;
members.Add(member);
}
memberDiv.Remove();
}
}
var nextPageButton = doc.DocumentNode.SelectNodes("//p[@class='page_button']//a");
if (nextPageButton != null)
{
foreach (var item in nextPageButton)
{
if (item.InnerText.StartsWith("次へ"))
{
isNextPageAvailable = true;
break;
}
}
}
}
catch (Exception ex)
{
if (doc != null)
{
var filename = "Dump for My Member Bookmark.html";
Log.Debug("Dumping My Member Bookmark page to: " + filename);
doc.Save(filename);
}
throw new NijieException(String.Format("Error when processing my member bookmark ==> {0}", ex.Message), ex, NijieException.IMAGE_UNKNOWN_ERROR);
}
return new Tuple<List<NijieMember>, bool>(members, isNextPageAvailable);
}
/// <summary>
/// Parse the image page
/// - Illustration
/// - Manga
/// </summary>
/// <param name="job"></param>
/// <param name="memberPage"></param>
/// <param name="imageTemp"></param>
private void processImage(JobDownloadViewModel job, NijieMember memberPage, NijieImage imageTemp)
{
if (isJobCancelled(job))
return;
MainWindow.Log.Debug("Processing Image:" + imageTemp.ImageId);
// skip if exists in DB
if (Properties.Settings.Default.SkipIfExistsInDB && !NijieDownloader.Library.Properties.Settings.Default.Overwrite)
{
using (var dao = new NijieContext())
{
var result = NijieImage.IsDownloadedInDB(imageTemp.ImageId);
if (result)
{
job.Message = String.Format("Image {0} already downloaded in DB", imageTemp.ImageId);
job.SkipCount++;
return;
}
}
}
var image = MainWindow.Bot.ParseImage(imageTemp, memberPage);
if (image.IsFriendOnly)
{
// sample: 74587
job.Message = "Image locked!";
return;
}
if (image.IsGoldenMember)
{
job.Message = "Image only for Gold Membership";
return;
}
if (image.IsManga)
{
processManga(job, image);
}
else
{
processIllustration(job, image);
}
}
public void TestMemberParserMethod()
{
var nijie = Nijie.GetInstance();
var member = new NijieMember() { MemberId = UpdateHtmlForm.MEMBER_1 };
// test member images
{
// https://nijie.info/members_illust.php?id=29353
var page = UpdateHtmlForm.PATH + "member-images.html";
Assert.IsTrue(File.Exists(page), "Test file is missing!");
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(File.ReadAllText(page));
var result = nijie.ParseMember(doc, member, MemberMode.Images);
Assert.AreEqual("mumei", result.UserName);
Assert.AreEqual(20, result.Images.Count, "Image counts differents");
foreach (var image in result.Images)
{
Assert.IsTrue(image.ImageId > 0, "Image Id not valid");
Assert.IsNotNull(image.ThumbImageUrl, "Thumbnail image missing!");
}
}
// test member bookmarked
{
// https://nijie.info/user_like_illust_view.php?id=29353
var page = UpdateHtmlForm.PATH + "member-bookmarked-images.html";
Assert.IsTrue(File.Exists(page), "Test file is missing!");
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(File.ReadAllText(page));
var result = nijie.ParseMember(doc, member, MemberMode.Bookmark);
Assert.AreEqual(44, result.Images.Count, "Image counts differents");
foreach (var image in result.Images)
{
Assert.IsTrue(image.ImageId > 0, "Image Id not valid");
Assert.IsNotNull(image.ThumbImageUrl, "Thumbnail image missing!");
}
Assert.IsTrue(result.IsNextAvailable, "Next Page should be available");
Assert.AreEqual(72, result.TotalImages, "Different image count");
}
// test member bookmarked - last page
{
// https://nijie.info/user_like_illust_view.php?p=2&id=29353
var page = UpdateHtmlForm.PATH + "member-bookmarked-images-lastpage.html";
Assert.IsTrue(File.Exists(page), "Test file is missing!");
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(File.ReadAllText(page));
var result = nijie.ParseMember(doc, member, MemberMode.Bookmark);
Assert.AreEqual(25, result.Images.Count, "Image counts differents");
foreach (var image in result.Images)
{
Assert.IsTrue(image.ImageId > 0, "Image Id not valid");
Assert.IsNotNull(image.ThumbImageUrl, "Thumbnail image missing!");
}
Assert.IsFalse(result.IsNextAvailable, "Next Page should not be available");
Assert.AreEqual(72, result.TotalImages, "Different image count");
}
// test member doujins
// need to be updated with proper member with doujin
var member2 = new NijieMember() { MemberId = UpdateHtmlForm.MEMBER_2 };
{
// https://nijie.info/members_dojin.php?id=251720
var page = UpdateHtmlForm.PATH + "member-doujins.html";
Assert.IsTrue(File.Exists(page), "Test file is missing!");
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(File.ReadAllText(page));
var result = nijie.ParseMember(doc, member2, MemberMode.Doujin);
Assert.AreEqual(0, result.Images.Count, "Image counts differents");
foreach (var image in result.Images)
{
Assert.IsTrue(image.ImageId > 0, "Image Id not valid");
Assert.IsNotNull(image.ThumbImageUrl, "Thumbnail image missing!");
}
}
}
private void ParseMemberProfile(HtmlDocument doc, NijieMember member)
{
var profileDiv = doc.DocumentNode.SelectSingleNode("//div[@id='pro']/p/a/img");
if (profileDiv != null)
{
member.UserName = profileDiv.Attributes["alt"].Value;
member.AvatarUrl = profileDiv.Attributes["src"].Value;
}
}
private void ParseMemberImages(HtmlDocument doc, NijieMember member)
{
var imagesDivs = doc.DocumentNode.SelectNodes("//div[@id='main-left-none']/div");
var imagesDiv = "";
foreach (var item in imagesDivs)
{
if(item.InnerHtml.Contains("class=\"nijie\""))
{
imagesDiv = item.InnerHtml;
break;
}
}
member.Images = ParseImageList(imagesDiv, member.MemberUrl);
foreach (var image in member.Images)
{
image.Member = member;
}
var imageCountElements = doc.DocumentNode.SelectNodes("//p[@class='mem-indent float-left']/em");
member.TotalImages = ParseTotalImageCount(imageCountElements);
}
private void ParseMemberBookmark(HtmlDocument doc, NijieMember member)
{
member.Images = new List<NijieImage>();
var bookmarkedImages = doc.DocumentNode.SelectNodes("//p[@class='nijiedao']");
if (bookmarkedImages == null || bookmarkedImages.Count == 0)
{
return;
}
foreach (var item in bookmarkedImages)
{
var div = new HtmlDocument();
div.LoadHtml(item.InnerHtml);
var imageId = div.DocumentNode.SelectSingleNode("//a").Attributes["href"].Value;
var res = re_image.Match(imageId);
if (res.Success)
{
NijieImage image = new NijieImage(Int32.Parse(res.Groups[1].Value));
image.Referer = member.MemberUrl;
var link = div.DocumentNode.SelectSingleNode("//a");
image.Title = link.Attributes["title"].Value;
var thumb = div.DocumentNode.SelectSingleNode("//a/img");
image.ThumbImageUrl = thumb.Attributes["src"].Value;
// check if image is friend only
// img src="//img.nijie.info/pic/common_icon/illust/friends.png"
image.IsFriendOnly = false;
if (image.ThumbImageUrl.EndsWith("friends.png"))
{
image.IsFriendOnly = true;
}
//"//img.nijie.info/pic/common_icon/illust/golden.png"
image.IsGoldenMember = false;
if (image.ThumbImageUrl.EndsWith("golden.png"))
{
image.IsGoldenMember = true;
}
// check manga icon
image.IsManga = false;
var icon = div.DocumentNode.SelectSingleNode("//div[@class='thumbnail-icon']/img");
if (icon != null)
{
if (icon.Attributes["src"].Value.EndsWith("thumbnail_comic.png"))
image.IsManga = true;
}
// check animation icon
image.IsAnimated = false;
var animeIcon = div.DocumentNode.SelectSingleNode("//div[@class='thumbnail-anime-icon']/img");
if (animeIcon != null)
{
if (animeIcon.Attributes["src"].Value.EndsWith("thumbnail_anime.png"))
image.IsAnimated = true;
}
image.BookmarkedBy = member;
member.Images.Add(image);
}
item.Remove();
}
// check next page
member.IsNextAvailable = false;
var navButtons = doc.DocumentNode.SelectNodes("//p[@class='page_button']/a");
if (navButtons != null)
{
foreach (var item in navButtons)
{
if (item.InnerText.StartsWith("次へ"))
{
member.IsNextAvailable = true;
break;
}
}
}
var imageCountElements = doc.DocumentNode.SelectNodes("//p[@class='mem-indent float-left']/em");
member.TotalImages = ParseTotalImageCount(imageCountElements);
}