public static bool CheckBlacklistedTag(DanbooruPost post, DanbooruPostDaoOption option) { if (option.BlacklistedTagsUseRegex) { if (option.IsBlacklistOnlyForGeneral) { return post.TagsEntity.Any(x => x.Type == DanbooruTagType.General && option.BlacklistedTagsRegex.IsMatch(x.Name)); } else return post.TagsEntity.Any(x => option.BlacklistedTagsRegex.IsMatch(x.Name)); } else { if (option.IsBlacklistOnlyForGeneral) { foreach (var tag in option.BlacklistedTags) { if (post.TagsEntity.Any(x => x.Type == DanbooruTagType.General && x.Name.Equals(tag.Name, StringComparison.InvariantCultureIgnoreCase))) return true; } } else { foreach (var tag in option.BlacklistedTags) { if (post.TagsEntity.Any(x => x.Name.Equals(tag.Name, StringComparison.InvariantCultureIgnoreCase))) return true; } } } return false; }
public static DanbooruPost ParsePost(DanbooruPost post, string postHtml) { HtmlDocument doc = new HtmlDocument(); if (String.IsNullOrEmpty(postHtml)) throw new Exception("No post html!"); doc.LoadHtml(postHtml); string file_url = ""; string sample_url = ""; // Flash Game or bmp // TODO: need to change the preview url if (post.PreviewUrl == "http://chan.sankakucomplex.com/download-preview.png") { var links = doc.DocumentNode.SelectNodes("//a"); foreach (var link in links) { // flash if (link.InnerText == "Save this flash (right click and save)") { file_url = link.GetAttributeValue("href", ""); break; } // bmp if (link.InnerText == "Download") { file_url = link.GetAttributeValue("href", ""); break; } } } else { var image = doc.DocumentNode.SelectSingleNode("//img[@id='image']"); if (image != null) { sample_url = image.GetAttributeValue("src", ""); } var links = doc.DocumentNode.SelectNodes("//a"); foreach (var link in links) { if (link.InnerText == "Original image") { file_url = link.GetAttributeValue("href", ""); break; } } } post.FileUrl = file_url; if (!string.IsNullOrWhiteSpace(file_url) && string.IsNullOrWhiteSpace(sample_url)) sample_url = file_url; post.SampleUrl = sample_url; return post; }
private static bool IsBlacklisted(DanbooruPost post, List<DanbooruTag> TagBlackList) { if (TagBlackList != null) { foreach (var tag in TagBlackList) { return post.TagsEntity.Any(x => x.Name.Equals(tag.Name, StringComparison.InvariantCultureIgnoreCase)); } } return false; }
private static void PostProcess(DanbooruPostDaoOption option, DanbooruPost post) { post.TagsEntity = Helper.ParseTags(post.Tags, option.Provider); if (option.BlacklistedTagsUseRegex) { post.Hidden = IsBlacklisted(post, option.BlacklistedTagsRegex); } else { post.Hidden = IsBlacklisted(post, option.BlacklistedTags); } post.Query = option.Query; post.SearchTags = option.SearchTags; post.Provider = option.Provider; }
private static void ReadRssMethod2(DanbooruPostDaoOption option, BindingList<DanbooruPost> posts, string xmldoc) { using (StringReader strReader = new StringReader(xmldoc)) { using (XmlReader reader = new XmlTextReader(strReader)) { XDocument doc = XDocument.Load(reader); var feeds = doc.Descendants("item"); XNamespace dc = "http://purl.org/dc/elements/1.1/"; foreach (var item in feeds) { DanbooruPost post = new DanbooruPost(); var titleData = item.Element("title").Value.Split(new char[] { '-' }, 2); post.Id = titleData[0].Trim(); post.Tags = titleData[1].Trim(); post.Referer = AppendHttp(item.Element("link").Value, option.Provider); post.CreatedAt = item.Element("pubDate").Value; post.CreatorId = item.Element(dc + "creator").Value; post.FileUrl = item.Element("enclosure").Attribute("url").Value; try { HtmlAgilityPack.HtmlDocument description = new HtmlAgilityPack.HtmlDocument(); description.LoadHtml(HttpUtility.HtmlDecode(item.Element("description").Value)); //post.Width = Convert.ToInt32(matches.Groups[1].Value); //post.Height = Convert.ToInt32(matches.Groups[2].Value); var img = description.DocumentNode.SelectSingleNode("//img"); post.PreviewWidth = Convert.ToInt32(img.Attributes["width"].Value); post.PreviewHeight = Convert.ToInt32(img.Attributes["height"].Value); post.PreviewUrl = img.Attributes["src"].Value; } catch (Exception) { } PostProcess(option, post); posts.Add(post); } } } }
/// <summary> /// For constructing TAGS_FILENAME. /// </summary> /// <param name="baseFolder"></param> /// <param name="format"></param> /// <param name="post"></param> /// <param name="limit"></param> /// <returns></returns> public static string MakeFilename(DanbooruFilenameFormat format, DanbooruPost post) { string filename = format.FilenameFormat; string provider = post.Provider.Name; string query = post.Query; string searchTags = post.SearchTags; string originalFileName = post.FileUrl.Split('/').Last(); // sanitizing the format filename = Helper.SanitizeFilename(filename, true); //remove extension originalFileName = originalFileName.Substring(0, originalFileName.LastIndexOf('.')); originalFileName = Uri.UnescapeDataString(originalFileName); filename = filename.Replace("%provider%", Helper.SanitizeFilename(provider)); filename = filename.Replace("%id%", post.Id); filename = filename.Replace("%rating%", post.Rating); filename = filename.Replace("%md5%", post.MD5); filename = filename.Replace("%searchParam%", Helper.SanitizeFilename(query)); filename = filename.Replace("%searchtag%", Helper.SanitizeFilename(searchTags)); filename = filename.Replace("%originalFilename%", Helper.SanitizeFilename(originalFileName)); // copy the tags entity to be grouped. var groupedTags = post.TagsEntity; groupedTags.Sort(); // remove ignored tags groupedTags = RemoveIgnoredTags(format, groupedTags); // artist var artist = FilterTags(post, groupedTags, DanbooruTagType.Artist, format.ArtistGroupLimit, format.ArtistGroupReplacement, format.MissingTagReplacement, format.IsReplaceMode); var artistStr = Helper.SanitizeFilename(artist).Trim(); if (String.IsNullOrEmpty(artistStr)) artistStr = DanbooruDownloader3.Properties.Settings.Default.tagNoArtistValue; filename = filename.Replace("%artist%", artistStr); // copyright var copyright = FilterTags(post, groupedTags, DanbooruTagType.Copyright, format.CopyrightGroupLimit, format.CopyrightGroupReplacement, format.MissingTagReplacement, format.IsReplaceMode); var copyStr = Helper.SanitizeFilename(copyright.Trim()); if (String.IsNullOrEmpty(copyStr)) copyStr = DanbooruDownloader3.Properties.Settings.Default.tagNoCopyrightValue; filename = filename.Replace("%copyright%", copyStr); // character var character = FilterTags(post, groupedTags, DanbooruTagType.Character, format.CharacterGroupLimit, format.CharacterGroupReplacement, format.MissingTagReplacement, format.IsReplaceMode); var charaStr = Helper.SanitizeFilename(character.Trim()); if (String.IsNullOrEmpty(charaStr)) charaStr = DanbooruDownloader3.Properties.Settings.Default.tagNoCharaValue; filename = filename.Replace("%character%", charaStr); // cirle var circleSelection = post.TagsEntity.Where<DanbooruTag>(x => x.Type == DanbooruTagType.Circle).Select(x => x.Name); var circle = FilterTags(post, groupedTags, DanbooruTagType.Circle, format.CircleGroupLimit, format.CircleGroupReplacement, format.MissingTagReplacement, format.IsReplaceMode); var circleStr = Helper.SanitizeFilename(circle.Trim()); if (String.IsNullOrEmpty(circleStr)) circleStr = DanbooruDownloader3.Properties.Settings.Default.tagNoCircleValue; filename = filename.Replace("%circle%", circleStr); // faults var faultsSelection = post.TagsEntity.Where<DanbooruTag>(x => x.Type == DanbooruTagType.Faults).Select(x => x.Name); var faults = FilterTags(post, groupedTags, DanbooruTagType.Faults, format.FaultsGroupLimit, format.FaultsGroupReplacement, format.MissingTagReplacement, format.IsReplaceMode); var faultStr = Helper.SanitizeFilename(faults.Trim()); if (String.IsNullOrEmpty(faultStr)) faultStr = DanbooruDownloader3.Properties.Settings.Default.tagNoFaultValue; filename = filename.Replace("%faults%", faultStr); // all tags filename = filename.Replace("%tags%", Helper.SanitizeFilename(string.Join(" ", groupedTags.Select(x => x.Name)))); // append base folder from Save Folder text box if (format.BaseFolder.EndsWith(@"\")) filename = format.BaseFolder + filename; else if (!String.IsNullOrWhiteSpace(format.BaseFolder)) filename = format.BaseFolder + @"\" + filename; filename = filename.Substring(0, filename.Length < format.Limit ? filename.Length : format.Limit).Trim(); // check if contains subdirectory if (filename.Contains(@"\")) { string dir = filename.Substring(0, filename.LastIndexOf(@"\")); if (!Directory.Exists(dir)) Directory.CreateDirectory(dir); } return filename; }
public BindingList<DanbooruPost> Parse(string data, DanbooruSearchParam query) { BindingList<DanbooruPost> list = new BindingList<DanbooruPost>(); XDocument posts = XDocument.Parse(data); this.RawData = posts.ToString(); Success = true; var responses = posts.Descendants("response"); if (responses != null && responses.Count() > 0) { ResponseMessage = responses.First().Attribute("reason").Value.ToString(); Success = Convert.ToBoolean(responses.First().Attribute("success").Value); if (!Success) { return null; } } this.TotalPost = Convert.ToInt32(posts.Root.Attribute("count").Value); this.Offset = Convert.ToInt32(posts.Root.Attribute("offset").Value); string queryStr = GenerateQueryString(query); foreach (var post in posts.Descendants("post")) { DanbooruPost p = new DanbooruPost(); p.Id = post.Attribute("id").Value.ToString(); p.Tags = post.Attribute("tags").Value.ToString(); p.TagsEntity = Helper.ParseTags(p.Tags, SearchParam.Provider); p.Source = post.Attribute("source").Value.ToString(); p.Score = post.Attribute("score").Value.ToString(); p.Rating = post.Attribute("rating").Value.ToString(); p.FileUrl = AppendHttp(post.Attribute("file_url").Value.ToString(), query.Provider); p.Width = Convert.ToInt32(post.Attribute("width").Value); p.Height = Convert.ToInt32(post.Attribute("height").Value); p.PreviewUrl = AppendHttp(post.Attribute("preview_url").Value.ToString(), query.Provider); if (post.Attribute("actual_preview_width") != null && // moebooru extension post.Attribute("actual_preview_height") != null) { p.PreviewWidth = Convert.ToInt32(post.Attribute("actual_preview_width").Value); p.PreviewHeight = Convert.ToInt32(post.Attribute("actual_preview_height").Value); } else { p.PreviewWidth = Convert.ToInt32(post.Attribute("preview_width").Value); p.PreviewHeight = Convert.ToInt32(post.Attribute("preview_height").Value); } p.SampleUrl = AppendHttp(post.Attribute("sample_url").Value.ToString(), query.Provider); p.SampleWidth = Convert.ToInt32(post.Attribute("sample_width").Value); p.SampleHeight = Convert.ToInt32(post.Attribute("sample_height").Value); // moebooru extension p.JpegUrl = AppendHttp(post.Attribute("jpeg_url").Value.ToString(), query.Provider); p.JpegWidth = Convert.ToInt32(post.Attribute("jpeg_width").Value); p.JpegHeight = Convert.ToInt32(post.Attribute("jpeg_height").Value); p.Filesize = Convert.ToInt32(post.Attribute("file_size").Value); p.Status = post.Attribute("status").Value.ToString(); p.HasChildren = Convert.ToBoolean(post.Attribute("has_children").Value); p.ParentId = post.Attribute("parent_id").Value.ToString(); p.Change = post.Attribute("change").Value.ToString(); p.CreatorId = post.Attribute("creator_id").Value.ToString(); p.CreatedAt = post.Attribute("created_at").Value.ToString(); p.MD5 = post.Attribute("md5").Value.ToString(); p.Provider = query.Provider; p.Query = queryStr; p.SearchTags = query.Tag; p.Referer = query.Provider.Url + @"/post/show/" + p.Id; list.Add(p); } return list; }
public static DanbooruPost ParsePost(DanbooruPost post, string postHtml, bool overideTagParsing) { try { HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(postHtml); string file_url = ""; string sample_url = ""; // reparse tags with type if (overideTagParsing) { ReparseTags(post, doc); } // Flash Game or bmp if (post.PreviewUrl.EndsWith("download-preview.png")) { var links = doc.DocumentNode.SelectNodes("//a"); foreach (var link in links) { // flash if (link.InnerText == "Save this file (right click and save as)") { file_url = Helper.FixUrl(link.GetAttributeValue("href", ""), isHttps(post.Provider)); // http://cs.sankakucomplex.com/data/f6/23/f623ea7559ef39d96ebb0ca7530586b8.swf post.MD5 = file_url.Substring(file_url.LastIndexOf("/") + 1); post.MD5 = post.MD5.Substring(0, 32); break; } // bmp if (link.InnerText == "Download") { file_url = Helper.FixUrl(link.GetAttributeValue("href", ""), isHttps(post.Provider)); break; } } } else { var lowresElement = doc.DocumentNode.SelectSingleNode("//a[@id='lowres']"); if (lowresElement != null) { sample_url = Helper.FixUrl(lowresElement.GetAttributeValue("href", ""), isHttps(post.Provider)); } var highresElement = doc.DocumentNode.SelectSingleNode("//a[@id='highres']"); if (highresElement != null) { file_url = Helper.FixUrl(highresElement.GetAttributeValue("href", ""), isHttps(post.Provider)); } } post.FileUrl = file_url; if (!string.IsNullOrWhiteSpace(file_url) && string.IsNullOrWhiteSpace(sample_url)) sample_url = file_url; post.SampleUrl = sample_url; return post; } catch (Exception ex) { string filename = "Dump for Post " + post.Id + post.Provider.Name + " Query " + post.Query + ".txt"; bool result = Helper.DumpRawData(postHtml, filename); if (!result) Program.Logger.Error("Failed to dump rawdata to: " + filename, ex); throw; } }
private void ParsePostAttributes(XmlTextReader reader, DanbooruPost post) { while (reader.MoveToNextAttribute()) { switch (reader.Name.ToLowerInvariant()) { case "id": post.Id = reader.Value; break; case "tags": post.Tags = reader.Value; post.TagsEntity = Helper.ParseTags(post.Tags, Option.Provider); break; case "source": post.Source = reader.Value; break; case "creator_id": post.CreatorId = reader.Value; break; case "file_url": post.FileUrl = AppendHttp(reader.Value); break; case "width": post.Width = -1; try { post.Width = Int32.Parse(reader.Value); } catch (Exception) { #if DEBUG throw; #endif } break; case "height": post.Height = -1; try { post.Height = Int32.Parse(reader.Value); } catch (Exception) { #if DEBUG throw; #endif } break; case "change": post.Change = reader.Value; break; case "score": post.Score = reader.Value; break; case "rating": post.Rating = reader.Value; break; case "status": post.Status = reader.Value; break; case "has_children": int num; bool isNum = Int32.TryParse(reader.Value, out num); if (isNum) { post.HasChildren = Convert.ToBoolean(num); } else { post.HasChildren = Convert.ToBoolean(reader.Value); } break; case "created_at": post.CreatedAt = reader.Value; post.CreatedAtDateTime = ParseDateTime(post.CreatedAt, Option.Provider); break; case "md5": post.MD5 = reader.Value; break; case "preview_url": post.PreviewUrl = AppendHttp(reader.Value); break; case "preview_width": post.PreviewWidth = -1; try { post.PreviewWidth = Int32.Parse(reader.Value); } catch (Exception) { #if DEBUG throw; #endif } break; case "preview_height": post.PreviewHeight = -1; try { post.PreviewHeight = Int32.Parse(reader.Value); } catch (Exception) { #if DEBUG throw; #endif } break; case "parent_id": post.ParentId = reader.Value; break; case "sample_url": post.SampleUrl = AppendHttp(reader.Value); break; case "sample_width": post.SampleWidth = -1; try { post.SampleWidth = Int32.Parse(reader.Value); } catch (Exception) { #if DEBUG throw; #endif } break; case "sample_height": post.SampleHeight = -1; try { post.SampleHeight = Int32.Parse(reader.Value); } catch (Exception) { #if DEBUG throw; #endif } break; case "jpeg_url": post.JpegUrl = AppendHttp(reader.Value); break; case "jpeg_width": post.JpegWidth = -1; try { post.JpegWidth = Int32.Parse(reader.Value); } catch (Exception) { #if DEBUG throw; #endif } break; case "jpeg_height": post.JpegHeight = -1; try { post.JpegHeight = Int32.Parse(reader.Value); } catch (Exception) { #if DEBUG throw; #endif } break; } } }
/// <summary> /// For constructing TAGS_FILENAME. /// </summary> /// <param name="baseFolder"></param> /// <param name="format"></param> /// <param name="post"></param> /// <param name="limit"></param> /// <returns></returns> public static string MakeFilename(DanbooruFilenameFormat format, DanbooruPost post) { string filename = format.FilenameFormat; string provider = post.Provider.Name; string query = post.Query; string searchTags = post.SearchTags; string originalFileName = post.FileUrl.Split('/').Last(); // sanitizing the format filename = Helper.SanitizeFilename(filename, true); //remove extension originalFileName = originalFileName.Substring(0, originalFileName.LastIndexOf('.')); originalFileName = Uri.UnescapeDataString(originalFileName); filename = filename.Replace("%provider%", Helper.SanitizeFilename(provider)); filename = filename.Replace("%id%", post.Id); filename = filename.Replace("%rating%", post.Rating); filename = filename.Replace("%md5%", post.MD5); filename = filename.Replace("%searchParam%", Helper.SanitizeFilename(query)); filename = filename.Replace("%searchtag%", Helper.SanitizeFilename(searchTags)); filename = filename.Replace("%originalFilename%", Helper.SanitizeFilename(originalFileName)); string dtFormat = Properties.Settings.Default.DateTimeFormat; if (String.IsNullOrWhiteSpace(dtFormat)) dtFormat = "yyyy-mm-dd HH:mm:ss"; filename = filename.Replace("%uploadDateTime%", Helper.SanitizeFilename(post.CreatedAtDateTime.ToString(dtFormat))); // copy the tags entity to be grouped. var groupedTags = post.TagsEntity; // custom sort to prioritize some tags based on file definition // Issue #46 and #81 // regex support if (File.Exists(PRIORITY_TAGS)) { var priorityTags = ReadTagsFromTextFile(PRIORITY_TAGS); groupedTags.Sort((a, b) => { var containA = priorityTags.Exists(x => Regex.IsMatch(a.Name, x.Name)); var containB = priorityTags.Exists(x => Regex.IsMatch(b.Name, x.Name)); if (containA && !containB) return -1; if (!containA && containB) return 1; else return a.CompareTo(b); }); } else { groupedTags.Sort(); } // remove ignored tags groupedTags = RemoveIgnoredTags(format, groupedTags); // artist var artist = FilterTags(post, groupedTags, DanbooruTagType.Artist, format.ArtistGroupLimit, format.ArtistGroupReplacement, format.MissingTagReplacement, format.IsReplaceMode, format.TagReplaceUnderscoreToSpace); var artistStr = Helper.SanitizeFilename(artist).Trim(); filename = filename.Replace("%artist%", artistStr); // copyright var copyright = FilterTags(post, groupedTags, DanbooruTagType.Copyright, format.CopyrightGroupLimit, format.CopyrightGroupReplacement, format.MissingTagReplacement, format.IsReplaceMode, format.TagReplaceUnderscoreToSpace); var copyStr = Helper.SanitizeFilename(copyright.Trim()); filename = filename.Replace("%copyright%", copyStr); // character var character = FilterTags(post, groupedTags, DanbooruTagType.Character, format.CharacterGroupLimit, format.CharacterGroupReplacement, format.MissingTagReplacement, format.IsReplaceMode, format.TagReplaceUnderscoreToSpace); var charaStr = Helper.SanitizeFilename(character.Trim()); filename = filename.Replace("%character%", charaStr); // cirle var circle = FilterTags(post, groupedTags, DanbooruTagType.Circle, format.CircleGroupLimit, format.CircleGroupReplacement, format.MissingTagReplacement, format.IsReplaceMode, format.TagReplaceUnderscoreToSpace); var circleStr = Helper.SanitizeFilename(circle.Trim()); filename = filename.Replace("%circle%", circleStr); // faults var faults = FilterTags(post, groupedTags, DanbooruTagType.Faults, format.FaultsGroupLimit, format.FaultsGroupReplacement, format.MissingTagReplacement, format.IsReplaceMode, format.TagReplaceUnderscoreToSpace); var faultStr = Helper.SanitizeFilename(faults.Trim()); filename = filename.Replace("%faults%", faultStr); // general var general = FilterTags(post, groupedTags, DanbooruTagType.General, 0, "", "", format.IsReplaceMode, format.TagReplaceUnderscoreToSpace); var generalStr = Helper.SanitizeFilename(general.Trim()); filename = filename.Replace("%general%", generalStr); // all tags var allTempTags = groupedTags.Select(x => x.Name).ToList(); if (format.TagReplaceUnderscoreToSpace) { for (int i = 0; i < allTempTags.Count; i++) { allTempTags[i] = allTempTags[i].Replace("_", " ").Trim(); } } filename = filename.Replace("%tags%", Helper.SanitizeFilename(string.Join(" ", allTempTags))); // append base folder from Save Folder text box if (format.BaseFolder.EndsWith(@"\")) filename = format.BaseFolder + filename; else if (!String.IsNullOrWhiteSpace(format.BaseFolder)) filename = format.BaseFolder + @"\" + filename; filename = filename.Substring(0, filename.Length < format.Limit ? filename.Length : format.Limit).Trim(); return filename; }
private void ProcessXML(string rawData) { RawData = rawData; using (StringReader strReader = new StringReader(rawData)) { using (XmlTextReader reader = new XmlTextReader(strReader)) { if (Option.Provider.BoardType == BoardType.Shimmie2) { posts = Engine.ShimmieEngine.ParseRSS(rawData, Option); } else { while (reader.Read()) { switch (reader.NodeType) { case XmlNodeType.Element: // The node is an element. var nodeName = reader.Name.ToLowerInvariant(); if (nodeName.Equals("posts")) { while (reader.MoveToNextAttribute()) { if (reader.Name.ToLowerInvariant().Equals("count")) // Posts Count { postCount = int.Parse(reader.Value); } else if (reader.Name.ToLowerInvariant().Equals("offset")) // Post Offset { offset = int.Parse(reader.Value); } } } else if (nodeName.Equals("response")) { Success = true; while (reader.MoveToNextAttribute()) { if (reader.Name.ToLowerInvariant().Equals("reason")) // Posts Count { ResponseMessage = reader.Value; } else if (reader.Name.ToLowerInvariant().Equals("success")) // Posts Count { Success = bool.Parse(reader.Value); } } } else if (nodeName.Equals("post")) { DanbooruPost post = new DanbooruPost(); while (reader.MoveToNextAttribute()) { switch (reader.Name.ToLowerInvariant()) { case "id": post.Id = reader.Value; break; case "tags": post.Tags = reader.Value; post.TagsEntity = Helper.ParseTags(post.Tags, Option.Provider); break; case "source": post.Source = reader.Value; break; case "creator_id": post.CreatorId = reader.Value; break; case "file_url": post.FileUrl = AppendHttp(reader.Value); break; case "width": post.Width = -1; try { post.Width = Int32.Parse(reader.Value); } catch (Exception) { if (FormMain.Debug) throw; } break; case "height": post.Height = -1; try { post.Height = Int32.Parse(reader.Value); } catch (Exception) { if (FormMain.Debug) throw; } break; case "change": post.Change = reader.Value; break; case "score": post.Score = reader.Value; break; case "rating": post.Rating = reader.Value; break; case "status": post.Status = reader.Value; break; case "has_children": post.HasChildren = Boolean.Parse(reader.Value); break; case "created_at": post.CreatedAt = reader.Value; break; case "md5": post.MD5 = reader.Value; break; case "preview_url": post.PreviewUrl = AppendHttp(reader.Value); break; case "preview_width": post.PreviewWidth = -1; try { post.PreviewWidth = Int32.Parse(reader.Value); } catch (Exception) { if (FormMain.Debug) throw; } break; case "preview_height": post.PreviewHeight = -1; try { post.PreviewHeight = Int32.Parse(reader.Value); } catch (Exception) { if (FormMain.Debug) throw; } break; case "parent_id": post.ParentId = reader.Value; break; case "sample_url": post.SampleUrl = AppendHttp(reader.Value); break; case "sample_width": post.SampleWidth = -1; try { post.SampleWidth = Int32.Parse(reader.Value); } catch (Exception) { if (FormMain.Debug) throw; } break; case "sample_height": post.SampleHeight = -1; try { post.SampleHeight = Int32.Parse(reader.Value); } catch (Exception) { if (FormMain.Debug) throw; } break; case "jpeg_url": post.JpegUrl = AppendHttp(reader.Value); break; case "jpeg_width": post.JpegWidth = -1; try { post.JpegWidth = Int32.Parse(reader.Value); } catch (Exception) { if (FormMain.Debug) throw; } break; case "jpeg_height": post.JpegHeight = -1; try { post.JpegHeight = Int32.Parse(reader.Value); } catch (Exception) { if (FormMain.Debug) throw; } break; } } post.Hidden = Helper.CheckBlacklistedTag(post, Option); post.Provider = Option.Provider; post.Query = Option.Query; post.SearchTags = Option.SearchTags; if (Option.Provider.BoardType == BoardType.Danbooru || Option.Provider.BoardType == BoardType.Shimmie2) { post.Referer = Option.Provider.Url + @"/post/show/" + post.Id; } else if (Option.Provider.BoardType == BoardType.Gelbooru) { post.Referer = Option.Provider.Url + @"/index.php?page=post&s=view&id=" + post.Id; } posts.Add(post); actualCount++; } break; case XmlNodeType.EndElement: //Display the end of the element. //txtResult.AppendText("END"); break; } } } } } }
/// <summary> /// Get File Url for batch job /// </summary> /// <param name="_clientPost"></param> /// <param name="post"></param> private void ResolveFileUrlBatch(ExtendedWebClient _clientPost, DanbooruPost post) { if (!string.IsNullOrWhiteSpace(post.Referer)) { UpdateLog("DoBatchJob", "Getting file_url from " + post.Referer); int currRetry = 0; int maxRetry = Convert.ToInt32(txtRetry.Text); int delay = Convert.ToInt32(txtDelay.Text); while (currRetry < maxRetry) { try { string html = _clientPost.DownloadString(post.Referer); _clientPost.Timeout = Convert.ToInt32(txtTimeout.Text); if (post.Provider.BoardType == BoardType.Danbooru) { post = SankakuComplexParser.ParsePost(post, html, !chkUseGlobalProviderTags.Checked); //post.FileUrl = tempPost.FileUrl; //post.PreviewUrl = tempPost.PreviewUrl; } else if (post.Provider.BoardType == BoardType.Gelbooru) { var tempPost = GelbooruHtmlParser.ParsePost(post, html); post.FileUrl = tempPost.FileUrl; post.PreviewUrl = tempPost.PreviewUrl; } else { UpdateLog("DoBatchJob", "No HTML Parser available for : " + post.Provider.Name + "(" + post.Provider.BoardType.ToString() + ")"); } break; } catch (Exception ex) { if (currRetry >= maxRetry) { UpdateLog("DoBatchJob", "Giving Up Resolving FileUrl: " + ex.StackTrace, ex); post.FileUrl = ""; post.JpegUrl = ""; post.SampleUrl = ""; break; } ++currRetry; UpdateLog("DoBatchJob", "Error Resolving FileUrl (" + currRetry + " of " + maxRetry + "): " + ex.Message + " Wait for " + delay + "s.", ex); for (int wait = 0; wait < delay; ++wait) { //UpdateLog("DoBatchJob", "Wait for " + wait + " of " + delay); Thread.Sleep(1000); } UpdateLog("DoBatchJob", "Retrying..."); } } } }
private int DoDownloadBatch(string targetUrl, DanbooruBatchJob job, DanbooruPost post, string filename) { UpdateLog("DoBatchJob", "Download: " + targetUrl); _clientBatch.Referer = post.Referer; int currRetry = 0; int maxRetry = Convert.ToInt32(txtRetry.Text); int delay = Convert.ToInt32(txtDelay.Text); while (currRetry <= maxRetry) { try { var filename2 = filename + ".!tmp"; #if DEBUG UpdateLog("DoBatchJob", "DEBUG Saved To: " + filename); Thread.Sleep(100); #else if (File.Exists(filename2)) { UpdateLog("DoBatchJob", "Deleting temporary file: " + filename2); File.Delete(filename2); } _clientBatch.DownloadFile(targetUrl, filename2); File.Move(filename2, filename); UpdateLog("DoBatchJob", "Saved To: " + filename); #endif ++job.Downloaded; if (Properties.Settings.Default.WriteDownloadedFile) { // write to text file for downloaded file. Helper.WriteTextFile(filename + Environment.NewLine); } return 1; } catch (System.Net.WebException ex) { if (currRetry < maxRetry && cbxAbortOnError.Checked) throw; else { var message = ex.Message; if (ex.InnerException != null) message = ex.InnerException.Message; UpdateLog("DoBatchJob", "Error Download Batch Image (" + currRetry + " of " + maxRetry + "): " + message + " Wait for " + delay + "s.", ex); for (int wait = 0; wait < delay; wait++) { //UpdateLog("DoBatchJob", "Wait for " + wait + " of " + delay); Thread.Sleep(1000); } UpdateLog("DoBatchJob", "Retrying..."); } ++currRetry; if (currRetry > Convert.ToInt32(txtRetry.Text)) ++job.Error; } } return 0; // failed }
private static bool IsBlacklisted(DanbooruPost post, Regex regex) { if (regex != null) { return post.TagsEntity.Any(x => regex.IsMatch(x.Name)); } return false; }
private static void ReadRssMethod1(DanbooruPostDaoOption option, BindingList<DanbooruPost> posts, string xmldoc) { using (StringReader strReader = new StringReader(xmldoc)) { using (XmlReader reader = new XmlTextReader(strReader)) { XDocument doc = XDocument.Load(reader); string media = doc.Root.Attribute("{http://www.w3.org/2000/xmlns/}media").Value; foreach (var item in doc.Descendants("item")) { DanbooruPost post = new DanbooruPost(); var titleData = item.Element("title").Value.Split(new char[] { '-' }, 2); post.Id = titleData[0].Trim(); post.Tags = titleData[1].Trim(); post.Referer = AppendHttp(item.Element("link").Value, option.Provider); post.CreatedAt = item.Element("pubDate").Value; var data = item.Element("{" + media + "}thumbnail"); post.PreviewUrl = AppendHttp(data.Attribute("url").Value, option.Provider); data = item.Element("{" + media + "}content"); post.FileUrl = AppendHttp(data.Attribute("url").Value, option.Provider); try { string description = HttpUtility.HtmlDecode(item.Element("description").Value); Match matches = imageResolutionRegex.Match(description); post.Width = Convert.ToInt32(matches.Groups[1].Value); post.Height = Convert.ToInt32(matches.Groups[2].Value); post.PreviewWidth = Convert.ToInt32(matches.Groups[3].Value); post.PreviewHeight = Convert.ToInt32(matches.Groups[4].Value); post.CreatorId = matches.Groups[5].Value; } catch (Exception) { } PostProcess(option, post); posts.Add(post); } } } }
private void ProcessXML(string rawData) { RawData = rawData; using (StringReader strReader = new StringReader(rawData)) { using (XmlTextReader reader = new XmlTextReader(strReader)) { if (Option.Provider.BoardType == BoardType.Shimmie2) { posts = Engine.ShimmieEngine.ParseRSS(rawData, Option); } else { while (reader.Read()) { switch (reader.NodeType) { case XmlNodeType.Element: // The node is an element. var nodeName = reader.Name.ToLowerInvariant(); if (nodeName.Equals("posts")) { while (reader.MoveToNextAttribute()) { if (reader.Name.ToLowerInvariant().Equals("count")) // Posts Count { postCount = int.Parse(reader.Value); } else if (reader.Name.ToLowerInvariant().Equals("offset")) // Post Offset { offset = int.Parse(reader.Value); } } } else if (nodeName.Equals("response")) { Success = true; while (reader.MoveToNextAttribute()) { if (reader.Name.ToLowerInvariant().Equals("reason")) // Posts Count { ResponseMessage = reader.Value; } else if (reader.Name.ToLowerInvariant().Equals("success")) // Posts Count { Success = bool.Parse(reader.Value); } } } else if (nodeName.Equals("post")) { DanbooruPost post = new DanbooruPost(); ParsePostAttributes(reader, post); post.Hidden = Helper.CheckBlacklistedTag(post, Option); post.Provider = Option.Provider; post.Query = Option.Query; post.SearchTags = Option.SearchTags; if (Option.Provider.BoardType == BoardType.Danbooru || Option.Provider.BoardType == BoardType.Shimmie2) { post.Referer = Option.Provider.Url + @"/post/show/" + post.Id; } else if (Option.Provider.BoardType == BoardType.Gelbooru) { post.Referer = Option.Provider.Url + @"/index.php?page=post&s=view&id=" + post.Id; } posts.Add(post); actualCount++; } break; case XmlNodeType.EndElement: //Display the end of the element. //txtResult.AppendText("END"); break; } } } } } }
public BindingList<DanbooruPost> Parse(string data, DanbooruSearchParam searchParam) { this.SearchParam = searchParam; this.RawData = data; BindingList<DanbooruPost> posts = new BindingList<DanbooruPost>(); HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(data); // remove popular preview var popular = doc.DocumentNode.SelectSingleNode("//div[@id='popular-preview']"); if (popular != null) { popular.Remove(); } // get all thumbs var thumbs = doc.DocumentNode.SelectNodes("//span"); if (thumbs != null && thumbs.Count > 0) { foreach (var thumb in thumbs) { if (thumb.GetAttributeValue("class", "").Contains("thumb")) { DanbooruPost post = new DanbooruPost(); post.Id = thumb.GetAttributeValue("id", "_N/A").Substring(1); post.Provider = searchParam.Provider; post.SearchTags = searchParam.Tag; post.Query = GenerateQueryString(searchParam); int i = 0; // get the image link for (; i < thumb.ChildNodes.Count; ++i) { if (thumb.ChildNodes[i].Name == "a") break; } var a = thumb.ChildNodes[i]; post.Referer = searchParam.Provider.Url + "/" + System.Web.HttpUtility.HtmlDecode(a.GetAttributeValue("href", "")); if (post.Id == "N/A") { post.Id = a.GetAttributeValue("id", "N/A").Substring(1); } var img = a.ChildNodes[i]; var title = img.GetAttributeValue("title", ""); var title2 = title.ToString(); post.Tags = title.Substring(0, title.LastIndexOf("rating:")).Trim(); post.Tags = Helper.DecodeEncodedNonAsciiCharacters(post.Tags); post.TagsEntity = Helper.ParseTags(post.Tags, SearchParam.Provider); post.Hidden = Helper.CheckBlacklistedTag(post, searchParam.Option); post.PreviewUrl = img.GetAttributeValue("src", ""); post.PreviewHeight = img.GetAttributeValue("height", 0); post.PreviewWidth = img.GetAttributeValue("width", 0); post.Source = ""; post.Score = title.Substring(title.LastIndexOf("score:") + 6); post.Score = post.Score.Substring(0, post.Score.LastIndexOf(" ")).Trim(); title2 = title2.Substring(title2.LastIndexOf("rating:")); post.Rating = title2.Substring(7, 1).ToLower(); post.Status = ""; post.MD5 = post.PreviewUrl.Substring(post.PreviewUrl.LastIndexOf("/") + 1); post.MD5 = post.MD5.Substring(0, post.MD5.LastIndexOf(".")); post.MD5 = post.MD5.Replace("thumbnail_", ""); posts.Add(post); } } } TotalPost = posts.Count; if (!SearchParam.Page.HasValue) SearchParam.Page = 0; Offset = posts.Count * SearchParam.Page; return posts; }
public static DanbooruPost ParsePost(DanbooruPost post, string postHtml) { HtmlDocument doc = new HtmlDocument(); if (String.IsNullOrEmpty(postHtml)) throw new Exception("No post html!"); doc.LoadHtml(postHtml); string file_url = ""; string sample_url = ""; // Flash Game or bmp // TODO: need to change the preview url if (post.PreviewUrl == "http://chan.sankakucomplex.com/download-preview.png") { var links = doc.DocumentNode.SelectNodes("//a"); foreach (var link in links) { // flash if (link.InnerText == "Save this flash (right click and save)") { file_url = link.GetAttributeValue("href", ""); break; } // bmp if (link.InnerText == "Download") { file_url = link.GetAttributeValue("href", ""); break; } } } else { var image = doc.DocumentNode.SelectSingleNode("//img[@id='image']"); if (image != null) { sample_url = image.GetAttributeValue("src", ""); } var links = doc.DocumentNode.SelectNodes("//a"); foreach (var link in links) { if (link.InnerText == "Original image") { file_url = link.GetAttributeValue("href", ""); break; } } } post.FileUrl = file_url; if (!string.IsNullOrWhiteSpace(file_url) && string.IsNullOrWhiteSpace(sample_url)) sample_url = file_url; post.SampleUrl = sample_url; // parse datetime // TODO: untested var statsLIs = doc.DocumentNode.SelectNodes("//div[@id='stats']//li"); foreach (var item in statsLIs) { if (item.InnerHtml.StartsWith("Posted: ")) { post.CreatedAt = System.Text.RegularExpressions.Regex.Match(item.InnerHtml, "Posted: (.*)<").Value; post.CreatedAtDateTime = DanbooruPostDao.ParseDateTime(post.CreatedAt, post.Provider); break; } } return post; }
private void TransferDownloadRow(DanbooruPost post) { // try to get the file url if empty if (string.IsNullOrWhiteSpace(post.FileUrl) && (post.Status != "deleted" || chkProcessDeletedPost.Checked)) { if (!string.IsNullOrWhiteSpace(post.Referer)) { ResolveFileUrl(post); } } // reset status post.Completed = false; // resolve filename if (String.IsNullOrWhiteSpace(post.Filename) && !String.IsNullOrWhiteSpace(post.FileUrl)) { post.Filename = MakeCompleteFilename(post, post.FileUrl); } if (!String.IsNullOrWhiteSpace(post.Filename) && File.Exists(post.Filename)) { post.Progress = "File exists!"; } _downloadList.Add(post); }
private void ProcessJson(String json) { if (json.Length < 4) return; if (json.StartsWith("{")) { json = json.Substring(1, json.Length - 2); string[] splitter = { "\"," }; string[] node = json.Split(splitter, StringSplitOptions.None); foreach (string str2 in node) { string[] val = str2.Split(':'); switch (val[0].ToLowerInvariant()) { case "\"reason\"": ResponseMessage = val[1].Replace("\"", ""); break; case "\"success\"": Success = bool.Parse(val[1].Replace("\"", "")); break; } } } else { json = json.Substring(2, json.Length - 4); string[] splitter = { "},{" }; string[] split = json.Split(splitter, StringSplitOptions.None); foreach (string str in split) { DanbooruPost post = new DanbooruPost(); string[] node = str.Split(','); foreach (string str2 in node) { char[] splitter2 = { ':' }; string[] val = str2.Split(splitter2, 2); switch (val[0].ToLowerInvariant()) { case "\"id\"": post.Id = val[1].Replace("\"", ""); break; case "\"tags\"": post.Tags = val[1].Replace("\"", ""); post.Tags = Helper.DecodeEncodedNonAsciiCharacters(post.Tags); post.TagsEntity = Helper.ParseTags(post.Tags, Option.Provider); break; case "\"source\"": post.Source = val[1].Replace("\"", ""); break; case "\"creator_id\"": post.CreatorId = val[1].Replace("\"", ""); break; case "\"file_url\"": post.FileUrl = AppendHttp(val[1].Replace("\"", "")); break; case "\"width\"": post.Width = -1; try { post.Width = Convert.ToInt32(val[1]); } catch (Exception) { if (FormMain.Debug) throw; } break; case "\"height\"": post.Height = -1; try { post.Height = Convert.ToInt32(val[1]); } catch (Exception) { if (FormMain.Debug) throw; } break; case "\"score\"": post.Score = val[1]; break; case "\"rating\"": post.Rating = val[1].Replace("\"", ""); break; case "\"md5\"": post.MD5 = val[1].Replace("\"", ""); break; case "\"preview_url\"": post.PreviewUrl = AppendHttp(val[1].Replace("\"", "")); break; case "\"preview_width\"": post.PreviewWidth = -1; try { post.PreviewWidth = Convert.ToInt32(val[1]); } catch (Exception) { if (FormMain.Debug) throw; } break; case "\"preview_height\"": post.PreviewHeight = -1; try { post.PreviewHeight = Convert.ToInt32(val[1]); } catch (Exception) { if (FormMain.Debug) throw; } break; case "\"file_size\"": post.Filesize = -1; try { post.Filesize= Convert.ToInt32(val[1]); } catch (Exception) { if (FormMain.Debug) throw; } break; case "\"parent_id\"": post.ParentId = val[1]; break; case "\"status\"": post.Status = val[1]; break; case "\"created_at\"": post.CreatedAt = val[1]; break; case "\"has_children\"": post.HasChildren = false; try { post.HasChildren = Convert.ToBoolean(val[1]); } catch (Exception) { if (FormMain.Debug) throw; } break; case "\"sample_width\"": post.SampleWidth = -1; try { post.SampleWidth = Convert.ToInt32(val[1]); } catch (Exception) { if (FormMain.Debug) throw; } break; case "\"sample_height\"": post.SampleHeight = -1; try { post.SampleHeight = Convert.ToInt32(val[1]); } catch (Exception) { if (FormMain.Debug) throw; } break; case "\"sample_url\"": post.SampleUrl = AppendHttp(val[1].Replace("\"", "")); break; case "\"jpeg_width\"": post.JpegWidth = -1; try { post.JpegWidth = Convert.ToInt32(val[1]); } catch (Exception) { if (FormMain.Debug) throw; } break; case "\"jpeg_height\"": post.JpegHeight = -1; try { post.JpegHeight = Convert.ToInt32(val[1]); } catch (Exception) { if (FormMain.Debug) throw; } break; case "\"jpeg_url\"": post.JpegUrl = AppendHttp(val[1].Replace("\"", "")); break; default: break; } } post.Hidden = Helper.CheckBlacklistedTag(post, Option); post.Provider = Option.Provider; post.Query = Option.Query; post.SearchTags = Option.SearchTags; post.Referer = Option.Provider.Url + @"/post/show/" + post.Id; this.posts.Add(post); actualCount++; } } }
internal static void WriteTagFile(DanbooruPost post, string filename) { if (File.Exists(filename)) { File.Delete(filename); } using (TextWriter tw = File.CreateText(filename)) { foreach (var item in post.TagsEntity) { tw.WriteLine(item.Name); } } }
private void ResolveFileUrl(DanbooruPost post) { resolveQueue.Enqueue(post); post.FileUrl = Constants.LOADING_URL; if (!isResolverRunning) ResolveFileUrl(); }
/// <summary> /// Reparse tags from post details. /// </summary> /// <param name="post"></param> /// <param name="doc"></param> private static void ReparseTags(DanbooruPost post, HtmlDocument doc) { post.TagsEntity.Clear(); var tagsElement = doc.DocumentNode.SelectNodes("//ul[@id='tag-sidebar']/li"); foreach (var tag in tagsElement) { HtmlDocument tagDoc = new HtmlDocument(); tagDoc.LoadHtml(tag.OuterHtml); var tagEntity = new DanbooruTag(); var el = tagDoc.DocumentNode.SelectSingleNode("//li"); var cls = el.Attributes["class"].Value; switch (cls) { case "tag-type-idol": // idol complex tagEntity.Type = DanbooruTagType.Artist; break; case "tag-type-artist": // sankaku tagEntity.Type = DanbooruTagType.Artist; break; case "tag-type-photo_set": // idol complex: usually album name tagEntity.Type = DanbooruTagType.Circle; break; case "tag-type-studio": // sankaku: circlename tagEntity.Type = DanbooruTagType.Circle; break; case "tag-type-meta": // both tagEntity.Type = DanbooruTagType.Faults; break; case "tag-type-medium": // both tagEntity.Type = DanbooruTagType.Faults; break; case "tag-type-general": // both tagEntity.Type = DanbooruTagType.General; break; case "tag-type-copyright": // both tagEntity.Type = DanbooruTagType.Copyright; break; case "tag-type-character": // both tagEntity.Type = DanbooruTagType.Character; break; default: tagEntity.Type = DanbooruTagType.Unknown; break; } tagEntity.Name = tagDoc.DocumentNode.SelectSingleNode("//li/a").InnerText; var countStr = tagDoc.DocumentNode.SelectSingleNode("//li//span[@class='post-count']").InnerText.Trim(); tagEntity.Count = Int32.Parse(countStr); post.TagsEntity.Add(tagEntity); } post.TagsEntity = post.TagsEntity.OrderByDescending(x => x.Type).ThenBy(x => x.Name).ToList(); }
/// <summary> /// Return string from tag filtered by tag type. /// Replace the returned string with tagReplacement if tag count over tagLimit. /// Replace the returned string with missingTagReplacement if tag count is 0. /// </summary> /// <param name="post"></param> /// <param name="groupedTags"></param> /// <param name="tagType"></param> /// <param name="tagLimit"></param> /// <param name="tagReplacement"></param> /// <param name="missingTagReplacement"></param> /// <param name="isReplaceMode"></param> /// <param name="isReplaceUnderScore"></param> /// <returns>Return string from tag filtered by tag type.</returns> private static string FilterTags(DanbooruPost post, List<DanbooruTag> groupedTags, DanbooruTagType tagType, int tagLimit, string tagReplacement, string missingTagReplacement, bool isReplaceMode, bool isReplaceUnderScore) { var selectedTags = post.TagsEntity.Where<DanbooruTag>(x => x.Type == tagType).Select(x => x.Name).ToList(); var tagStr = ""; if (selectedTags != null) { if (isReplaceUnderScore) { for (int i = 0; i < selectedTags.Count; i++) { selectedTags[i] = selectedTags[i].Replace("_", " ").Trim(); } } if (tagLimit > 0 && selectedTags.Count() >= tagLimit) { if (isReplaceMode && !string.IsNullOrWhiteSpace(tagReplacement)) { tagStr = tagReplacement; if (isReplaceUnderScore) tagStr = tagReplacement.Replace("_", " ").Trim(); //groupedTags.RemoveAll(x => x.Type == tagType); //groupedTags.Add(new DanbooruTag() { Name = tagReplacement }); } else { var tempTags = selectedTags.Take(tagLimit); tagStr = String.Join(" ", tempTags); } } else { tagStr = string.Join(" ", selectedTags); } } if (string.IsNullOrWhiteSpace(tagStr)) tagStr = missingTagReplacement; return tagStr; }
private string MakeCompleteFilename(DanbooruPost post, string url) { var format = new DanbooruFilenameFormat() { FilenameFormat = txtFilenameFormat.Text, Limit = Convert.ToInt32(txtFilenameLength.Text), BaseFolder = txtSaveFolder.Text, MissingTagReplacement = txtTagReplacement.Text, ArtistGroupLimit = Convert.ToInt32(txtArtistTagGrouping.Text), CharacterGroupLimit = Convert.ToInt32(txtCharaTagGrouping.Text), CopyrightGroupLimit = Convert.ToInt32(txtCopyTagGrouping.Text), CircleGroupLimit = Convert.ToInt32(txtCircleTagGrouping.Text), FaultsGroupLimit = Convert.ToInt32(txtFaultsTagGrouping.Text), IgnoredTags = DanbooruTagsDao.Instance.ParseTagsString(txtIgnoredTags.Text.Replace(Environment.NewLine, " ")), IgnoredTagsRegex = txtIgnoredTags.Text.Trim().Replace(Environment.NewLine, "|"), IgnoreTagsUseRegex = chkIgnoreTagsUseRegex.Checked, IsReplaceMode = chkReplaceMode.Checked, IgnoredTagsOnlyForGeneral = chkIgnoreForGeneralTag.Checked, TagReplaceUnderscoreToSpace = chkIsReplaceUnderscoreTag.Checked }; string extension = Helper.getFileExtensions(url); string filename = Helper.MakeFilename(format, post) + extension; return filename; }
public BindingList<DanbooruPost> Parse(string data, DanbooruSearchParam searchParam) { try { this.SearchParam = searchParam; this.RawData = data; BindingList<DanbooruPost> posts = new BindingList<DanbooruPost>(); HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(data); // remove popular preview var popular = doc.DocumentNode.SelectSingleNode("//div[@id='popular-preview']"); if (popular != null) { popular.Remove(); } // get all thumbs var thumbs = doc.DocumentNode.SelectNodes("//span"); if (thumbs != null && thumbs.Count > 0) { foreach (var thumb in thumbs) { if (thumb.GetAttributeValue("class", "").Contains("thumb")) { DanbooruPost post = new DanbooruPost(); post.Id = thumb.GetAttributeValue("id", "-1").Substring(1); post.Provider = searchParam.Provider; post.SearchTags = searchParam.Tag; post.Query = GenerateQueryString(searchParam); int i = 0; // get the image link for (; i < thumb.ChildNodes.Count; ++i) { if (thumb.ChildNodes[i].Name == "a") break; } var a = thumb.ChildNodes[i]; post.Referer = Helper.FixUrl(searchParam.Provider.Url + a.GetAttributeValue("href", ""), isHttps(post.Provider)); var img = a.ChildNodes[i]; var title = img.GetAttributeValue("title", ""); post.Tags = title.Substring(0, title.LastIndexOf("Rating:")).Trim(); post.Tags = Helper.DecodeEncodedNonAsciiCharacters(post.Tags); post.TagsEntity = Helper.ParseTags(post.Tags, SearchParam.Provider); post.Hidden = Helper.CheckBlacklistedTag(post, searchParam.Option); var status = img.GetAttributeValue("class", "").Replace("preview", "").Trim(); if (status.Contains("deleted")) post.Status = "deleted"; else if (status.Contains("pending")) post.Status = "pending"; else post.Status = status; post.PreviewUrl = Helper.FixUrl(img.GetAttributeValue("src", ""), isHttps(post.Provider)); post.PreviewHeight = img.GetAttributeValue("height", 0); post.PreviewWidth = img.GetAttributeValue("width", 0); // Rating:Explicit Score:4.5 Size:1080x1800 User:System post.Source = ""; post.Score = title.Substring(title.LastIndexOf("Score:") + 6); post.Score = post.Score.Substring(0, post.Score.IndexOf(" ")).Trim(); string resolution = title.Substring(title.LastIndexOf("Size:") + 5); resolution = resolution.Substring(0, resolution.IndexOf(" ")).Trim(); string[] resArr = resolution.Split('x'); post.Width = Int32.Parse(resArr[0]); post.Height = Int32.Parse(resArr[1]); string rating = title.Substring(title.LastIndexOf("Rating:") + 7, 1); //rating = rating.Substring(0, rating.IndexOf(" ")).Trim(); post.Rating = rating.ToLower(); post.CreatorId = title.Substring(title.LastIndexOf("User:"******"/") + 1); post.MD5 = post.MD5.Substring(0, post.MD5.LastIndexOf(".")); posts.Add(post); } } } TotalPost = posts.Count; if (!SearchParam.Page.HasValue) SearchParam.Page = 1; Offset = TotalPost * SearchParam.Page; return posts; } catch (Exception ex) { var filename = Helper.SanitizeFilename("Dump for Sankaku Image List - " + searchParam.Tag + " - page " + searchParam.Page + ".txt"); var result = Helper.DumpRawData(data, filename); if (!result) Program.Logger.Error("Failed to dump rawdata to: " + filename, ex); throw; } }
public void TestDumpRawData() { string dump = "test124"; DanbooruPost post = new DanbooruPost(); post.Id = "123"; post.Query = ""; post.Provider = new DanbooruProvider() { Name = "TestProvider" }; string filename = "Dump for Post " + post.Id + post.Provider.Name + " Query " + post.Query + ".txt"; bool result = Helper.DumpRawData(dump, filename); Assert.IsTrue(result); }
private static void ReparseTags(DanbooruPost post, HtmlDocument doc) { post.TagsEntity.Clear(); var tagsElement = doc.DocumentNode.SelectNodes("//ul[@id='tag-sidebar']/li"); foreach (var tag in tagsElement) { HtmlDocument tagDoc = new HtmlDocument(); tagDoc.LoadHtml(tag.OuterHtml); var tagEntity = new DanbooruTag(); var el = tagDoc.DocumentNode.SelectSingleNode("//li"); var cls = el.Attributes["class"].Value; switch (cls) { case "tag-type-general": tagEntity.Type = DanbooruTagType.General; break; case "tag-type-artist": tagEntity.Type = DanbooruTagType.Artist; break; case "tag-type-copyright": tagEntity.Type = DanbooruTagType.Copyright; break; case "tag-type-character": tagEntity.Type = DanbooruTagType.Character; break; //case "tag-type-medium": // tagEntity.Type = DanbooruTagType.Circle; // break; case "tag-type-medium": tagEntity.Type = DanbooruTagType.Faults; break; default: tagEntity.Type = DanbooruTagType.Unknown; break; } tagEntity.Name = tagDoc.DocumentNode.SelectSingleNode("//li/a").InnerText; var countStr = tagDoc.DocumentNode.SelectSingleNode("//li//span[@class='post-count']").InnerText.Trim(); tagEntity.Count = Int32.Parse(countStr); post.TagsEntity.Add(tagEntity); } }
public void TestGelbooruParser() { DanbooruProviderDao pd = DanbooruProviderDao.GetInstance(); string target = @"../../../DanbooruDownloader3.test/TestXml/gelbooru_post.htm"; var data = File.ReadAllText(target); var query = new DanbooruSearchParam(); query.Provider = pd.Read(sourceProvider).Where(x => x.Name == "gelbooru.com").First(); query.Tag = ""; query.OrderBy = "score"; var post = new DanbooruPost(); post.Provider = query.Provider; GelbooruHtmlParser.ParsePost(post, data); Assert.IsNotNull(post.FileUrl); Assert.IsTrue(post.FileUrl == @"http://cdn2.gelbooru.com//images/1559/303b7ed1fcba0c1d9520f76ee34ec37e.jpg", "Actual: " + post.FileUrl); }