public string GetQueryString(DanbooruSearchParam searchParam)
{
var queryStr = "";
if (BoardType == BoardType.Shimmie2)
{
if (Preferred == PreferredMethod.Html)
{
queryStr = new ShimmieHtmlParser().GenerateQueryString(searchParam);
}
else
{
queryStr = ShimmieEngine.GetQueryString(this, searchParam);
}
}
else if (Url.Contains("sankakucomplex.com"))
{
queryStr = new SankakuComplexParser().GenerateQueryString(searchParam);
}
else
{
queryStr = DanbooruXmlEngine.GetQueryString(this, searchParam);
}
return(queryStr);
}
public static string GetQueryString(DanbooruProvider provider, DanbooruSearchParam query)
{
var queryStr = "";
// Clean up txtTags
var tags = query.Tag;
while (tags.Contains(" "))
{
tags = tags.Replace(" ", " ");
}
tags = tags.Trim();
tags = System.Web.HttpUtility.UrlEncode(tags);
//StartPage
var page = 1;
if (query.Page.HasValue && query.Page > 0)
{
page = query.Page.Value;
}
queryStr = tags;
if (!String.IsNullOrWhiteSpace(tags)) queryStr += "/";
queryStr += page;
return queryStr;
}
public void TestDanbooruEngineParser()
{
DanbooruProviderDao pd = DanbooruProviderDao.GetInstance();
DanbooruXmlEngine e = new DanbooruXmlEngine();
{
XDocument doc = XDocument.Load(sourceDanbooruXml);
var searchQuery = new DanbooruSearchParam();
searchQuery.Provider = pd.Read(sourceProvider).Where<DanbooruProvider>(x => x.BoardType == BoardType.Danbooru && x.Name.Contains("danbooru")).First<DanbooruProvider>();
BindingList<DanbooruPost> result = e.Parse(doc.ToString(), searchQuery);
Assert.IsNotNull(result);
Assert.IsNotNull(e.RawData);
Assert.IsTrue(e.TotalPost == 1021107);
Assert.IsTrue(result.Count == 20);
Assert.IsTrue(result[0].PreviewUrl == "http://danbooru.donmai.us/ssd/data/preview/73531fc4dda535ef87e57df633caf756.jpg");
}
{
XDocument doc = XDocument.Load(sourceYandereXml);
var searchQuery = new DanbooruSearchParam();
searchQuery.Provider = pd.Read(sourceProvider).Where<DanbooruProvider>(x => x.BoardType == BoardType.Danbooru && x.Name.Contains("yande.re")).First<DanbooruProvider>();
BindingList<DanbooruPost> result = e.Parse(doc.ToString(), searchQuery);
Assert.IsNotNull(result);
Assert.IsNotNull(e.RawData);
Assert.IsTrue(e.TotalPost == 160753);
Assert.IsTrue(result.Count == 16);
Assert.IsTrue(result[0].PreviewUrl == "https://yande.re/data/preview/d3/41/d34184030ee19c6e63051967cf135f65.jpg");
}
}
public string GetQueryUrl(DanbooruSearchParam searchParam)
{
var queryStr = GetQueryString(searchParam);
var queryRootUrl = "";;
switch (this.Preferred)
{
case PreferredMethod.Xml:
queryRootUrl = this.QueryStringXml;
break;
case PreferredMethod.Json:
queryRootUrl = this.QueryStringJson;
break;
case PreferredMethod.Html:
queryRootUrl = this.QueryStringHtml;
break;
default:
break;
}
queryStr = queryRootUrl.Replace("%_query%", queryStr);
switch (this.LoginType)
{
case Entity.LoginType.UserPass:
{
var hash = this.PasswordHash;
if (String.IsNullOrWhiteSpace(hash))
{
hash = Helper.GeneratePasswordHash(this.Password, this.PasswordSalt);
this.PasswordHash = hash;
}
string authString = "login="******"&password_hash=" + hash;
queryStr = queryStr + "&" + authString;
}
break;
case Entity.LoginType.Cookie:
case Entity.LoginType.CookieAlwaysAsk:
// need to inject csv cookie to the webclient
var cookies = Helper.ParseCookie(this.UserName, this.Url);
foreach (var cookie in cookies)
{
ExtendedWebClient.CookieJar.Add(cookie);
}
break;
default:
break;
}
return(this.Url + queryStr);
}
/// <summary>
/// Get search parameter from Main Tab and Option Panels
/// </summary>
/// <returns></returns>
public DanbooruSearchParam GetSearchParams()
{
var option = new DanbooruPostDaoOption()
{
BlacklistedTags = TagBlacklist,
BlacklistedTagsRegex = TagBlacklistRegex,
BlacklistedTagsUseRegex = chkBlacklistTagsUseRegex.Checked,
IgnoredTags = TagIgnore,
IgnoredTagsRegex = TagIgnoreRegex,
IgnoredTagsUseRegex = chkIgnoreTagsUseRegex.Checked,
Provider = _currProvider,
SearchTags = !String.IsNullOrWhiteSpace(txtTags.Text) ? txtTags.Text : "",
IsBlacklistOnlyForGeneral = chkBlacklistOnlyGeneral.Checked
};
DanbooruSearchParam searchParam = new DanbooruSearchParam();
searchParam.Provider = option.Provider;
searchParam.Tag = option.SearchTags;
searchParam.Source = txtSource.Text.Trim();
int limit = 0;
if (Int32.TryParse(txtLimit.Text, out limit) && limit > 0) searchParam.Limit = limit;
else searchParam.Limit = null;
int page = _currProvider.BoardType == BoardType.Gelbooru ? 0 : 1;
if (Int32.TryParse(txtPage.Text, out page) && page > 0) searchParam.Page = page;
else searchParam.Page = null;
searchParam.IsNotRating = chkNotRating.Checked;
if (cbxRating.SelectedValue != null)
{
if (cbxRating.SelectedValue.GetType() == typeof(string))
searchParam.Rating = cbxRating.SelectedValue.ToString();
else
{
var rating = (KeyValuePair<string, string>)cbxRating.SelectedValue;
searchParam.Rating = rating.Value;
}
}
if (cbxOrder.SelectedValue != null)
{
if (cbxOrder.SelectedValue.GetType() == typeof(string))
searchParam.OrderBy = cbxOrder.SelectedValue.ToString();
else
{
var order = (KeyValuePair<string, string>)cbxOrder.SelectedValue;
searchParam.OrderBy = order.Value;
}
}
searchParam.Option = option;
return searchParam;
}
public string GetQueryUrl(DanbooruSearchParam searchParam)
{
var queryStr = GetQueryString(searchParam);
var queryRootUrl = "";;
switch (Preferred)
{
case PreferredMethod.Xml:
queryRootUrl = QueryStringXml;
break;
case PreferredMethod.Json:
queryRootUrl = QueryStringJson;
break;
case PreferredMethod.Html:
queryRootUrl = QueryStringHtml;
break;
default:
break;
}
queryStr = queryRootUrl.Replace("%_query%", queryStr);
switch (LoginType)
{
case LoginType.UserPass:
{
var hash = PasswordHash;
if (String.IsNullOrWhiteSpace(hash))
{
hash = Helper.GeneratePasswordHash(Password, PasswordSalt);
PasswordHash = hash;
}
queryStr = $"{queryStr}&login={UserName}&password_hash={hash}";
}
break;
case LoginType.Cookie:
case LoginType.CookieAlwaysAsk:
// need to inject csv cookie to the webclient
ReloadCookie(UserName);
break;
default:
break;
}
return($"{Url}{queryStr}");
}
/// <summary>
/// parse xml/json list stream and close it.
/// </summary>
/// <param name="input"></param>
/// <param name="option"></param>
public DanbooruPostDao(Stream input, DanbooruPostDaoOption option)
{
string rawData = "";
this.Option = option;
try
{
using (StreamReader reader = new StreamReader(input))
{
rawData = reader.ReadToEnd();
}
this.Option = option;
switch (option.Provider.Preferred)
{
case PreferredMethod.Xml:
ReadXML(rawData, option);
break;
case PreferredMethod.Json:
ReadJSON(rawData, option);
break;
case PreferredMethod.Html:
DanbooruSearchParam param = new DanbooruSearchParam()
{
Provider = option.Provider,
Tag = option.SearchTags,
Option = option
};
if (option.Provider.BoardType == BoardType.Danbooru)
{
SankakuComplexParser parser = new SankakuComplexParser();
posts = parser.Parse(rawData, param);
}
else if (option.Provider.BoardType == BoardType.Gelbooru)
{
GelbooruHtmlParser parser = new GelbooruHtmlParser();
posts = parser.Parse(rawData, param);
}
else
{
throw new NotImplementedException("No HTML Parser for: " + option.Provider.Name);
}
break;
}
}
catch (Exception)
{
Helper.DumpRawData(rawData, Option.Provider, option.Query);
throw;
}
}
public string GetQueryString(DanbooruSearchParam searchParam)
{
var queryStr = "";
if (this.BoardType == BoardType.Shimmie2)
{
queryStr = DanbooruDownloader3.Engine.ShimmieEngine.GetQueryString(this, searchParam);
}
else
{
queryStr = DanbooruDownloader3.Engine.DanbooruXmlEngine.GetQueryString(this, searchParam);
}
return(queryStr);
}
public string GetQueryString(DanbooruSearchParam searchParam)
{
var queryStr = "";
if (this.BoardType == BoardType.Shimmie2)
{
queryStr = DanbooruDownloader3.Engine.ShimmieEngine.GetQueryString(this, searchParam);
}
else if (this.Url.Contains("sankakucomplex.com"))
{
queryStr = new SankakuComplexParser().GenerateQueryString(searchParam);
}
else
{
queryStr = DanbooruDownloader3.Engine.DanbooruXmlEngine.GetQueryString(this, searchParam);
}
return(queryStr);
}
public string GetQueryUrl(DanbooruSearchParam searchParam)
{
var queryStr = GetQueryString(searchParam);
var queryRootUrl = "";;
switch (this.Preferred)
{
case PreferredMethod.Xml:
queryRootUrl = this.QueryStringXml;
break;
case PreferredMethod.Json:
queryRootUrl = this.QueryStringJson;
break;
case PreferredMethod.Html:
queryRootUrl = this.QueryStringHtml;
break;
default:
break;
}
queryStr = queryRootUrl.Replace("%_query%", queryStr);
if (this.UseAuth)
{
var hash = this.PasswordHash;
if (String.IsNullOrWhiteSpace(hash))
{
hash = Helper.GeneratePasswordHash(this.Password, this.PasswordSalt);
this.PasswordHash = hash;
}
string authString = "login="******"&password_hash=" + hash;
queryStr = queryStr + "&" + authString;
}
return(this.Url + queryStr);
}
public static string GetQueryString(DanbooruProvider provider, DanbooruSearchParam query)
{
var queryStr = "";
// Clean up txtTags
var tags = query.Tag;
while (tags.Contains(" "))
{
tags = tags.Replace(" ", " ");
}
tags = tags.Trim();
tags = System.Web.HttpUtility.UrlEncode(tags);
List<string> queryList = new List<string>();
List<string> tagsList = new List<string>();
//Tags
if (tags.Length > 0) tagsList.Add(tags.Replace(' ', '+'));
//Rating
if (!String.IsNullOrWhiteSpace(query.Rating)) tagsList.Add(query.IsNotRating ? "-" + query.Rating : "" + query.Rating);
//Source
if (!String.IsNullOrWhiteSpace(query.Source)) tagsList.Add("source:" + query.Source);
//Order
if (!String.IsNullOrWhiteSpace(query.OrderBy)) tagsList.Add(query.OrderBy);
if (tagsList.Count > 0) queryList.Add("tags=" + String.Join("+", tagsList));
//Limit
if (query.Limit > 0) queryList.Add("limit=" + query.Limit);
//StartPage
if (query.Page > 0)
{
if (provider.BoardType == BoardType.Danbooru) queryList.Add("page=" + query.Page);
else if (provider.BoardType == BoardType.Gelbooru) queryList.Add("pid=" + query.Page);
}
if (queryList.Count > 0) queryStr = String.Join("&", queryList);
return queryStr;
}
/// <summary>
/// Get Search Param from Batch Job
/// </summary>
/// <param name="job"></param>
/// <returns></returns>
public DanbooruSearchParam GetSearchParamsFromJob(DanbooruBatchJob job, int currPage)
{
var option = new DanbooruPostDaoOption()
{
BlacklistedTags = TagBlacklist,
BlacklistedTagsRegex = TagBlacklistRegex,
BlacklistedTagsUseRegex = chkBlacklistTagsUseRegex.Checked,
IgnoredTags = TagIgnore,
IgnoredTagsRegex = TagIgnoreRegex,
IgnoredTagsUseRegex = chkIgnoreTagsUseRegex.Checked,
Provider = _currProvider,
SearchTags = !String.IsNullOrWhiteSpace(job.TagQuery) ? job.TagQuery : "",
IsBlacklistOnlyForGeneral = chkBlacklistOnlyGeneral.Checked
};
DanbooruSearchParam searchParam = new DanbooruSearchParam();
searchParam.Provider = option.Provider;
searchParam.Tag = option.SearchTags;
searchParam.Source = "";
// check if given limit is more than the hard limit
if (job.Limit > job.Provider.HardLimit) searchParam.Limit = job.Provider.HardLimit;
else searchParam.Limit = job.Limit;
// reflect to current page
searchParam.Page = job.StartPage + currPage;
searchParam.IsNotRating = false;
searchParam.Rating = job.Rating;
searchParam.OrderBy = "";
searchParam.Option = option;
return searchParam;
}
public BindingList<DanbooruPost> Parse(string data, DanbooruSearchParam searchParam)
{
this.SearchParam = searchParam;
this.RawData = data;
BindingList<DanbooruPost> posts = new BindingList<DanbooruPost>();
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(data);
// remove popular preview
var popular = doc.DocumentNode.SelectSingleNode("//div[@id='popular-preview']");
if (popular != null)
{
popular.Remove();
}
// get all thumbs
var thumbs = doc.DocumentNode.SelectNodes("//span");
if (thumbs != null && thumbs.Count > 0)
{
foreach (var thumb in thumbs)
{
if (thumb.GetAttributeValue("class", "").Contains("thumb"))
{
DanbooruPost post = new DanbooruPost();
post.Id = thumb.GetAttributeValue("id", "_N/A").Substring(1);
post.Provider = searchParam.Provider;
post.SearchTags = searchParam.Tag;
post.Query = GenerateQueryString(searchParam);
int i = 0;
// get the image link
for (; i < thumb.ChildNodes.Count; ++i)
{
if (thumb.ChildNodes[i].Name == "a") break;
}
var a = thumb.ChildNodes[i];
post.Referer = searchParam.Provider.Url + "/" + System.Web.HttpUtility.HtmlDecode(a.GetAttributeValue("href", ""));
if (post.Id == "N/A")
{
post.Id = a.GetAttributeValue("id", "N/A").Substring(1);
}
var img = a.ChildNodes[i];
var title = img.GetAttributeValue("title", "");
var title2 = title.ToString();
post.Tags = title.Substring(0, title.LastIndexOf("rating:")).Trim();
post.Tags = Helper.DecodeEncodedNonAsciiCharacters(post.Tags);
post.TagsEntity = Helper.ParseTags(post.Tags, SearchParam.Provider);
post.Hidden = Helper.CheckBlacklistedTag(post, searchParam.Option);
post.PreviewUrl = img.GetAttributeValue("src", "");
post.PreviewHeight = img.GetAttributeValue("height", 0);
post.PreviewWidth = img.GetAttributeValue("width", 0);
post.Source = "";
post.Score = title.Substring(title.LastIndexOf("score:") + 6);
post.Score = post.Score.Substring(0, post.Score.LastIndexOf(" ")).Trim();
title2 = title2.Substring(title2.LastIndexOf("rating:"));
post.Rating = title2.Substring(7, 1).ToLower();
post.Status = "";
post.MD5 = post.PreviewUrl.Substring(post.PreviewUrl.LastIndexOf("/") + 1);
post.MD5 = post.MD5.Substring(0, post.MD5.LastIndexOf("."));
post.MD5 = post.MD5.Replace("thumbnail_", "");
posts.Add(post);
}
}
}
TotalPost = posts.Count;
if (!SearchParam.Page.HasValue) SearchParam.Page = 0;
Offset = posts.Count * SearchParam.Page;
return posts;
}
public string GenerateQueryString(DanbooruSearchParam query)
{
string tmp = "";
if (!String.IsNullOrWhiteSpace(query.Tag))
{
// convert spaces into '_'
tmp += query.Tag.Replace(' ', '_');
}
if (!String.IsNullOrWhiteSpace(query.Source))
{
if (!string.IsNullOrWhiteSpace(tmp))
{
tmp += "+";
}
tmp += "source:" + query.Source;
}
if (!String.IsNullOrWhiteSpace(query.OrderBy))
{
if (!string.IsNullOrWhiteSpace(tmp))
{
tmp += "+";
}
tmp += "order:" + query.OrderBy;
}
if (!String.IsNullOrWhiteSpace(query.Rating))
{
if (!string.IsNullOrWhiteSpace(tmp))
{
tmp += "+";
}
tmp += "rating:" + query.Rating;
}
if (!string.IsNullOrWhiteSpace(tmp))
{
tmp = "tags=" + tmp;
}
// page
if (query.Page.HasValue)
{
if (!string.IsNullOrWhiteSpace(tmp))
{
tmp += "&";
}
tmp += "page=" + query.Page.Value.ToString();
}
// limit
if (query.Limit.HasValue)
{
if (!string.IsNullOrWhiteSpace(tmp))
{
tmp += "&";
}
tmp += "limit=" + query.Limit.Value.ToString();
}
return tmp;
}
public string GetQueryUrl(DanbooruSearchParam searchParam)
{
var queryStr = GetQueryString(searchParam);
var queryRootUrl = ""; ;
switch (this.Preferred)
{
case PreferredMethod.Xml:
queryRootUrl = this.QueryStringXml;
break;
case PreferredMethod.Json:
queryRootUrl = this.QueryStringJson;
break;
case PreferredMethod.Html:
queryRootUrl = this.QueryStringHtml;
break;
default:
break;
}
queryStr = queryRootUrl.Replace("%_query%", queryStr);
if (this.UseAuth)
{
var hash = this.PasswordHash;
if (String.IsNullOrWhiteSpace(hash))
{
hash = Helper.GeneratePasswordHash(this.Password, this.PasswordSalt);
this.PasswordHash = hash;
}
string authString = "login="******"&password_hash=" + hash;
queryStr = queryStr + "&" + authString;
}
return this.Url + queryStr;
}
public string GetQueryString(DanbooruSearchParam searchParam)
{
var queryStr = "";
if (this.BoardType == BoardType.Shimmie2)
{
queryStr = DanbooruDownloader3.Engine.ShimmieEngine.GetQueryString(this, searchParam);
}
else if (this.Url.Contains("sankakucomplex.com"))
{
queryStr = new SankakuComplexParser().GenerateQueryString(searchParam);
}
else
{
queryStr = DanbooruDownloader3.Engine.DanbooruXmlEngine.GetQueryString(this, searchParam);
}
return queryStr;
}
public string GetQueryUrl(DanbooruSearchParam searchParam)
{
var queryStr = GetQueryString(searchParam);
var queryRootUrl = ""; ;
switch (this.Preferred)
{
case PreferredMethod.Xml:
queryRootUrl = this.QueryStringXml;
break;
case PreferredMethod.Json:
queryRootUrl = this.QueryStringJson;
break;
case PreferredMethod.Html:
queryRootUrl = this.QueryStringHtml;
break;
default:
break;
}
queryStr = queryRootUrl.Replace("%_query%", queryStr);
switch (this.LoginType)
{
case Entity.LoginType.UserPass:
{
var hash = this.PasswordHash;
if (String.IsNullOrWhiteSpace(hash))
{
hash = Helper.GeneratePasswordHash(this.Password, this.PasswordSalt);
this.PasswordHash = hash;
}
string authString = "login="******"&password_hash=" + hash;
queryStr = queryStr + "&" + authString;
}
break;
case Entity.LoginType.Cookie:
// need to inject csv cookie to the webclient
var cookies = Helper.ParseCookie(this.UserName, this.Url);
foreach (var cookie in cookies)
{
ExtendedWebClient.CookieJar.Add(cookie);
}
break;
default:
break;
}
return this.Url + queryStr;
}
public void TestGelbooruParser()
{
DanbooruProviderDao pd = DanbooruProviderDao.GetInstance();
string target = @"../../../DanbooruDownloader3.test/TestXml/gelbooru_post.htm";
var data = File.ReadAllText(target);
var query = new DanbooruSearchParam();
query.Provider = pd.Read(sourceProvider).Where(x => x.Name == "gelbooru.com").First();
query.Tag = "";
query.OrderBy = "score";
var post = new DanbooruPost();
post.Provider = query.Provider;
GelbooruHtmlParser.ParsePost(post, data);
Assert.IsNotNull(post.FileUrl);
Assert.IsTrue(post.FileUrl == @"http://cdn2.gelbooru.com//images/1559/303b7ed1fcba0c1d9520f76ee34ec37e.jpg", "Actual: " + post.FileUrl);
}
public BindingList<DanbooruPost> Parse(string data, DanbooruSearchParam searchParam)
{
try
{
this.SearchParam = searchParam;
this.RawData = data;
BindingList<DanbooruPost> posts = new BindingList<DanbooruPost>();
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(data);
// remove popular preview
var popular = doc.DocumentNode.SelectSingleNode("//div[@id='popular-preview']");
if (popular != null)
{
popular.Remove();
}
// get all thumbs
var thumbs = doc.DocumentNode.SelectNodes("//span");
if (thumbs != null && thumbs.Count > 0)
{
foreach (var thumb in thumbs)
{
if (thumb.GetAttributeValue("class", "").Contains("thumb"))
{
DanbooruPost post = new DanbooruPost();
post.Id = thumb.GetAttributeValue("id", "-1").Substring(1);
post.Provider = searchParam.Provider;
post.SearchTags = searchParam.Tag;
post.Query = GenerateQueryString(searchParam);
int i = 0;
// get the image link
for (; i < thumb.ChildNodes.Count; ++i)
{
if (thumb.ChildNodes[i].Name == "a") break;
}
var a = thumb.ChildNodes[i];
post.Referer = Helper.FixUrl(searchParam.Provider.Url + a.GetAttributeValue("href", ""), isHttps(post.Provider));
var img = a.ChildNodes[i];
var title = img.GetAttributeValue("title", "");
post.Tags = title.Substring(0, title.LastIndexOf("Rating:")).Trim();
post.Tags = Helper.DecodeEncodedNonAsciiCharacters(post.Tags);
post.TagsEntity = Helper.ParseTags(post.Tags, SearchParam.Provider);
post.Hidden = Helper.CheckBlacklistedTag(post, searchParam.Option);
var status = img.GetAttributeValue("class", "").Replace("preview", "").Trim();
if (status.Contains("deleted"))
post.Status = "deleted";
else if (status.Contains("pending"))
post.Status = "pending";
else
post.Status = status;
post.PreviewUrl = Helper.FixUrl(img.GetAttributeValue("src", ""), isHttps(post.Provider));
post.PreviewHeight = img.GetAttributeValue("height", 0);
post.PreviewWidth = img.GetAttributeValue("width", 0);
// Rating:Explicit Score:4.5 Size:1080x1800 User:System
post.Source = "";
post.Score = title.Substring(title.LastIndexOf("Score:") + 6);
post.Score = post.Score.Substring(0, post.Score.IndexOf(" ")).Trim();
string resolution = title.Substring(title.LastIndexOf("Size:") + 5);
resolution = resolution.Substring(0, resolution.IndexOf(" ")).Trim();
string[] resArr = resolution.Split('x');
post.Width = Int32.Parse(resArr[0]);
post.Height = Int32.Parse(resArr[1]);
string rating = title.Substring(title.LastIndexOf("Rating:") + 7, 1);
//rating = rating.Substring(0, rating.IndexOf(" ")).Trim();
post.Rating = rating.ToLower();
post.CreatorId = title.Substring(title.LastIndexOf("User:"******"/") + 1);
post.MD5 = post.MD5.Substring(0, post.MD5.LastIndexOf("."));
posts.Add(post);
}
}
}
TotalPost = posts.Count;
if (!SearchParam.Page.HasValue) SearchParam.Page = 1;
Offset = TotalPost * SearchParam.Page;
return posts;
}
catch (Exception ex)
{
var filename = Helper.SanitizeFilename("Dump for Sankaku Image List - " + searchParam.Tag + " - page " + searchParam.Page + ".txt");
var result = Helper.DumpRawData(data, filename);
if (!result) Program.Logger.Error("Failed to dump rawdata to: " + filename, ex);
throw;
}
}
public string GenerateQueryString(DanbooruSearchParam query)
{
string tmp = "";
if (!String.IsNullOrWhiteSpace(query.Tag))
{
tmp += System.Web.HttpUtility.UrlEncode(query.Tag);
}
if (!String.IsNullOrWhiteSpace(query.Source))
{
if (!string.IsNullOrWhiteSpace(tmp))
{
tmp += "+";
}
tmp += "source:" + query.Source;
}
if (!String.IsNullOrWhiteSpace(query.OrderBy))
{
if (!string.IsNullOrWhiteSpace(tmp))
{
tmp += "+";
}
tmp += "order:" + query.OrderBy;
}
if (!String.IsNullOrWhiteSpace(query.Rating))
{
if (!string.IsNullOrWhiteSpace(tmp))
{
tmp += "+";
}
tmp += "rating:" + query.Rating;
}
if (!string.IsNullOrWhiteSpace(tmp))
{
tmp = "tags=" + tmp;
}
// page
if (query.Page.HasValue)
{
if (!string.IsNullOrWhiteSpace(tmp))
{
tmp += "&";
}
tmp += "page=" + query.Page.Value.ToString();
}
// limit
if (query.Limit.HasValue)
{
if (!string.IsNullOrWhiteSpace(tmp))
{
tmp += "&";
}
tmp += "limit=" + query.Limit.Value.ToString();
}
return tmp;
}
public void TestSankakuParser()
{
DanbooruProviderDao pd = DanbooruProviderDao.GetInstance();
string target = @"../../../DanbooruDownloader3.test/TestXml/sankaku_paging.htm";
var data = File.ReadAllText(target);
var query = new DanbooruSearchParam();
query.Provider = pd.Read(sourceProvider).Where(x => x.Name == "Sankaku Complex").First();
query.Tag = "";
query.OrderBy = "score";
var parser = new SankakuComplexParser();
var result = parser.Parse(data, query);
Assert.IsNotNull(result);
Assert.IsTrue(result.Count == 20, "Count: " + result.Count);
Assert.IsTrue(result[0].Id == "1929657", "Id: " + result[0].Id);
Assert.IsTrue(result[0].Provider.Name == "Sankaku Complex", "Provider: " + result[0].Provider.Name);
Assert.IsTrue(result[0].SearchTags == "", "SearchTags: " + result[0].SearchTags);
Assert.IsTrue(result[0].Query == "tags=order:score", "Query: " + result[0].Query);
Assert.IsTrue(result[0].Tags == "fate_(series) code_geass fate/zero gilgamesh kotomine_kirei 3boys androgynous armlet blonde bracelet brown_hair clamp_(style) cross cross_necklace earrings enkidu_(fate/strange_fake) fate/strange_fake green_eyes green_hair hand_on_own_face jewelry long_hair multiple_boys necklace parody red_eyes ruchi style_parody toga", "Tags: " + result[0].Tags);
Assert.IsTrue(result[0].PreviewUrl == "http://c2.sankakustatic.com/data/preview/85/f5/85f54efd7fea7ba91b20ca09ad5823c7.jpg", "PreviewUrl: " + result[0].PreviewUrl);
Assert.IsTrue(result[0].PreviewHeight == 144, "PreviewHeight: " + result[0].PreviewHeight);
Assert.IsTrue(result[0].PreviewWidth == 150, "PreviewWidth: " + result[0].PreviewWidth);
Assert.IsTrue(result[0].Score == "0.0", "Score: " + result[0].Score);
Assert.IsTrue(result[0].Rating == "s", "Rating: " + result[0].Rating);
}
public string GenerateQueryString(DanbooruSearchParam query)
{
string tmp = "";
if (!String.IsNullOrWhiteSpace(query.Tag))
{
// convert spaces into '_'
tmp += query.Tag.Replace(' ', '_');
}
if (!String.IsNullOrWhiteSpace(query.Source))
{
if (!string.IsNullOrWhiteSpace(tmp))
{
tmp += "+";
}
tmp += "source:" + query.Source;
}
if (!String.IsNullOrWhiteSpace(query.OrderBy))
{
if (!string.IsNullOrWhiteSpace(tmp))
{
tmp += "+";
}
tmp += query.OrderBy;
}
if (!String.IsNullOrWhiteSpace(query.Rating))
{
if (!string.IsNullOrWhiteSpace(tmp))
{
tmp += "+";
}
tmp += query.Rating;
}
if (!string.IsNullOrWhiteSpace(tmp))
{
tmp = "tags=" + tmp;
}
// page
if (query.Page.HasValue)
{
if (!string.IsNullOrWhiteSpace(tmp))
{
tmp += "&";
}
tmp += "page=" + query.Page.Value.ToString();
}
// remove limit counter as it is not useful for sankaku
//// limit
//if (query.Limit.HasValue)
//{
// if (!string.IsNullOrWhiteSpace(tmp))
// {
// tmp += "&";
// }
// tmp += "limit=" + query.Limit.Value.ToString();
//}
return tmp;
}
public BindingList<DanbooruPost> Parse(string data, DanbooruSearchParam query)
{
BindingList<DanbooruPost> list = new BindingList<DanbooruPost>();
XDocument posts = XDocument.Parse(data);
this.RawData = posts.ToString();
Success = true;
var responses = posts.Descendants("response");
if (responses != null && responses.Count() > 0)
{
ResponseMessage = responses.First().Attribute("reason").Value.ToString();
Success = Convert.ToBoolean(responses.First().Attribute("success").Value);
if (!Success)
{
return null;
}
}
this.TotalPost = Convert.ToInt32(posts.Root.Attribute("count").Value);
this.Offset = Convert.ToInt32(posts.Root.Attribute("offset").Value);
string queryStr = GenerateQueryString(query);
foreach (var post in posts.Descendants("post"))
{
DanbooruPost p = new DanbooruPost();
p.Id = post.Attribute("id").Value.ToString();
p.Tags = post.Attribute("tags").Value.ToString();
p.TagsEntity = Helper.ParseTags(p.Tags, SearchParam.Provider);
p.Source = post.Attribute("source").Value.ToString();
p.Score = post.Attribute("score").Value.ToString();
p.Rating = post.Attribute("rating").Value.ToString();
p.FileUrl = AppendHttp(post.Attribute("file_url").Value.ToString(), query.Provider);
p.Width = Convert.ToInt32(post.Attribute("width").Value);
p.Height = Convert.ToInt32(post.Attribute("height").Value);
p.PreviewUrl = AppendHttp(post.Attribute("preview_url").Value.ToString(), query.Provider);
if (post.Attribute("actual_preview_width") != null && // moebooru extension
post.Attribute("actual_preview_height") != null)
{
p.PreviewWidth = Convert.ToInt32(post.Attribute("actual_preview_width").Value);
p.PreviewHeight = Convert.ToInt32(post.Attribute("actual_preview_height").Value);
}
else
{
p.PreviewWidth = Convert.ToInt32(post.Attribute("preview_width").Value);
p.PreviewHeight = Convert.ToInt32(post.Attribute("preview_height").Value);
}
p.SampleUrl = AppendHttp(post.Attribute("sample_url").Value.ToString(), query.Provider);
p.SampleWidth = Convert.ToInt32(post.Attribute("sample_width").Value);
p.SampleHeight = Convert.ToInt32(post.Attribute("sample_height").Value);
// moebooru extension
p.JpegUrl = AppendHttp(post.Attribute("jpeg_url").Value.ToString(), query.Provider);
p.JpegWidth = Convert.ToInt32(post.Attribute("jpeg_width").Value);
p.JpegHeight = Convert.ToInt32(post.Attribute("jpeg_height").Value);
p.Filesize = Convert.ToInt32(post.Attribute("file_size").Value);
p.Status = post.Attribute("status").Value.ToString();
p.HasChildren = Convert.ToBoolean(post.Attribute("has_children").Value);
p.ParentId = post.Attribute("parent_id").Value.ToString();
p.Change = post.Attribute("change").Value.ToString();
p.CreatorId = post.Attribute("creator_id").Value.ToString();
p.CreatedAt = post.Attribute("created_at").Value.ToString();
p.MD5 = post.Attribute("md5").Value.ToString();
p.Provider = query.Provider;
p.Query = queryStr;
p.SearchTags = query.Tag;
p.Referer = query.Provider.Url + @"/post/show/" + p.Id;
list.Add(p);
}
return list;
}
