public void TestShimmie2Parser()
{
var shimmie2Provider = DanbooruProviderDao.GetInstance().Read(sourceProvider).Where <DanbooruProvider>(x => x.BoardType == BoardType.Shimmie2).First <DanbooruProvider>();
var xml = "";
using (StreamReader reader = new StreamReader(@"../../../DanbooruDownloader3.test/TestXml/shimmie2.xml"))
{
xml = reader.ReadToEnd();
}
DanbooruPostDaoOption option = new DanbooruPostDaoOption()
{
Provider = shimmie2Provider,
Query = "",
SearchTags = "",
BlacklistedTags = null,
BlacklistedTagsRegex = new Regex("$^"),
BlacklistedTagsUseRegex = false,
IsBlacklistOnlyForGeneral = false,
};
var list = ShimmieEngine.ParseRSS(xml, option);
Assert.IsNotNull(list);
Assert.IsTrue(list.Count == 9);
}
private void clientList_DownloadFileCompleted(object sender, AsyncCompletedEventArgs e)
{
tsProgressBar.Visible = false;
UpdateLog("clientList_DownloadFileCompleted", "Download Complete: " + e.UserState);
txtListFile.Text = saveFileDialog1.FileName;
gbxSearch.Enabled = true;
gbxList.Enabled = true;
btnGet.Enabled = true;
btnListCancel.Enabled = false;
if (chkAutoLoadList.Checked)
{
DanbooruPostDaoOption option = new DanbooruPostDaoOption()
{
Provider = _currProvider,
Url = txtListFile.Text,
Referer = _clientList.Referer,
Query = txtListFile.Text.Split('\\').Last(),
SearchTags = "",
BlacklistedTags = TagBlacklist,
BlacklistedTagsRegex = TagBlacklistRegex,
BlacklistedTagsUseRegex = chkBlacklistTagsUseRegex.Checked,
IgnoredTags = TagIgnore,
IgnoredTagsRegex = TagIgnoreRegex,
IgnoredTagsUseRegex = chkIgnoreTagsUseRegex.Checked
};
DanbooruPostDao newPosts = new DanbooruPostDao(option);
LoadList(newPosts);
}
_isLoadingList = false;
}
/// <summary>
/// parse xml/json list stream and close it.
/// </summary>
/// <param name="input"></param>
/// <param name="option"></param>
public DanbooruPostDao(Stream input, DanbooruPostDaoOption option, int?currentPage)
{
string rawData = "";
this.Option = option;
try
{
using (StreamReader reader = new StreamReader(input))
{
rawData = reader.ReadToEnd();
}
this.Option = option;
switch (option.Provider.Preferred)
{
case PreferredMethod.Xml:
ReadXML(rawData, option);
break;
case PreferredMethod.Json:
ReadJSON(rawData, option);
break;
case PreferredMethod.Html:
DanbooruSearchParam param = new DanbooruSearchParam()
{
Provider = option.Provider,
Tag = option.SearchTags,
Page = currentPage,
Option = option
};
if (option.Provider.BoardType == BoardType.Danbooru)
{
SankakuComplexParser parser = new SankakuComplexParser();
posts = parser.Parse(rawData, param);
NextId = param.NextKey;
if (parser.TotalPost.HasValue)
{
postCount = parser.TotalPost.Value;
}
}
else if (option.Provider.BoardType == BoardType.Gelbooru)
{
GelbooruHtmlParser parser = new GelbooruHtmlParser();
posts = parser.Parse(rawData, param);
}
else
{
throw new NotImplementedException("No HTML Parser for: " + option.Provider.Name);
}
break;
}
}
catch (Exception)
{
Helper.DumpRawData(rawData, Option.Provider, option.Query);
throw;
}
}
private void ReadXML(string rawData, DanbooruPostDaoOption option)
{
posts = new BindingList <DanbooruPost>();
actualCount = 0;
// Issue #60
// modify xml to insert html entity
rawData = Regex.Replace(rawData, @"(<\?xml.*\?>)", "$1<!DOCTYPE document SYSTEM \"xhtml.ent\">");
ProcessXML(rawData);
}
//int currFileRetry = 0;
private void backgroundWorker1_DoWork(object sender, DoWorkEventArgs e)
{
BackgroundWorker worker = sender as BackgroundWorker;
object[] param = (object[])e.Argument;
Stream ms = (Stream)param[0];
DanbooruPostDaoOption option = (DanbooruPostDaoOption)param[1];
DanbooruPostDao newPosts = new DanbooruPostDao(ms, option);
e.Result = newPosts;
}
public void ReadJSON(string rawData, DanbooruPostDaoOption option)
{
posts = new BindingList <DanbooruPost>();
actualCount = 0;
using (StringReader reader = new StringReader(rawData))
{
ProcessJson(rawData);
//RawData = json;
RawData = rawData;
}
}
/// <summary>
/// parse xml/json list file
/// </summary>
/// <param name="option"></param>
public DanbooruPostDao(DanbooruPostDaoOption option)
{
this.Option = option;
if (option.Url.ToLower().EndsWith(".xml"))
{
ReadXML(option.Url);
}
else
{
ReadJSON(option.Url);
}
}
private static void ReadRssMethod2(DanbooruPostDaoOption option, BindingList <DanbooruPost> posts, string xmldoc)
{
XmlReaderSettings readerSettings = new XmlReaderSettings
{
DtdProcessing = DtdProcessing.Parse,
XmlResolver = new LocalXhtmlXmlResolver()
};
using (StringReader strReader = new StringReader(xmldoc))
{
using (XmlReader reader = XmlReader.Create(strReader, readerSettings))
{
XDocument doc = XDocument.Load(reader);
var feeds = doc.Descendants("item");
XNamespace dc = "http://purl.org/dc/elements/1.1/";
foreach (var item in feeds)
{
DanbooruPost post = new DanbooruPost();
var titleData = item.Element("title").Value.Split(new char[] { '-' }, 2);
post.Id = titleData[0].Trim();
post.Tags = titleData[1].Trim();
post.Referer = AppendHttp(item.Element("link").Value, option.Provider);
post.CreatedAt = item.Element("pubDate").Value;
post.CreatorId = item.Element(dc + "creator").Value;
post.FileUrl = item.Element("enclosure").Attribute("url").Value;
try
{
HtmlAgilityPack.HtmlDocument description = new HtmlAgilityPack.HtmlDocument();
description.LoadHtml(HttpUtility.HtmlDecode(item.Element("description").Value));
//post.Width = Convert.ToInt32(matches.Groups[1].Value);
//post.Height = Convert.ToInt32(matches.Groups[2].Value);
var img = description.DocumentNode.SelectSingleNode("//img");
post.PreviewWidth = Convert.ToInt32(img.Attributes["width"].Value);
post.PreviewHeight = Convert.ToInt32(img.Attributes["height"].Value);
post.PreviewUrl = img.Attributes["src"].Value;
}
catch (Exception) { }
PostProcess(option, post);
posts.Add(post);
}
}
}
}
private static void ReadRssMethod1(DanbooruPostDaoOption option, BindingList <DanbooruPost> posts, string xmldoc)
{
XmlReaderSettings readerSettings = new XmlReaderSettings
{
DtdProcessing = DtdProcessing.Parse,
XmlResolver = new LocalXhtmlXmlResolver()
};
using (StringReader strReader = new StringReader(xmldoc))
{
using (XmlReader reader = XmlReader.Create(strReader, readerSettings))
{
XDocument doc = XDocument.Load(reader);
string media = doc.Root.Attribute("{http://www.w3.org/2000/xmlns/}media").Value;
foreach (var item in doc.Descendants("item"))
{
DanbooruPost post = new DanbooruPost();
var titleData = item.Element("title").Value.Split(new char[] { '-' }, 2);
post.Id = titleData[0].Trim();
post.Tags = titleData[1].Trim();
post.Referer = AppendHttp(item.Element("link").Value, option.Provider);
post.CreatedAt = item.Element("pubDate").Value;
var data = item.Element("{" + media + "}thumbnail");
post.PreviewUrl = AppendHttp(data.Attribute("url").Value, option.Provider);
data = item.Element("{" + media + "}content");
post.FileUrl = AppendHttp(data.Attribute("url").Value, option.Provider);
try
{
string description = HttpUtility.HtmlDecode(item.Element("description").Value);
Match matches = imageResolutionRegex.Match(description);
post.Width = Convert.ToInt32(matches.Groups[1].Value);
post.Height = Convert.ToInt32(matches.Groups[2].Value);
post.PreviewWidth = Convert.ToInt32(matches.Groups[3].Value);
post.PreviewHeight = Convert.ToInt32(matches.Groups[4].Value);
post.CreatorId = matches.Groups[5].Value;
}
catch (Exception) { }
PostProcess(option, post);
posts.Add(post);
}
}
}
}
public static BindingList <DanbooruPost> ParseRSS(string xmldoc, DanbooruPostDaoOption option)
{
BindingList <DanbooruPost> posts = new BindingList <DanbooruPost>();
try
{
ReadRssMethod1(option, posts, xmldoc);
}
catch (Exception ex)
{
Program.Logger.Error("Using method2", ex);
ReadRssMethod2(option, posts, xmldoc);
}
return(posts);
}
private static void PostProcess(DanbooruPostDaoOption option, DanbooruPost post)
{
post.TagsEntity = Helper.ParseTags(post.Tags, option.Provider);
if (option.BlacklistedTagsUseRegex)
{
post.Hidden = IsBlacklisted(post, option.BlacklistedTagsRegex);
}
else
{
post.Hidden = IsBlacklisted(post, option.BlacklistedTags);
}
post.Query = option.Query;
post.SearchTags = option.SearchTags;
post.Provider = option.Provider;
}
public static bool CheckBlacklistedTag(DanbooruPost post, DanbooruPostDaoOption option)
{
if (option.BlacklistedTagsUseRegex)
{
return(post.TagsEntity.Any(x => option.BlacklistedTagsRegex.IsMatch(x.Name)));
}
else
{
foreach (var tag in option.BlacklistedTags)
{
if (post.TagsEntity.Any(x => x.Name.Equals(tag.Name, StringComparison.InvariantCultureIgnoreCase)))
{
return(true);
}
}
}
return(false);
}
/// <summary>
/// Get Search Param from Batch Job
/// </summary>
/// <param name="job"></param>
/// <returns></returns>
public DanbooruSearchParam GetSearchParamsFromJob(DanbooruBatchJob job)
{
var option = new DanbooruPostDaoOption()
{
BlacklistedTags = TagBlacklist,
BlacklistedTagsRegex = TagBlacklistRegex,
BlacklistedTagsUseRegex = chkBlacklistTagsUseRegex.Checked,
IgnoredTags = TagIgnore,
IgnoredTagsRegex = TagIgnoreRegex,
IgnoredTagsUseRegex = chkIgnoreTagsUseRegex.Checked,
Provider = _currProvider,
SearchTags = !String.IsNullOrWhiteSpace(job.TagQuery) ? job.TagQuery : "",
IsBlacklistOnlyForGeneral = chkBlacklistOnlyGeneral.Checked
};
DanbooruSearchParam searchParam = new DanbooruSearchParam
{
Provider = option.Provider,
Tag = option.SearchTags,
Source = ""
};
// check if given limit is more than the hard limit
if (job.Limit > job.Provider.HardLimit)
{
searchParam.Limit = job.Provider.HardLimit;
}
else
{
searchParam.Limit = job.Limit;
}
// reflect to current page
searchParam.Page = job.StartPage + job.CalculatedCurrentPage;
searchParam.IsNotRating = false;
searchParam.Rating = job.Rating;
searchParam.OrderBy = "";
searchParam.Option = option;
return(searchParam);
}
/// <summary>
/// Get search parameter from Main Tab and Option Panels
/// </summary>
/// <returns></returns>
public DanbooruSearchParam GetSearchParams()
{
var option = new DanbooruPostDaoOption()
{
BlacklistedTags = TagBlacklist,
BlacklistedTagsRegex = TagBlacklistRegex,
BlacklistedTagsUseRegex = chkBlacklistTagsUseRegex.Checked,
IgnoredTags = TagIgnore,
IgnoredTagsRegex = TagIgnoreRegex,
IgnoredTagsUseRegex = chkIgnoreTagsUseRegex.Checked,
Provider = _currProvider,
SearchTags = !String.IsNullOrWhiteSpace(txtTags.Text) ? txtTags.Text : "",
IsBlacklistOnlyForGeneral = chkBlacklistOnlyGeneral.Checked
};
DanbooruSearchParam searchParam = new DanbooruSearchParam
{
Provider = option.Provider,
Tag = option.SearchTags,
Source = txtSource.Text.Trim()
};
if (Int32.TryParse(txtLimit.Text, out int limit) && limit > 0)
{
searchParam.Limit = limit;
}
else
{
searchParam.Limit = null;
}
int page = _currProvider.BoardType == BoardType.Gelbooru ? 0 : 1;
if (Int32.TryParse(txtPage.Text, out page) && page > 0)
{
searchParam.Page = page;
}
else
{
searchParam.Page = null;
}
searchParam.IsNotRating = chkNotRating.Checked;
if (cbxRating.SelectedValue != null)
{
if (cbxRating.SelectedValue.GetType() == typeof(string))
{
searchParam.Rating = cbxRating.SelectedValue.ToString();
}
else
{
var rating = (KeyValuePair <string, string>)cbxRating.SelectedValue;
searchParam.Rating = rating.Value;
}
}
if (cbxOrder.SelectedValue != null)
{
if (cbxOrder.SelectedValue.GetType() == typeof(string))
{
searchParam.OrderBy = cbxOrder.SelectedValue.ToString();
}
else
{
var order = (KeyValuePair <string, string>)cbxOrder.SelectedValue;
searchParam.OrderBy = order.Value;
}
}
searchParam.Option = option;
searchParam.NextKey = _lastId;
return(searchParam);
}
private void clientList_DownloadDataCompleted(object sender, DownloadDataCompletedEventArgs e)
{
try
{
Program.Logger.Debug("Download list completed");
tsProgressBar.Visible = false;
MemoryStream ms = new MemoryStream(e.Result);
DanbooruPostDaoOption option = new DanbooruPostDaoOption()
{
Provider = _currProvider,
//Url = txtListFile.Text,
Referer = _clientList.Referer,
Query = txtQuery.Text,
SearchTags = txtTags.Text,
BlacklistedTags = TagBlacklist,
BlacklistedTagsRegex = TagBlacklistRegex,
BlacklistedTagsUseRegex = chkBlacklistTagsUseRegex.Checked,
IgnoredTags = TagIgnore,
IgnoredTagsRegex = TagIgnoreRegex,
IgnoredTagsUseRegex = chkIgnoreTagsUseRegex.Checked
};
tsStatus.Text = "Loading downloaded list...";
tsProgressBar.Style = ProgressBarStyle.Marquee;
tsProgressBar.Visible = true;
_isLoadingList = true;
backgroundWorker1 = new BackgroundWorker();
backgroundWorker1.DoWork += new DoWorkEventHandler(backgroundWorker1_DoWork);
backgroundWorker1.RunWorkerCompleted += new RunWorkerCompletedEventHandler(backgroundWorker1_RunWorkerCompleted);
backgroundWorker1.RunWorkerAsync(new object[] { ms, option });
}
catch (Exception ex)
{
var message = ex.Message;
if (ex.InnerException != null)
{
message = ex.InnerException.Message;
var wex = ex.InnerException as System.Net.WebException;
if (wex != null && wex.Status == WebExceptionStatus.ProtocolError)
{
var status = wex.Response != null ? wex.Response.Headers["Status"] : null;
var response = wex.Response as HttpWebResponse;
if (status == "403" || (response != null && response.StatusCode == HttpStatusCode.Forbidden))
{
message += Environment.NewLine + "Please check your login information.";
}
else
{
using (var responseStream = wex.Response.GetResponseStream())
{
if (responseStream != null)
{
DanbooruPostDaoOption option = new DanbooruPostDaoOption()
{
Provider = _currProvider,
//Url = "",
Referer = "",
Query = "",
SearchTags = "",
BlacklistedTags = TagBlacklist,
BlacklistedTagsRegex = TagBlacklistRegex,
BlacklistedTagsUseRegex = chkBlacklistTagsUseRegex.Checked,
IgnoredTags = TagIgnore,
IgnoredTagsRegex = TagIgnoreRegex,
IgnoredTagsUseRegex = chkIgnoreTagsUseRegex.Checked
};
var resp = new DanbooruPostDao(responseStream, option);
message = "Server Message: " + resp.ResponseMessage;
if (status != "200")
{
message += "\nStatus Code: " + wex.Status.ToString() + " (" + status + ")";
}
}
}
}
}
}
MessageBox.Show(message, "Download List");
Program.Logger.Error(message, ex);
UpdateLog("clientList_DownloadDataCompleted", "Error: " + (ex.InnerException == null ? ex.Message : ex.InnerException.Message), ex);
UpdateLog("clientList_DownloadDataCompleted", "Referer: " + _clientList.Referer);
chkAutoLoadNext.Checked = false;
btnGet.Enabled = true;
btnListCancel.Enabled = false;
_isLoadingList = false;
}
}
/// <summary>
/// parse xml/json list stream and close it.
/// </summary>
/// <param name="input"></param>
/// <param name="option"></param>
public DanbooruPostDao(Stream input, DanbooruPostDaoOption option, int?currentPage)
{
string rawData = "";
this.Option = option;
try
{
using (StreamReader reader = new StreamReader(input))
{
rawData = reader.ReadToEnd();
}
this.Option = option;
if (rawData.Contains("DDoS protection by <a rel=\"noopener noreferrer\" href=\"https://www.cloudflare.com/5xx-error-landing/\" target=\"_blank\">Cloudflare</a>"))
{
throw new NotImplementedException($"Cloudflare DDoS protection enabled for : {option.Provider.Name}, please use the cookie method to access.");
}
switch (option.Provider.Preferred)
{
case PreferredMethod.Xml:
ReadXML(rawData, option);
break;
case PreferredMethod.Json:
ReadJSON(rawData, option);
break;
case PreferredMethod.Html:
DanbooruSearchParam param = new DanbooruSearchParam()
{
Provider = option.Provider,
Tag = option.SearchTags,
Page = currentPage,
Option = option
};
if (option.Provider.BoardType == BoardType.Danbooru)
{
SankakuComplexParser parser = new SankakuComplexParser();
posts = parser.Parse(rawData, param);
NextId = param.NextKey;
if (parser.TotalPost.HasValue)
{
postCount = parser.TotalPost.Value;
}
}
else if (option.Provider.BoardType == BoardType.Gelbooru)
{
GelbooruHtmlParser parser = new GelbooruHtmlParser();
posts = parser.Parse(rawData, param);
}
else if (option.Provider.BoardType == BoardType.Shimmie2)
{
var parser = new ShimmieHtmlParser();
posts = parser.Parse(rawData, param);
}
else
{
throw new NotImplementedException("No HTML Parser for: " + option.Provider.Name);
}
break;
}
}
catch (Exception)
{
Helper.DumpRawData(rawData, Option.Provider, option.Query);
throw;
}
}
private void ReadXML(string rawData, DanbooruPostDaoOption option)
{
posts = new BindingList <DanbooruPost>();
actualCount = 0;
ProcessXML(rawData);
}