public void TestSankakuParser()
{
DanbooruProviderDao pd = DanbooruProviderDao.GetInstance();
string target = @"../../../DanbooruDownloader3.test/TestXml/sankaku_paging.htm";
var data = File.ReadAllText(target);
var query = new DanbooruSearchParam();
query.Provider = pd.Read(sourceProvider).Where(x => x.Name == "Sankaku Complex").First();
query.Tag = "";
query.OrderBy = "score";
var parser = new SankakuComplexParser();
var result = parser.Parse(data, query);
Assert.IsNotNull(result);
Assert.IsTrue(result.Count == 20, "Count: " + result.Count);
Assert.IsTrue(result[0].Id == "1929657", "Id: " + result[0].Id);
Assert.IsTrue(result[0].Provider.Name == "Sankaku Complex", "Provider: " + result[0].Provider.Name);
Assert.IsTrue(result[0].SearchTags == "", "SearchTags: " + result[0].SearchTags);
Assert.IsTrue(result[0].Query == "tags=order:score", "Query: " + result[0].Query);
Assert.IsTrue(result[0].Tags == "fate_(series) code_geass fate/zero gilgamesh kotomine_kirei 3boys androgynous armlet blonde bracelet brown_hair clamp_(style) cross cross_necklace earrings enkidu_(fate/strange_fake) fate/strange_fake green_eyes green_hair hand_on_own_face jewelry long_hair multiple_boys necklace parody red_eyes ruchi style_parody toga", "Tags: " + result[0].Tags);
Assert.IsTrue(result[0].PreviewUrl == "http://c2.sankakustatic.com/data/preview/85/f5/85f54efd7fea7ba91b20ca09ad5823c7.jpg", "PreviewUrl: " + result[0].PreviewUrl);
Assert.IsTrue(result[0].PreviewHeight == 144, "PreviewHeight: " + result[0].PreviewHeight);
Assert.IsTrue(result[0].PreviewWidth == 150, "PreviewWidth: " + result[0].PreviewWidth);
Assert.IsTrue(result[0].Score == "0.0", "Score: " + result[0].Score);
Assert.IsTrue(result[0].Rating == "s", "Rating: " + result[0].Rating);
}
public void TestSankakuTagParser()
{
string target = @"../../../DanbooruDownloader3.test/TestXml/sankakutagspage.htm";
var data = File.ReadAllText(target);
var parser = new SankakuComplexParser();
var result = parser.parseTagsPage(data, 1);
Assert.IsNotNull(result);
Assert.IsTrue(result.Tag.Length == 50, "Count: " + result.Tag.Length);
List <DanbooruTag> newTagList = new List <DanbooruTag>();
target = @"../../../DanbooruDownloader3.test/TestXml/sankakutagspage-invalid.htm";
data = File.ReadAllText(target);
parser = new SankakuComplexParser();
result = parser.parseTagsPage(data, 1);
Assert.IsNotNull(result);
Assert.IsTrue(result.Tag.Length == 50, "Count: " + result.Tag.Length);
var filename = "dummy.xml";
DanbooruTagsDao.Save(filename, result.Tag.ToList());
DanbooruTagsDao dao = new DanbooruTagsDao(filename);
}
public string GetQueryString(DanbooruSearchParam searchParam)
{
var queryStr = "";
if (BoardType == BoardType.Shimmie2)
{
if (Preferred == PreferredMethod.Html)
{
queryStr = new ShimmieHtmlParser().GenerateQueryString(searchParam);
}
else
{
queryStr = ShimmieEngine.GetQueryString(this, searchParam);
}
}
else if (Url.Contains("sankakucomplex.com"))
{
queryStr = new SankakuComplexParser().GenerateQueryString(searchParam);
}
else
{
queryStr = DanbooruXmlEngine.GetQueryString(this, searchParam);
}
return(queryStr);
}
/// <summary>
/// parse xml/json list stream and close it.
/// </summary>
/// <param name="input"></param>
/// <param name="option"></param>
public DanbooruPostDao(Stream input, DanbooruPostDaoOption option, int?currentPage)
{
string rawData = "";
this.Option = option;
try
{
using (StreamReader reader = new StreamReader(input))
{
rawData = reader.ReadToEnd();
}
this.Option = option;
switch (option.Provider.Preferred)
{
case PreferredMethod.Xml:
ReadXML(rawData, option);
break;
case PreferredMethod.Json:
ReadJSON(rawData, option);
break;
case PreferredMethod.Html:
DanbooruSearchParam param = new DanbooruSearchParam()
{
Provider = option.Provider,
Tag = option.SearchTags,
Page = currentPage,
Option = option
};
if (option.Provider.BoardType == BoardType.Danbooru)
{
SankakuComplexParser parser = new SankakuComplexParser();
posts = parser.Parse(rawData, param);
NextId = param.NextKey;
if (parser.TotalPost.HasValue)
{
postCount = parser.TotalPost.Value;
}
}
else if (option.Provider.BoardType == BoardType.Gelbooru)
{
GelbooruHtmlParser parser = new GelbooruHtmlParser();
posts = parser.Parse(rawData, param);
}
else
{
throw new NotImplementedException("No HTML Parser for: " + option.Provider.Name);
}
break;
}
}
catch (Exception)
{
Helper.DumpRawData(rawData, Option.Provider, option.Query);
throw;
}
}
public string GetQueryString(DanbooruSearchParam searchParam)
{
var queryStr = "";
if (this.BoardType == BoardType.Shimmie2)
{
queryStr = DanbooruDownloader3.Engine.ShimmieEngine.GetQueryString(this, searchParam);
}
else if (this.Url.Contains("sankakucomplex.com"))
{
queryStr = new SankakuComplexParser().GenerateQueryString(searchParam);
}
else
{
queryStr = DanbooruDownloader3.Engine.DanbooruXmlEngine.GetQueryString(this, searchParam);
}
return(queryStr);
}
private List <DanbooruTag> CombineLoopTag(int lastPage)
{
// construct the tag list from each xml
List <DanbooruTag> newTagList = new List <DanbooruTag>();
for (int i = 1; i < lastPage; ++i)
{
Program.Logger.Debug("[Download Tags] Merging Page: " + i);
lblStatus.Text = "Status: Merging Page: " + i + ", this might take some times.";
lblStatus.Invalidate();
lblStatus.Update();
lblStatus.Refresh();
Application.DoEvents();
string tempName = TAGS_FILENAME + "." + i + ".!tmp";
DanbooruTagCollection tempTag;
if (isSankaku)
{
SankakuComplexParser parser = new SankakuComplexParser();
tempTag = parser.parseTagsPage(File.ReadAllText(tempName), i);
}
else
{
tempTag = new DanbooruTagsDao(tempName).Tags;
}
if (tempTag != null && tempTag.Tag != null)
{
newTagList.AddRange(tempTag.Tag);
}
}
for (int i = 1; i < lastPage; ++i)
{
string tempName = TAGS_FILENAME + "." + i + ".!tmp";
File.Delete(tempName);
}
return(newTagList);
}
private void HandleLoop(string tempName)
{
DanbooruTagCollection tempTags = null;
if (isSankaku)
{
string data = File.ReadAllText(tempName);
if (!string.IsNullOrWhiteSpace(data))
{
tempTags = new SankakuComplexParser().parseTagsPage(data, Page);
}
else
{
string message = "Got empty response!";
Program.Logger.Error(message);
++retry;
if (retry > Int32.Parse(DanbooruDownloader3.Properties.Settings.Default.retry))
{
WaitForDelay(message);
ProcessLoop(--Page);
}
else
{
MessageBox.Show(message);
}
}
}
else
{
tempTags = new DanbooruTagsDao(tempName).Tags;
}
if (tempTags == null || tempTags.Tag == null || tempTags.Tag.Length == 0 ||
(prevTag != null && prevTag.Tag.Last().Name == tempTags.Tag.Last().Name))
{
// no more tags
chkUseLoop.Enabled = true;
var newTagList = CombineLoopTag(Page);
if (SelectedProvider != null)
{
string targetXml = "tags-" + SelectedProvider.Name + ".xml";
if (File.Exists(targetXml))
{
if (chkBackup.Checked)
{
string backupName = targetXml + ".bak";
if (File.Exists(backupName))
{
File.Delete(backupName);
}
File.Move(targetXml, backupName);
}
else
{
File.Delete(targetXml);
}
}
DanbooruTagsDao.Save(targetXml, newTagList);
SelectedProvider.LoadProviderTagCollection();
Program.Logger.Info(String.Format("[Download Tags] Private Tags.xml saved to {0}.", targetXml));
}
if (chkMerge.Checked)
{
Program.Logger.Debug("[Download Tags] Merging Old Tags.");
lblStatus.Text = "Status: Merging Old Tags, this might take some times.";
lblStatus.Invalidate();
lblStatus.Update();
lblStatus.Refresh();
Application.DoEvents();
DanbooruTagsDao.Save(TAGS_FILENAME + ".merge", newTagList);
var message = DanbooruTagsDao.Merge(TAGS_FILENAME + ".merge", TAGS_FILENAME);
Program.Logger.Info("[Download Tags] " + message);
MessageBox.Show(message, "Tags.xml merged.");
File.Delete(TAGS_FILENAME + ".merge");
}
else
{
// write back to TAGS_FILENAME
DanbooruTagsDao.Save(TAGS_FILENAME, newTagList);
}
DanbooruTagsDao.Instance = new DanbooruTagsDao(TAGS_FILENAME);
Program.Logger.Info("[Download Tags] Complete.");
lblStatus.Text = "Status: Download complete.";
if (chkAutoClose.Checked)
{
this.Close();
}
}
else
{
// continue next page
ProcessLoop(Page);
prevTag = tempTags;
}
}
/// <summary>
/// parse xml/json list stream and close it.
/// </summary>
/// <param name="input"></param>
/// <param name="option"></param>
public DanbooruPostDao(Stream input, DanbooruPostDaoOption option, int?currentPage)
{
string rawData = "";
this.Option = option;
try
{
using (StreamReader reader = new StreamReader(input))
{
rawData = reader.ReadToEnd();
}
this.Option = option;
if (rawData.Contains("DDoS protection by <a rel=\"noopener noreferrer\" href=\"https://www.cloudflare.com/5xx-error-landing/\" target=\"_blank\">Cloudflare</a>"))
{
throw new NotImplementedException($"Cloudflare DDoS protection enabled for : {option.Provider.Name}, please use the cookie method to access.");
}
switch (option.Provider.Preferred)
{
case PreferredMethod.Xml:
ReadXML(rawData, option);
break;
case PreferredMethod.Json:
ReadJSON(rawData, option);
break;
case PreferredMethod.Html:
DanbooruSearchParam param = new DanbooruSearchParam()
{
Provider = option.Provider,
Tag = option.SearchTags,
Page = currentPage,
Option = option
};
if (option.Provider.BoardType == BoardType.Danbooru)
{
SankakuComplexParser parser = new SankakuComplexParser();
posts = parser.Parse(rawData, param);
NextId = param.NextKey;
if (parser.TotalPost.HasValue)
{
postCount = parser.TotalPost.Value;
}
}
else if (option.Provider.BoardType == BoardType.Gelbooru)
{
GelbooruHtmlParser parser = new GelbooruHtmlParser();
posts = parser.Parse(rawData, param);
}
else if (option.Provider.BoardType == BoardType.Shimmie2)
{
var parser = new ShimmieHtmlParser();
posts = parser.Parse(rawData, param);
}
else
{
throw new NotImplementedException("No HTML Parser for: " + option.Provider.Name);
}
break;
}
}
catch (Exception)
{
Helper.DumpRawData(rawData, Option.Provider, option.Query);
throw;
}
}
