/// <summary> /// parse xml/json list stream and close it. /// </summary> /// <param name="input"></param> /// <param name="option"></param> public DanbooruPostDao(Stream input, DanbooruPostDaoOption option) { string rawData = ""; this.Option = option; try { using (StreamReader reader = new StreamReader(input)) { rawData = reader.ReadToEnd(); } this.Option = option; switch (option.Provider.Preferred) { case PreferredMethod.Xml: ReadXML(rawData, option); break; case PreferredMethod.Json: ReadJSON(rawData, option); break; case PreferredMethod.Html: DanbooruSearchParam param = new DanbooruSearchParam() { Provider = option.Provider, Tag = option.SearchTags, Option = option }; if (option.Provider.BoardType == BoardType.Danbooru) { SankakuComplexParser parser = new SankakuComplexParser(); posts = parser.Parse(rawData, param); } else if (option.Provider.BoardType == BoardType.Gelbooru) { GelbooruHtmlParser parser = new GelbooruHtmlParser(); posts = parser.Parse(rawData, param); } else { throw new NotImplementedException("No HTML Parser for: " + option.Provider.Name); } break; } } catch (Exception) { Helper.DumpRawData(rawData, Option.Provider, option.Query); throw; } }
public string GetQueryString(DanbooruSearchParam searchParam) { var queryStr = ""; if (this.BoardType == BoardType.Shimmie2) { queryStr = DanbooruDownloader3.Engine.ShimmieEngine.GetQueryString(this, searchParam); } else if (this.Url.Contains("sankakucomplex.com")) { queryStr = new SankakuComplexParser().GenerateQueryString(searchParam); } else { queryStr = DanbooruDownloader3.Engine.DanbooruXmlEngine.GetQueryString(this, searchParam); } return queryStr; }
public void TestSankakuTagParser() { string target = @"../../../DanbooruDownloader3.test/TestXml/sankakutagspage.htm"; var data = File.ReadAllText(target); var parser = new SankakuComplexParser(); var result = parser.parseTagsPage(data, 1); Assert.IsNotNull(result); Assert.IsTrue(result.Tag.Length == 50, "Count: " + result.Tag.Length); List<DanbooruTag> newTagList = new List<DanbooruTag>(); target = @"../../../DanbooruDownloader3.test/TestXml/sankakutagspage-invalid.htm"; data = File.ReadAllText(target); parser = new SankakuComplexParser(); result = parser.parseTagsPage(data, 1); Assert.IsNotNull(result); Assert.IsTrue(result.Tag.Length == 50, "Count: " + result.Tag.Length); var filename = "dummy.xml"; DanbooruTagsDao.Save(filename, result.Tag.ToList()); DanbooruTagsDao dao = new DanbooruTagsDao(filename); }
public void TestSankakuParser() { DanbooruProviderDao pd = DanbooruProviderDao.GetInstance(); string target = @"../../../DanbooruDownloader3.test/TestXml/sankaku_paging.htm"; var data = File.ReadAllText(target); var query = new DanbooruSearchParam(); query.Provider = pd.Read(sourceProvider).Where(x => x.Name == "Sankaku Complex").First(); query.Tag = ""; query.OrderBy = "score"; var parser = new SankakuComplexParser(); var result = parser.Parse(data, query); Assert.IsNotNull(result); Assert.IsTrue(result.Count == 20, "Count: " + result.Count); Assert.IsTrue(result[0].Id == "1929657", "Id: " + result[0].Id); Assert.IsTrue(result[0].Provider.Name == "Sankaku Complex", "Provider: " + result[0].Provider.Name); Assert.IsTrue(result[0].SearchTags == "", "SearchTags: " + result[0].SearchTags); Assert.IsTrue(result[0].Query == "tags=order:score", "Query: " + result[0].Query); Assert.IsTrue(result[0].Tags == "fate_(series) code_geass fate/zero gilgamesh kotomine_kirei 3boys androgynous armlet blonde bracelet brown_hair clamp_(style) cross cross_necklace earrings enkidu_(fate/strange_fake) fate/strange_fake green_eyes green_hair hand_on_own_face jewelry long_hair multiple_boys necklace parody red_eyes ruchi style_parody toga", "Tags: " + result[0].Tags); Assert.IsTrue(result[0].PreviewUrl == "http://c2.sankakustatic.com/data/preview/85/f5/85f54efd7fea7ba91b20ca09ad5823c7.jpg", "PreviewUrl: " + result[0].PreviewUrl); Assert.IsTrue(result[0].PreviewHeight == 144, "PreviewHeight: " + result[0].PreviewHeight); Assert.IsTrue(result[0].PreviewWidth == 150, "PreviewWidth: " + result[0].PreviewWidth); Assert.IsTrue(result[0].Score == "0.0", "Score: " + result[0].Score); Assert.IsTrue(result[0].Rating == "s", "Rating: " + result[0].Rating); }
private List<DanbooruTag> CombineLoopTag(int lastPage) { // construct the tag list from each xml List<DanbooruTag> newTagList = new List<DanbooruTag>(); for (int i = 1; i < lastPage; ++i) { Program.Logger.Debug("[Download Tags] Merging Page: " + i); lblStatus.Text = "Status: Merging Page: " + i + ", this might take some times."; lblStatus.Invalidate(); lblStatus.Update(); lblStatus.Refresh(); Application.DoEvents(); string tempName = TAGS_FILENAME + "." + i + ".!tmp"; DanbooruTagCollection tempTag; if (isSankaku) { SankakuComplexParser parser = new SankakuComplexParser(); tempTag = parser.parseTagsPage(File.ReadAllText(tempName), i); } else { tempTag = new DanbooruTagsDao(tempName).Tags; } if (tempTag != null && tempTag.Tag != null) { newTagList.AddRange(tempTag.Tag); } } for (int i = 1; i < lastPage; ++i) { string tempName = TAGS_FILENAME + "." + i + ".!tmp"; File.Delete(tempName); } return newTagList; }
private void HandleLoop(string tempName) { DanbooruTagCollection tempTags = null; if (isSankaku) { string data = File.ReadAllText(tempName); if (!string.IsNullOrWhiteSpace(data)) { tempTags = new SankakuComplexParser().parseTagsPage(data, Page); } else { string message = "Got empty response!"; Program.Logger.Error(message); ++retry; if (retry > Int32.Parse(DanbooruDownloader3.Properties.Settings.Default.retry)) { WaitForDelay(message); ProcessLoop(--Page); } else { MessageBox.Show(message); } } } else { tempTags = new DanbooruTagsDao(tempName).Tags; } if (tempTags == null || tempTags.Tag == null || tempTags.Tag.Length == 0 || (prevTag != null && prevTag.Tag.Last().Name == tempTags.Tag.Last().Name)) { // no more tags chkUseLoop.Enabled = true; var newTagList = CombineLoopTag(Page); if (SelectedProvider != null) { string targetXml = "tags-" + SelectedProvider.Name + ".xml"; if (File.Exists(targetXml)) { if (chkBackup.Checked) { string backupName = targetXml + ".bak"; if (File.Exists(backupName)) File.Delete(backupName); File.Move(targetXml, backupName); } else { File.Delete(targetXml); } } DanbooruTagsDao.Save(targetXml, newTagList); SelectedProvider.LoadProviderTagCollection(); Program.Logger.Info(String.Format("[Download Tags] Private Tags.xml saved to {0}.", targetXml)); } if (chkMerge.Checked) { Program.Logger.Debug("[Download Tags] Merging Old Tags."); lblStatus.Text = "Status: Merging Old Tags, this might take some times."; lblStatus.Invalidate(); lblStatus.Update(); lblStatus.Refresh(); Application.DoEvents(); DanbooruTagsDao.Save(TAGS_FILENAME + ".merge", newTagList); var message = DanbooruTagsDao.Merge(TAGS_FILENAME + ".merge", TAGS_FILENAME); Program.Logger.Info("[Download Tags] " + message); MessageBox.Show(message, "Tags.xml merged."); File.Delete(TAGS_FILENAME + ".merge"); } else { // write back to TAGS_FILENAME DanbooruTagsDao.Save(TAGS_FILENAME, newTagList); } DanbooruTagsDao.Instance = new DanbooruTagsDao(TAGS_FILENAME); Program.Logger.Info("[Download Tags] Complete."); lblStatus.Text = "Status: Download complete."; if (chkAutoClose.Checked) { this.Close(); } } else { // continue next page ProcessLoop(Page); prevTag = tempTags; } }