public void TestDanbooruTags()
{
{
DanbooruTagCollection tags = (DanbooruTagCollection)ser.Deserialize(File.OpenText(sourceDanbooruTagsXml));
Assert.IsTrue(tags.Tag.Length == 151190);
Assert.IsTrue(tags.Tag[0].Name == "geordi_la_forge");
Assert.IsTrue(tags.Tag[0].Ambiguous == false);
Assert.IsTrue(tags.Tag[0].Type == DanbooruTagType.Character);
Assert.IsTrue(tags.Tag[0].Count == 1);
Assert.IsTrue(tags.Tag[0].Id == "525178");
var AmbiguousTags = tags.Tag.First <DanbooruTag>(x => x.Id == "1723");
Assert.IsTrue(AmbiguousTags.Name == "parody");
Assert.IsTrue(AmbiguousTags.Type == DanbooruTagType.General);
Assert.IsTrue(AmbiguousTags.Count == 16546);
Assert.IsTrue(AmbiguousTags.Ambiguous == true);
Assert.IsTrue(tags.GeneralTag.Length == 23011);
Assert.IsTrue(tags.ArtistTag.Length == 67705);
Assert.IsTrue(tags.CopyrightTag.Length == 12209);
Assert.IsTrue(tags.CharacterTag.Length == 48265);
Assert.IsTrue(tags.CircleTag.Length == 0);
Assert.IsTrue(tags.FaultsTag.Length == 0);
Assert.IsTrue(tags.GeneralTag.Length + tags.ArtistTag.Length + tags.CopyrightTag.Length + tags.CharacterTag.Length + tags.CircleTag.Length + tags.FaultsTag.Length == tags.Tag.Length);
}
}
public static void Save(string target, List <DanbooruTag> sourceInstance)
{
using (StreamWriter s = File.CreateText(target))
{
DanbooruTagCollection col = new DanbooruTagCollection();
col.Tag = sourceInstance.ToArray();
_ser.Serialize(s, col);
}
}
public DanbooruTagsDao(string xmlTagFile)
{
XmlSerializer ser = new XmlSerializer(typeof(DanbooruTagCollection));
if (!File.Exists(xmlTagFile)) throw new FileNotFoundException("Cannot load tags.xml", xmlTagFile);
using (StreamReader s = File.OpenText(xmlTagFile))
{
this.Tags = (DanbooruTagCollection)ser.Deserialize(s);
}
}
public List <DanbooruTag> ParseTagsString(string tagsStr, DanbooruTagCollection tagCollection)
{
List <DanbooruTag> tags = new List <DanbooruTag>();
var tokens = tagsStr.Split(' ');
foreach (var item in tokens)
{
tags.Add(GetTag(item.Trim(), tagCollection));
}
return(tags);
}
public DanbooruTagsDao(string xmlTagFile)
{
XmlSerializer ser = new XmlSerializer(typeof(DanbooruTagCollection));
if (!File.Exists(xmlTagFile))
{
throw new FileNotFoundException("Cannot load tags.xml", xmlTagFile);
}
using (StreamReader s = File.OpenText(xmlTagFile))
{
this.Tags = (DanbooruTagCollection)ser.Deserialize(s);
}
}
public DanbooruTagsDao(string xmlTagFile)
{
XmlSerializer ser = new XmlSerializer(typeof(DanbooruTagCollection));
if (!File.Exists(xmlTagFile)) throw new FileNotFoundException("Cannot load tags.xml", xmlTagFile);
using (StreamReader s = File.OpenText(xmlTagFile))
{
try
{
this.Tags = (DanbooruTagCollection)ser.Deserialize(s);
}
catch (Exception ex)
{
Program.Logger.Error("Failed to parse: " + xmlTagFile, ex);
}
}
}
public DanbooruTagsDao(string xmlTagFile)
{
if (!File.Exists(xmlTagFile))
{
throw new FileNotFoundException("Cannot load tags.xml", xmlTagFile);
}
using (StreamReader s = File.OpenText(xmlTagFile))
{
try
{
this.Tags = (DanbooruTagCollection)_ser.Deserialize(s);
}
catch (Exception ex)
{
Program.Logger.Error("Failed to parse: " + xmlTagFile, ex);
this.Tags = new DanbooruTagCollection();
}
}
}
public DanbooruTag GetTag(string tag, DanbooruTagCollection tagCollection)
{
// TODO: Hot spot for perfomance
var result = tagCollection.Tag.FirstOrDefault <DanbooruTag>(x => x.Name == tag);
if (result != null)
{
return(result);
}
else
{
var unknownTag = new DanbooruTag()
{
Name = tag,
Type = DanbooruTagType.Unknown,
Count = -1,
Id = "-1"
};
return(unknownTag);
}
}
public List <DanbooruTag> ParseTagsString(string tagsStr, DanbooruTagCollection tagCollection)
{
List <DanbooruTag> tags = new List <DanbooruTag>();
if (!String.IsNullOrWhiteSpace(tagsStr))
{
var tokens = tagsStr.Split(' ');
if (tagCollection == null)
{
foreach (var item in tokens)
{
tags.Add(GetTag(item.Trim()));
}
}
else
{
foreach (var item in tokens)
{
tags.Add(GetTag(item.Trim(), tagCollection));
}
}
}
return(tags);
}
public DanbooruTagCollection parseTagsPage(string data, int page)
{
DanbooruTagCollection tagCol = new DanbooruTagCollection();
List<DanbooruTag> tags = new List<DanbooruTag>();
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(data);
int index = 1 + ((page - 1) * 50);
// select all tags
var tables = doc.DocumentNode.SelectNodes("//table[contains(@class,'highlightable')]");
foreach (var table in tables)
{
if (!(table.Attributes["class"].Value == "highlightable"))
{
table.Remove();
continue;
}
var rows = table.SelectNodes("//table[contains(@class,'highlightable')]//tr");
int countIndex = 1, nameIndex = 3, typeIndex = 9;
foreach (var row in rows)
{
//if (row.ChildNodes.Count != 11 && row.ChildNodes.Count != 7) continue;
var cols = row.ChildNodes;
if (cols[1].Name == "th")
{
for (int i = 0; i < cols.Count; ++i)
{
if (cols[i].Name == "th")
{
if (cols[i].InnerText.Replace("\n", "") == "Posts")
{
countIndex = i;
continue;
}
if (cols[i].InnerText.Replace("\n", "") == "Name")
{
nameIndex = i;
continue;
}
if (cols[i].InnerText.Replace("\n", "") == "Type")
{
typeIndex = i;
continue;
}
}
}
continue;
}
if (cols[1].Name != "td") continue;
DanbooruTag tag = new DanbooruTag();
tag.Id = index.ToString();
tag.Count = Int32.Parse(cols[countIndex].InnerText);
tag.Name = Helper.RemoveControlCharacters(System.Net.WebUtility.HtmlDecode(cols[nameIndex].ChildNodes[3].InnerText.Replace("\n", "")));
string tagType = cols[typeIndex].InnerText.Replace("\n", "");
if (tagType.EndsWith("(edit)")) tagType = tagType.Substring(0, tagType.Length - 6);
tagType = tagType.ToLowerInvariant();
if (tagType == "general")
tag.Type = DanbooruTagType.General;
else if (tagType == "character")
tag.Type = DanbooruTagType.Character;
else if (tagType == "artist")
tag.Type = DanbooruTagType.Artist;
else if (tagType == "copyright")
tag.Type = DanbooruTagType.Copyright;
else if (tagType == "idol")
tag.Type = DanbooruTagType.Artist;
else if (tagType == "photo_set")
tag.Type = DanbooruTagType.Circle;
else
tag.Type = DanbooruTagType.Faults;
tags.Add(tag);
++index;
}
}
tagCol.Tag = tags.ToArray();
return tagCol;
}
public DanbooruTagCollection parseTagsPage(string data, int page)
{
DanbooruTagCollection tagCol = new DanbooruTagCollection();
List <DanbooruTag> tags = new List <DanbooruTag>();
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(data);
int index = 1 + ((page - 1) * 50);
// select all tags
var tables = doc.DocumentNode.SelectNodes("//table[contains(@class,'highlightable')]");
foreach (var table in tables)
{
if (!(table.Attributes["class"].Value == "highlightable"))
{
table.Remove();
continue;
}
var rows = table.SelectNodes("//table[contains(@class,'highlightable')]//tr");
int countIndex = 1, nameIndex = 3, typeIndex = 9;
foreach (var row in rows)
{
//if (row.ChildNodes.Count != 11 && row.ChildNodes.Count != 7) continue;
var cols = row.ChildNodes;
if (cols[1].Name == "th")
{
for (int i = 0; i < cols.Count; ++i)
{
if (cols[i].Name == "th")
{
if (cols[i].InnerText.Replace("\n", "") == "Posts")
{
countIndex = i;
continue;
}
if (cols[i].InnerText.Replace("\n", "") == "Name")
{
nameIndex = i;
continue;
}
if (cols[i].InnerText.Replace("\n", "") == "Type")
{
typeIndex = i;
continue;
}
}
}
continue;
}
if (cols[1].Name != "td")
{
continue;
}
DanbooruTag tag = new DanbooruTag();
tag.Id = index.ToString();
tag.Count = Int32.Parse(cols[countIndex].InnerText);
tag.Name = Helper.RemoveControlCharacters(System.Net.WebUtility.HtmlDecode(cols[nameIndex].ChildNodes[3].InnerText.Replace("\n", "")));
string tagType = cols[typeIndex].InnerText.Replace("\n", "");
if (tagType.EndsWith("(edit)"))
{
tagType = tagType.Substring(0, tagType.Length - 6);
}
tagType = tagType.ToLowerInvariant();
if (tagType == "general")
{
tag.Type = DanbooruTagType.General;
}
else if (tagType == "character")
{
tag.Type = DanbooruTagType.Character;
}
else if (tagType == "artist")
{
tag.Type = DanbooruTagType.Artist;
}
else if (tagType == "copyright")
{
tag.Type = DanbooruTagType.Copyright;
}
else if (tagType == "idol")
{
tag.Type = DanbooruTagType.Artist;
}
else if (tagType == "photo_set")
{
tag.Type = DanbooruTagType.Circle;
}
else
{
tag.Type = DanbooruTagType.Faults;
}
tags.Add(tag);
++index;
}
}
tagCol.Tag = tags.ToArray();
return(tagCol);
}
private void HandleLoop(string tempName)
{
DanbooruTagCollection tempTags = null;
if (isSankaku)
{
string data = File.ReadAllText(tempName);
if (!string.IsNullOrWhiteSpace(data))
{
tempTags = new SankakuComplexParser().parseTagsPage(data, Page);
}
else
{
string message = "Got empty response!";
Program.Logger.Error(message);
++retry;
if (retry > Int32.Parse(DanbooruDownloader3.Properties.Settings.Default.retry))
{
WaitForDelay(message);
ProcessLoop(--Page);
}
else
{
MessageBox.Show(message);
}
}
}
else
{
tempTags = new DanbooruTagsDao(tempName).Tags;
}
if (tempTags == null || tempTags.Tag == null || tempTags.Tag.Length == 0 ||
(prevTag != null && prevTag.Tag.Last().Name == tempTags.Tag.Last().Name))
{
// no more tags
chkUseLoop.Enabled = true;
var newTagList = CombineLoopTag(Page);
if (SelectedProvider != null)
{
string targetXml = "tags-" + SelectedProvider.Name + ".xml";
if (File.Exists(targetXml))
{
if (chkBackup.Checked)
{
string backupName = targetXml + ".bak";
if (File.Exists(backupName))
{
File.Delete(backupName);
}
File.Move(targetXml, backupName);
}
else
{
File.Delete(targetXml);
}
}
DanbooruTagsDao.Save(targetXml, newTagList);
SelectedProvider.LoadProviderTagCollection();
Program.Logger.Info(String.Format("[Download Tags] Private Tags.xml saved to {0}.", targetXml));
}
if (chkMerge.Checked)
{
Program.Logger.Debug("[Download Tags] Merging Old Tags.");
lblStatus.Text = "Status: Merging Old Tags, this might take some times.";
lblStatus.Invalidate();
lblStatus.Update();
lblStatus.Refresh();
Application.DoEvents();
DanbooruTagsDao.Save(TAGS_FILENAME + ".merge", newTagList);
var message = DanbooruTagsDao.Merge(TAGS_FILENAME + ".merge", TAGS_FILENAME);
Program.Logger.Info("[Download Tags] " + message);
MessageBox.Show(message, "Tags.xml merged.");
File.Delete(TAGS_FILENAME + ".merge");
}
else
{
// write back to TAGS_FILENAME
DanbooruTagsDao.Save(TAGS_FILENAME, newTagList);
}
DanbooruTagsDao.Instance = new DanbooruTagsDao(TAGS_FILENAME);
Program.Logger.Info("[Download Tags] Complete.");
lblStatus.Text = "Status: Download complete.";
if (chkAutoClose.Checked)
{
this.Close();
}
}
else
{
// continue next page
ProcessLoop(Page);
prevTag = tempTags;
}
}
public DanbooruTag GetTag(string tag, DanbooruTagCollection tagCollection)
{
// TODO: Hot spot for perfomance
var result = tagCollection.Tag.FirstOrDefault<DanbooruTag>(x => x.Name == tag);
if (result != null) return result;
else
{
var unknownTag = new DanbooruTag()
{
Name = tag,
Type = DanbooruTagType.Unknown,
Count = -1,
Id = "-1"
};
return unknownTag;
}
}
public static void Save(string target, List<DanbooruTag> sourceInstance)
{
XmlSerializer ser = new XmlSerializer(typeof(DanbooruTagCollection));
using (StreamWriter s = File.CreateText(target))
{
DanbooruTagCollection col = new DanbooruTagCollection();
col.Tag = sourceInstance.ToArray();
ser.Serialize(s, col);
}
}
public List<DanbooruTag> ParseTagsString(string tagsStr, DanbooruTagCollection tagCollection)
{
List<DanbooruTag> tags = new List<DanbooruTag>();
var tokens = tagsStr.Split(' ');
foreach (var item in tokens)
{
tags.Add(GetTag(item.Trim(), tagCollection));
}
return tags;
}
private void HandleLoop(string tempName)
{
DanbooruTagCollection tempTags = null;
if (isSankaku)
{
string data = File.ReadAllText(tempName);
if (!string.IsNullOrWhiteSpace(data))
{
tempTags = new SankakuComplexParser().parseTagsPage(data, Page);
}
else
{
string message = "Got empty response!";
Program.Logger.Error(message);
++retry;
if (retry > Int32.Parse(DanbooruDownloader3.Properties.Settings.Default.retry))
{
WaitForDelay(message);
ProcessLoop(--Page);
}
else
{
MessageBox.Show(message);
}
}
}
else
{
tempTags = new DanbooruTagsDao(tempName).Tags;
}
if (tempTags == null || tempTags.Tag == null || tempTags.Tag.Length == 0 ||
(prevTag != null && prevTag.Tag.Last().Name == tempTags.Tag.Last().Name))
{
// no more tags
chkUseLoop.Enabled = true;
var newTagList = CombineLoopTag(Page);
if (SelectedProvider != null)
{
string targetXml = "tags-" + SelectedProvider.Name + ".xml";
if (File.Exists(targetXml))
{
if (chkBackup.Checked)
{
string backupName = targetXml + ".bak";
if (File.Exists(backupName))
File.Delete(backupName);
File.Move(targetXml, backupName);
}
else
{
File.Delete(targetXml);
}
}
DanbooruTagsDao.Save(targetXml, newTagList);
SelectedProvider.LoadProviderTagCollection();
Program.Logger.Info(String.Format("[Download Tags] Private Tags.xml saved to {0}.", targetXml));
}
if (chkMerge.Checked)
{
Program.Logger.Debug("[Download Tags] Merging Old Tags.");
lblStatus.Text = "Status: Merging Old Tags, this might take some times.";
lblStatus.Invalidate();
lblStatus.Update();
lblStatus.Refresh();
Application.DoEvents();
DanbooruTagsDao.Save(TAGS_FILENAME + ".merge", newTagList);
var message = DanbooruTagsDao.Merge(TAGS_FILENAME + ".merge", TAGS_FILENAME);
Program.Logger.Info("[Download Tags] " + message);
MessageBox.Show(message, "Tags.xml merged.");
File.Delete(TAGS_FILENAME + ".merge");
}
else
{
// write back to TAGS_FILENAME
DanbooruTagsDao.Save(TAGS_FILENAME, newTagList);
}
DanbooruTagsDao.Instance = new DanbooruTagsDao(TAGS_FILENAME);
Program.Logger.Info("[Download Tags] Complete.");
lblStatus.Text = "Status: Download complete.";
if (chkAutoClose.Checked)
{
this.Close();
}
}
else
{
// continue next page
ProcessLoop(Page);
prevTag = tempTags;
}
}
