public void BuildIndex()
{
int lastId = 0;
int j = 1;
string output;
string tagName;
XmlElement tagRoot;
XmlNodeList tagNodes;
Console.Clear();
Console.WriteLine("Building index...");
Stopwatch watch = new Stopwatch();
watch.Start();
// Fetch character tags
do
{
tagRoot = client.GetPosts(urlToUse + $"tags.xml?search[category]=4&limit={TagsLimit}&page=a{lastId}&only=name,id").Result;
tagNodes = tagRoot.SelectNodes("tag");
for (int i = 0; i < tagNodes.Count; i++)
{
tagName = tagNodes[i].SelectSingleNode("name").InnerText;
if (!tagName.Contains("#"))
{
tagLinks.TryAdd(tagName, new List <string>());
output = $"Fetching character tags ({j++}).";
PrintUtils.PrintRow(output, 0, 0);
}
if (i == 0)
{
lastId = int.Parse(tagNodes[i].SelectSingleNode("id").InnerText);
}
}
}while (tagNodes.Count != 0);
// Fetch series tags
lastId = 0;
j = 1;
do
{
tagRoot = client.GetPosts(urlToUse + $"tags.xml?search[category]=3&limit={TagsLimit}&page=a{lastId}&only=name,id").Result;
tagNodes = tagRoot.SelectNodes("tag");
for (int i = 0; i < tagNodes.Count; i++)
{
tagName = tagNodes[i].SelectSingleNode("name").InnerText;
if (!tagName.Contains("#"))
{
seriesTags.TryAdd(tagName, new HashSet <string>());
output = $"Fetching series tags ({j++}).";
PrintUtils.PrintRow(output, 0, 0);
}
if (i == 0)
{
lastId = int.Parse(tagNodes[i].SelectSingleNode("id").InnerText);
}
}
}while (tagNodes.Count != 0);
// Write empty tag and series names to backup
_backup.BackupTagNames(tagLinks.Keys.ToList());
_backup.BackupSeriesNames(seriesTags.Keys.ToList());
// Fetch data from external source and write to backup
FetchData();
// Persist to database
Console.Clear();
Console.WriteLine("Writing to database...");
_persistence.CleanTagLinks();
_persistence.PersistTagLinks(tagLinks);
_persistence.CleanSeries();
_persistence.PersistSeriesTags(seriesTags);
_persistence.CountTagLinks();
_persistence.CountSeriesLinks();
SwitchDatabase();
watch.Stop();
TimeSpan timespan = TimeSpan.FromMilliseconds(watch.ElapsedMilliseconds);
string timeString = string.Format("{0:D2}h:{1:D2}m:{2:D2}s:{3:D3}ms", timespan.Hours, timespan.Minutes, timespan.Seconds, timespan.Milliseconds);
Resources.SystemLogger.Log($"Downloaded {tagLinks.Keys.Count} tags in {timeString} using {numThreads} thread(s).");
}
private List <FileData> ApiQuery(string tags)
{
// XML element results are fetched from the API
XmlElement posts = client.GetPosts(BaseParams + $"&tags={tags}").Result;
XmlNodeList postList;
List <XmlElement> allElements = new List <XmlElement>();
int count = int.Parse(posts.GetAttribute("count"));
int numPages = count / HardLimit;
for (int pageNum = 0; pageNum <= numPages; pageNum++)
{
posts = client.GetPosts(BaseParams + $"&tags={tags}&pid={pageNum}").Result;
postList = posts.SelectNodes("post");
foreach (XmlElement post in postList)
{
allElements.Add(post);
}
}
// Images are selected, based on the number of desired images
List <XmlElement> selected = new List <XmlElement>();
int random;
List <int> chosenRands = new List <int>();
if (count < ImgsToFetch)
{
foreach (XmlElement element in allElements)
{
selected.Add(element);
}
}
else
{
for (int i = 0; i < ImgsToFetch; i++)
{
random = RandomIndex(ref chosenRands, count);
selected.Add(allElements[random]);
}
}
// Images are downloaded.
string fileUrl;
string fileExt;
List <FileData> fileData = new List <FileData>();
byte[] data;
Stream stream;
Bitmap image;
bool tried = false;
XmlElement tempElement;
string id = " ";
using (WebClient webClient = new WebClient())
{
foreach (XmlElement element in selected)
{
tempElement = element;
do
{
if (tried == true)
{
Resources.ImageLogger.Log($"Banned image encountered for tags '{tags}'. ID = {id}.");
random = RandomIndex(ref chosenRands, count);
tempElement = allElements[random];
}
id = tempElement.GetAttribute("id");
tried = true;
}while (banFilter.Contains(id));
fileUrl = tempElement.GetAttribute("file_url");
fileExt = "." + fileUrl.Split('.').Last();
data = webClient.DownloadData(fileUrl);
stream = new MemoryStream(data);
image = new Bitmap(stream);
fileData.Add(new ImageData(fileExt, image, id));
tried = false;
}
}
Resources.ImageLogger.Log($"Downloaded {selected.Count}/{ImgsToFetch} images for '{tags}'. Total number of images = {count}.");
return(fileData);
}
