public void BuildIndex() { int lastId = 0; int j = 1; string output; string tagName; XmlElement tagRoot; XmlNodeList tagNodes; Console.Clear(); Console.WriteLine("Building index..."); Stopwatch watch = new Stopwatch(); watch.Start(); // Fetch character tags do { tagRoot = client.GetPosts(urlToUse + $"tags.xml?search[category]=4&limit={TagsLimit}&page=a{lastId}&only=name,id").Result; tagNodes = tagRoot.SelectNodes("tag"); for (int i = 0; i < tagNodes.Count; i++) { tagName = tagNodes[i].SelectSingleNode("name").InnerText; if (!tagName.Contains("#")) { tagLinks.TryAdd(tagName, new List <string>()); output = $"Fetching character tags ({j++})."; PrintUtils.PrintRow(output, 0, 0); } if (i == 0) { lastId = int.Parse(tagNodes[i].SelectSingleNode("id").InnerText); } } }while (tagNodes.Count != 0); // Fetch series tags lastId = 0; j = 1; do { tagRoot = client.GetPosts(urlToUse + $"tags.xml?search[category]=3&limit={TagsLimit}&page=a{lastId}&only=name,id").Result; tagNodes = tagRoot.SelectNodes("tag"); for (int i = 0; i < tagNodes.Count; i++) { tagName = tagNodes[i].SelectSingleNode("name").InnerText; if (!tagName.Contains("#")) { seriesTags.TryAdd(tagName, new HashSet <string>()); output = $"Fetching series tags ({j++})."; PrintUtils.PrintRow(output, 0, 0); } if (i == 0) { lastId = int.Parse(tagNodes[i].SelectSingleNode("id").InnerText); } } }while (tagNodes.Count != 0); // Write empty tag and series names to backup _backup.BackupTagNames(tagLinks.Keys.ToList()); _backup.BackupSeriesNames(seriesTags.Keys.ToList()); // Fetch data from external source and write to backup FetchData(); // Persist to database Console.Clear(); Console.WriteLine("Writing to database..."); _persistence.CleanTagLinks(); _persistence.PersistTagLinks(tagLinks); _persistence.CleanSeries(); _persistence.PersistSeriesTags(seriesTags); _persistence.CountTagLinks(); _persistence.CountSeriesLinks(); SwitchDatabase(); watch.Stop(); TimeSpan timespan = TimeSpan.FromMilliseconds(watch.ElapsedMilliseconds); string timeString = string.Format("{0:D2}h:{1:D2}m:{2:D2}s:{3:D3}ms", timespan.Hours, timespan.Minutes, timespan.Seconds, timespan.Milliseconds); Resources.SystemLogger.Log($"Downloaded {tagLinks.Keys.Count} tags in {timeString} using {numThreads} thread(s)."); }
private List <FileData> ApiQuery(string tags) { // XML element results are fetched from the API XmlElement posts = client.GetPosts(BaseParams + $"&tags={tags}").Result; XmlNodeList postList; List <XmlElement> allElements = new List <XmlElement>(); int count = int.Parse(posts.GetAttribute("count")); int numPages = count / HardLimit; for (int pageNum = 0; pageNum <= numPages; pageNum++) { posts = client.GetPosts(BaseParams + $"&tags={tags}&pid={pageNum}").Result; postList = posts.SelectNodes("post"); foreach (XmlElement post in postList) { allElements.Add(post); } } // Images are selected, based on the number of desired images List <XmlElement> selected = new List <XmlElement>(); int random; List <int> chosenRands = new List <int>(); if (count < ImgsToFetch) { foreach (XmlElement element in allElements) { selected.Add(element); } } else { for (int i = 0; i < ImgsToFetch; i++) { random = RandomIndex(ref chosenRands, count); selected.Add(allElements[random]); } } // Images are downloaded. string fileUrl; string fileExt; List <FileData> fileData = new List <FileData>(); byte[] data; Stream stream; Bitmap image; bool tried = false; XmlElement tempElement; string id = " "; using (WebClient webClient = new WebClient()) { foreach (XmlElement element in selected) { tempElement = element; do { if (tried == true) { Resources.ImageLogger.Log($"Banned image encountered for tags '{tags}'. ID = {id}."); random = RandomIndex(ref chosenRands, count); tempElement = allElements[random]; } id = tempElement.GetAttribute("id"); tried = true; }while (banFilter.Contains(id)); fileUrl = tempElement.GetAttribute("file_url"); fileExt = "." + fileUrl.Split('.').Last(); data = webClient.DownloadData(fileUrl); stream = new MemoryStream(data); image = new Bitmap(stream); fileData.Add(new ImageData(fileExt, image, id)); tried = false; } } Resources.ImageLogger.Log($"Downloaded {selected.Count}/{ImgsToFetch} images for '{tags}'. Total number of images = {count}."); return(fileData); }