private void UpdateLookups() { _logger.Info("Updating lookup dictionary"); Lookups.Clear(); // Don't hammer the website: SurgeProtection.CheckBeforeRequest(); Uri categoryApiUri = new Uri(HostUri, "CategoryAPI"); string categoryApiPageSource; try { categoryApiPageSource = _webClient.DownloadString(categoryApiUri); _logger.Debug($"Downloaded '{categoryApiUri}'"); } catch (Exception ex) { _logger.Error(ex, $"The file '{categoryApiUri}' page failed to download because of an exception"); return; } HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(categoryApiPageSource); var nodes = doc.DocumentNode.SelectNodes(@"//*[@id='content']//*[contains(@class,'searchresults')]//a"); _logger.Debug($"Found {nodes.Count} categories for updating the lookup dictionary"); if (nodes.Any()) { foreach (var node in nodes) { string name = node.InnerText; string link = node.GetAttributeValue("href", string.Empty); int linkQueryIndex = link.IndexOf('?'); // Skip anything that looks like a category but has an invalid URI if (!Uri.TryCreate(HostUri, link.Substring(0, linkQueryIndex >= 0 ? linkQueryIndex : link.Length), out Uri uri)) { _logger.Warn($"Skipping category named '{name}' for lookup dictionary because it has an invalid URI: {link}"); continue; } Lookups.Add(name, uri); } } else { _logger.Warn("There weren't any items on the categories by name page to fill the lookup dictionary"); } }