private void UpdateLookups()
{
_logger.Info("Updating lookup dictionary");
Lookups.Clear();
// Don't hammer the website:
SurgeProtection.CheckBeforeRequest();
Uri categoryApiUri = new Uri(HostUri, "CategoryAPI");
string categoryApiPageSource;
try
{
categoryApiPageSource = _webClient.DownloadString(categoryApiUri);
_logger.Debug($"Downloaded '{categoryApiUri}'");
}
catch (Exception ex)
{
_logger.Error(ex, $"The file '{categoryApiUri}' page failed to download because of an exception");
return;
}
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(categoryApiPageSource);
var nodes = doc.DocumentNode.SelectNodes(@"//*[@id='content']//*[contains(@class,'searchresults')]//a");
_logger.Debug($"Found {nodes.Count} categories for updating the lookup dictionary");
if (nodes.Any())
{
foreach (var node in nodes)
{
string name = node.InnerText;
string link = node.GetAttributeValue("href", string.Empty);
int linkQueryIndex = link.IndexOf('?');
// Skip anything that looks like a category but has an invalid URI
if (!Uri.TryCreate(HostUri, link.Substring(0, linkQueryIndex >= 0 ? linkQueryIndex : link.Length), out Uri uri))
{
_logger.Warn($"Skipping category named '{name}' for lookup dictionary because it has an invalid URI: {link}");
continue;
}
Lookups.Add(name, uri);
}
}
else
{
_logger.Warn("There weren't any items on the categories by name page to fill the lookup dictionary");
}
}
