private void UpdateLookups()
{
_logger.Info("Updating lookup dictionary");
Lookups.Clear();
// Don't hammer the website:
SurgeProtection.CheckBeforeRequest();
Uri categoryApiUri = new Uri(HostUri, "CategoryAPI");
string categoryApiPageSource;
try
{
categoryApiPageSource = _webClient.DownloadString(categoryApiUri);
_logger.Debug($"Downloaded '{categoryApiUri}'");
}
catch (Exception ex)
{
_logger.Error(ex, $"The file '{categoryApiUri}' page failed to download because of an exception");
return;
}
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(categoryApiPageSource);
var nodes = doc.DocumentNode.SelectNodes(@"//*[@id='content']//*[contains(@class,'searchresults')]//a");
_logger.Debug($"Found {nodes.Count} categories for updating the lookup dictionary");
if (nodes.Any())
{
foreach (var node in nodes)
{
string name = node.InnerText;
string link = node.GetAttributeValue("href", string.Empty);
int linkQueryIndex = link.IndexOf('?');
// Skip anything that looks like a category but has an invalid URI
if (!Uri.TryCreate(HostUri, link.Substring(0, linkQueryIndex >= 0 ? linkQueryIndex : link.Length), out Uri uri))
{
_logger.Warn($"Skipping category named '{name}' for lookup dictionary because it has an invalid URI: {link}");
continue;
}
Lookups.Add(name, uri);
}
}
else
{
_logger.Warn("There weren't any items on the categories by name page to fill the lookup dictionary");
}
}
public SDLWikiApiItem GetWikiItem(string name, out string errorMessage)
{
if (!Cache.TryGetItem(name, out SDLWikiApiItem item) || DateTime.Now - item.LastUpdate >= SDLWikiApiItem.Expiration)
{
// Maybe the casing is wrong, or maybe there's a partial match:
if (Lookups.Any())
{
var match = Search(name).FirstOrDefault();
if (match != default && match.Name != name)
{
return(GetWikiItem(match.Name, out errorMessage));
}
}
// Don't hammer the website:
SurgeProtection.CheckBeforeRequest();
// Download the document:
string documentRawText;
Uri documentUri = new Uri(HostUri, $"{name.Replace("sdl_", "SDL_")}?action=raw");
try
{
documentRawText = _webClient.DownloadString(documentUri);
}
catch (WebException ex)
{
errorMessage = ex.Message;
return(null);
}
// The site responds with content indicating the Wiki document doesn't exist:
string notFoundText = $"Page {name} not found";
if (string.Concat(documentRawText.Take(notFoundText.Length + 1)).ToLowerInvariant().Contains(notFoundText))
{
errorMessage = documentRawText;
return(null);
}
// The page should not be valid HTML since ?action=raw should return MoinMoin markup:
var htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(documentRawText);
if (!htmlDoc.ParseErrors.Any())
{
errorMessage = $"Couldn't receive the correct document because the content was in HTML rather than Markup";
return(null);
}
item = new SDLWikiApiItem(name, new Uri(HostUri, name));
item.Update(documentRawText);
item.Live = true;
Cache.AddOrUpdate(item);
Export(CacheFile);
}
else
{
// Pulled directly from cache:
if (item != null)
{
item.Live = false;
}
}
errorMessage = string.Empty;
return(item);
}
