private string GetRouteOnly(RssFeedItem item)
{
string url = item.OpenGraphAttributes.GetValueOrDefault("og:url") ?? "";
// Make sure the Url is complete
if (!url.StartsWith("http"))
{
url = item.HtmlAttributes.GetValueOrDefault("Url") ?? item.FeedAttributes.Url;
}
if (!url.StartsWith("http"))
{
url = _webUtils.RepairUrl(url, item.FeedAttributes.Url);
}
Uri uri = new Uri(url);
return(uri.GetComponents(UriComponents.Path, UriFormat.Unescaped).ToLower());
}
private RssFeedItem CreateNodeLinks(List <string> filters, string location, int count, string title, string linkUrl, string feedUrl, bool isHeadline)
{
// Sometimes Drudge has completely empty links, ignore them
if (string.IsNullOrEmpty(linkUrl))
{
return(null);
}
// Repair any protocol typos if possible
if (!linkUrl.ToLower().StartsWith("http"))
{
linkUrl = webUtils.RepairUrl(linkUrl, feedUrl);
}
if (!Uri.TryCreate(linkUrl, UriKind.Absolute, out Uri uri))
{
log.Warning("Unable to parse Uri {linkUrl}", linkUrl);
return(null);
}
// Calculate the MD5 hash for the link so we can be sure of uniqueness
string hash = utils.CreateMD5Hash(uri.AbsoluteUri.ToLower());
if (filters.Contains(hash))
{
log.Information("Hash '{hash}':'{uri}' found in filter list", hash, uri);
return(null);
}
if (linkUrl.Length > 0 && title.Length > 0)
{
return(new RssFeedItem()
{
FeedAttributes = new FeedAttributes()
{
Title = WebUtility.HtmlDecode(title),
Url = uri.AbsoluteUri,
UrlHash = hash,
DateAdded = DateTime.Now.ToUniversalTime(),
LinkLocation = $"{location}, article {count}",
IsUrlShortened = uri.Host.ToLower() == "t.co",
IsHeadline = isHeadline
}
});
}
return(null);
}