public async Task <IEnumerable <BlogRssItem> > GetUprofileBlogPosts() { var runtimeCache = ApplicationContext.Current.ApplicationCache.RuntimeCache; var cached = (IEnumerable <BlogRssItem>)runtimeCache.GetCacheItem("CommunityUProfileBlogPosts"); if (cached != null) { return(cached); } try { using (var client = new HttpClient()) { var result = await client.GetStringAsync("https://umbraco.com/blog/rss-uprofile-feed/"); if (result == null) { return(null); } var feed = XElement.Parse(result); var posts = new List <BlogRssItem>(); var channel = feed.Element("channel"); var items = channel.GetElements("item"); foreach (var post in items) { var title = post.GetElementValue <string>("title")?.Trim(); var link = post.GetElementValue <string>("link")?.Trim(); var description = post.GetElementValue <string>("description")?.Trim(); var image = post.Element("image"); var thumbnail = image?.GetElementValue <string>("url")?.Trim(); var item = new BlogRssItem() { Title = title, Link = link, Thumbnail = thumbnail, Description = description }; posts.Add(item); } runtimeCache.InsertCacheItem("CommunityUProfileBlogPosts", () => posts, TimeSpan.FromHours(2)); return(posts); } } catch (Exception ex) { LogHelper.Error <BlogPostsService>("Unable to fetch uprofile posts", ex); } return(null); }
public BlogRssItem[] GetBlogPosts(PerformContext context) { var posts = new List <BlogRssItem>(); var progressBar = context.WriteProgressBar(); var blogs = GetBlogs(); foreach (var blog in blogs.WithProgress(progressBar, blogs.Length)) { try { string raw; const string userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3393.4 Safari/537.36"; context.WriteLine($"Processing blog {blog.Title}"); // Initialize a new web client (with the encoding specified for the blog) using (var wc = new WebClient()) { wc.Headers.Add(HttpRequestHeader.UserAgent, userAgent); wc.Encoding = blog.Encoding; // Download the raw XML raw = wc.DownloadString(blog.RssUrl); raw = RemoveLeadingCharacters(raw).Replace("a10:updated", "pubDate"); } // Parse the XML into a new instance of XElement var feed = XElement.Parse(raw); var channel = feed.Element("channel"); var channelTitle = channel.GetElementValue("title"); var channelLink = channel.GetElementValue("link"); var channelDescription = channel.GetElementValue("description"); var channelLastBuildDate = channel.GetElementValue("lastBuildDate"); var channelLangauge = channel.GetElementValue("language"); var rssChannel = new BlogRssChannel { Id = blog.Id, Title = channelTitle, Link = channelLink }; var items = channel.GetElements("item"); foreach (var item in items) { var title = item.GetElementValue("title"); var link = (string.IsNullOrEmpty(item.GetElementValue("link")) ? item.GetElementValue("guid") : item.GetElementValue("link")) .Trim(); var pubDate = GetPublishDate(item); if (pubDate == default(DateTimeOffset)) { continue; } var approvedCategories = new List <string> { "umbraco", "codegarden", "articulate", "examine" }; var categories = item.GetElements("category"); if (categories.Any()) { var includeItem = title.ToLowerInvariant().ContainsAny(approvedCategories); foreach (var category in categories) { // no need to check more if the item is already approved if (includeItem) { continue; } foreach (var approvedCategory in approvedCategories) { if (category.Value.ToLowerInvariant().Contains(approvedCategory.ToLowerInvariant())) { includeItem = true; } } } if (includeItem == false) { var allCategories = string.Join(",", categories.Select(i => i.Value)); context.SetTextColor(ConsoleTextColor.DarkYellow); context.WriteLine($"Not including post titled {title} because it was not in an approved category. The categories it was found in: {allCategories}. [{link}]"); context.ResetTextColor(); continue; } } // Blog has no category info and posts things unrelated to Umbraco, check there's related keywords in the title if (blog.CheckTitles) { var includeItem = false; foreach (var approvedCategory in approvedCategories) { if (title.ToLowerInvariant().Contains(approvedCategory.ToLowerInvariant())) { includeItem = true; } } // Blog post seems unrelated to Umbraco, skip it if (includeItem == false) { continue; } } var blogPost = new BlogRssItem { Channel = rssChannel, Title = title, // some sites store the link in the <guid/> element Link = link, PublishedDate = pubDate }; posts.Add(blogPost); } // Get the avatar locally so that we can use ImageProcessor and serve it over https using (var wc = new WebClient()) { wc.Headers.Add(HttpRequestHeader.UserAgent, userAgent); var baseLogoPath = HostingEnvironment.MapPath("~/media/blogs/"); if (Directory.Exists(baseLogoPath) == false) { Directory.CreateDirectory(baseLogoPath); } var logoExtension = GetFileExtension(blog.LogoUrl); var logoPath = baseLogoPath + blog.Id + logoExtension; wc.DownloadFile(blog.LogoUrl, logoPath); } } catch (Exception ex) { context.SetTextColor(ConsoleTextColor.Red); context.WriteLine("Unable to get blog posts for: " + blog.RssUrl, ex); context.ResetTextColor(); } } return(posts.OrderByDescending(x => x.PublishedDate).ToArray()); }
public BlogCachedRssItem(BlogInfo blog, BlogRssItem item) { Blog = blog; Item = item; }
public BlogRssItem[] GetBlogPosts(PerformContext context) { var posts = new List <BlogRssItem>(); var progressBar = context.WriteProgressBar(); var blogs = GetBlogs(); foreach (var blog in blogs.WithProgress(progressBar, blogs.Length)) { try { string raw; context.WriteLine($"Processing blog {blog.Title}"); // Initialize a new web client (with the encoding specified for the blog) using (var wc = new WebClient()) { wc.Encoding = blog.Encoding; // Download the raw XML raw = wc.DownloadString(blog.RssUrl); raw = RemoveLeadingCharacters(raw).Replace("a10:updated", "pubDate"); } // Parse the XML into a new instance of XElement var feed = XElement.Parse(raw); var channel = feed.Element("channel"); var channelTitle = channel.GetElementValue("title"); var channelLink = channel.GetElementValue("link"); var channelDescription = channel.GetElementValue("description"); var channelLastBuildDate = channel.GetElementValue("lastBuildDate"); var channelLangauge = channel.GetElementValue("language"); var rssChannel = new BlogRssChannel { Id = blog.Id, Title = channelTitle, Link = channelLink }; var items = channel.GetElements("item"); foreach (var item in items) { var title = item.GetElementValue("title"); var link = (string.IsNullOrEmpty(item.GetElementValue("link")) ? item.GetElementValue("guid") : item.GetElementValue("link")) .Trim(); var pubDate = GetPublishDate(item); if (pubDate == default(DateTimeOffset)) { continue; } var approvedCategories = new List <string> { "umbraco", "codegarden", "articulate", "examine" }; var categories = item.GetElements("category"); if (categories.Any()) { var includeItem = title.ToLowerInvariant().ContainsAny(approvedCategories); foreach (var category in categories) { // no need to check more if the item is already approved if (includeItem) { continue; } foreach (var approvedCategory in approvedCategories) { if (category.Value.ToLowerInvariant().Contains(approvedCategory.ToLowerInvariant())) { includeItem = true; } } } if (includeItem == false) { var allCategories = string.Join(",", categories.Select(i => i.Value)); context.SetTextColor(ConsoleTextColor.Red); context.WriteLine($"Not including post titled {title} because it was not in an approved category. The categories it was found in: {allCategories}. [{link}]"); context.ResetTextColor(); continue; } } // Blog has no category info and posts things unrelated to Umbraco, check there's related keywords in the title if (blog.CheckTitles) { var includeItem = false; foreach (var approvedCategory in approvedCategories) { if (title.ToLowerInvariant().Contains(approvedCategory.ToLowerInvariant())) { includeItem = true; } } // Blog post seems unrelated to Umbraco, skip it if (includeItem == false) { continue; } } var blogPost = new BlogRssItem { Channel = rssChannel, Title = title, // some sites store the link in the <guid/> element Link = link, PublishedDate = pubDate }; posts.Add(blogPost); } } catch (Exception ex) { context.SetTextColor(ConsoleTextColor.Red); context.WriteLine("Unable to get blog posts for: " + blog.RssUrl, ex); context.ResetTextColor(); } } return(posts.OrderByDescending(x => x.PublishedDate).ToArray()); }
public BlogRssItem[] GetBlogPosts() { var posts = new List <BlogRssItem>(); foreach (var blog in GetBlogs()) { try { string raw; // Need to make sure we try TLS 1.2 first else the connection will just be closed in us // No other protocols allowed SSL * and TLS 1.0 are considered insecure ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12 | SecurityProtocolType.Tls11; // Initialize a new web client (with the encoding specified for the blog) using (var wc = new WebClient()) { wc.Encoding = blog.Encoding; // Download the raw XML raw = wc.DownloadString(blog.RssUrl); raw = RemoveLeadingCharacters(raw).Replace("a10:updated", "pubDate"); } // Parse the XML into a new instance of XElement var feed = XElement.Parse(raw); var channel = feed.Element("channel"); var channelTitle = channel.GetElementValue("title"); var channelLink = channel.GetElementValue("link"); var channelDescription = channel.GetElementValue("description"); var channelLastBuildDate = channel.GetElementValue("lastBuildDate"); var channelLangauge = channel.GetElementValue("language"); var rssChannel = new BlogRssChannel { Id = blog.Id, Title = channelTitle, Link = channelLink }; foreach (var item in channel.GetElements("item")) { var title = item.GetElementValue("title"); var link = (string.IsNullOrEmpty(item.GetElementValue("link")) ? item.GetElementValue("guid") : item.GetElementValue("link")) .Trim(); var pubDate = GetPublishDate(item); if (pubDate == default(DateTimeOffset)) { continue; } var approvedCategories = new List <string> { "umbraco", "codegarden", "articulate", "examine" }; var categories = item.GetElements("category"); if (categories.Any()) { var includeItem = title.ToLowerInvariant().ContainsAny(approvedCategories); foreach (var category in categories) { // no need to check more if the item is already approved if (includeItem) { continue; } foreach (var approvedCategory in approvedCategories) { if (category.Value.ToLowerInvariant().Contains(approvedCategory.ToLowerInvariant())) { includeItem = true; } } } if (includeItem == false) { var allCategories = string.Join(",", categories.Select(i => i.Value)); LogHelper.Info <BlogPostsService>(string.Format("Not including post titled {0} because it was not in an approved category. The categories it was found in: {1}. [{2}]", title, allCategories, link)); continue; } } var blogPost = new BlogRssItem { Channel = rssChannel, Title = title, // some sites store the link in the <guid/> element Link = link, PublishedDate = pubDate }; posts.Add(blogPost); } } catch (Exception ex) { LogHelper.Error <BlogPostsService>("Unable to get blog posts for: " + blog.RssUrl, ex); } } return(posts.OrderByDescending(x => x.PublishedDate).ToArray()); }