public FeedItem(ISyndicationItem item, DateTimeOffset recent) { Title = item.Title; Link = item.Links.FirstOrDefault(x => x.RelationshipType == RssLinkTypes.Alternate)?.Uri.ToString() ?? item.Id; Published = item.Published != default ? item.Published : item.LastUpdated; Recent = Published > recent; Description = item.Description; Links = item.Links .Where(x => !string.IsNullOrEmpty(x.MediaType)) .GroupBy(x => x.MediaType) .Select(x => x.First()) .ToDictionary(x => x.MediaType, x => x.Uri.ToString()); ISyndicationPerson person = item.Contributors.FirstOrDefault(x => x.RelationshipType == "author"); if (person != null) { Author = person.Name ?? person.Email; } AtomEntry atom = item as AtomEntry; if (atom != null && !string.IsNullOrEmpty(atom.Summary)) { Description = atom.Summary; } }
public async Task ReadPerson() { using (XmlReader xmlReader = XmlReader.Create(@"..\..\..\TestFeeds\simpleAtomFeed.xml", new XmlReaderSettings { Async = true })) { var persons = new List <ISyndicationPerson>(); var reader = new AtomFeedReader(xmlReader); while (await reader.Read()) { if (reader.ElementType == SyndicationElementType.Person) { ISyndicationPerson person = await reader.ReadPerson(); persons.Add(person); } } Assert.True(persons.Count() == 2); Assert.True(persons[0].Name == "Mark Pilgrim"); Assert.True(persons[0].Email == "*****@*****.**"); Assert.True(persons[0].Uri == "http://example.org/"); Assert.True(persons[1].Name == "Sam Ruby"); } }
private async Task <ISyndicationPerson> ReadPerson(Rss20FeedReader reader) { ISyndicationPerson person = null; person = await reader.ReadPerson(); return(person); }
// Read an RssFeed public static async Task CreateRssFeedReaderExample(string filePath) { // Create an XmlReader // Example: ..\tests\TestFeeds\rss20-2items.xml using (var xmlReader = XmlReader.Create(filePath, new XmlReaderSettings() { Async = true })) { // Instantiate an Rss20FeedReader using the XmlReader. // This will assign as default an Rss20FeedParser as the parser. var feedReader = new RssFeedReader(xmlReader); // // Read the feed while (await feedReader.Read()) { switch (feedReader.ElementType) { // Read category case SyndicationElementType.Category: ISyndicationCategory category = await feedReader.ReadCategory(); break; // Read Image case SyndicationElementType.Image: ISyndicationImage image = await feedReader.ReadImage(); break; // Read Item case SyndicationElementType.Item: ISyndicationItem item = await feedReader.ReadItem(); break; // Read link case SyndicationElementType.Link: ISyndicationLink link = await feedReader.ReadLink(); break; // Read Person case SyndicationElementType.Person: ISyndicationPerson person = await feedReader.ReadPerson(); break; // Read content default: ISyndicationContent content = await feedReader.ReadContent(); break; } } } }
public static async Task TestReadFeedElements(XmlReader outerXmlReader) { using (var xmlReader = outerXmlReader) { var reader = new RssFeedReader(xmlReader); int items = 0; while (await reader.Read()) { switch (reader.ElementType) { case SyndicationElementType.Person: ISyndicationPerson person = await reader.ReadPerson(); Assert.True(person.Email == "John Smith"); break; case SyndicationElementType.Link: ISyndicationLink link = await reader.ReadLink(); Assert.True(link.Length == 123); Assert.True(link.MediaType == "testType"); Assert.True(link.Uri.OriginalString == "http://example.com/"); break; case SyndicationElementType.Image: ISyndicationImage image = await reader.ReadImage(); Assert.True(image.Title == "Microsoft News"); Assert.True(image.Description == "Test description"); Assert.True(image.Url.OriginalString == "http://2.bp.blogspot.com/-NA5Jb-64eUg/URx8CSdcj_I/AAAAAAAAAUo/eCx0irI0rq0/s1600/bg_Microsoft_logo3-20120824073001907469-620x349.jpg"); break; case SyndicationElementType.Item: items++; ISyndicationItem item = await reader.ReadItem(); if (items == 1) { Assert.True(item.Title == "Lorem ipsum 2017-07-06T20:25:00+00:00"); Assert.True(item.Description == "Exercitation sit dolore mollit et est eiusmod veniam aute officia veniam ipsum."); Assert.True(item.Links.Count() == 3); } else if (items == 2) { Assert.True(item.Title == "Lorem ipsum 2017-07-06T20:24:00+00:00"); Assert.True(item.Description == "Do ipsum dolore veniam minim est cillum aliqua ea."); Assert.True(item.Links.Count() == 3); } break; default: break; } } } }
private async Task <bool> retrievePosts(Feed feed) { using (var xmlReader = XmlReader.Create("https://azurecomcdn.azureedge.net/en-us/blog/feed/", new XmlReaderSettings() { Async = true })) { // Instantiate an Rss20FeedReader using the XmlReader. // This will assign as default an Rss20FeedParser as the parser. var feedReader = new RssFeedReader(xmlReader); // // Read the feed while (await feedReader.Read()) { switch (feedReader.ElementType) { // Read category case SyndicationElementType.Category: ISyndicationCategory category = await feedReader.ReadCategory(); break; // Read Image case SyndicationElementType.Image: ISyndicationImage image = await feedReader.ReadImage(); break; // Read Item case SyndicationElementType.Item: ISyndicationItem item = await feedReader.ReadItem(); break; // Read link case SyndicationElementType.Link: ISyndicationLink link = await feedReader.ReadLink(); break; // Read Person case SyndicationElementType.Person: ISyndicationPerson person = await feedReader.ReadPerson(); break; // Read content default: ISyndicationContent content = await feedReader.ReadContent(); break; } } } return(true); }
private static async Task <List <object> > ParseExport(XmlReader xmlReader) { List <object> result = new List <object>(); { var feedReader = new AtomFeedReader(xmlReader); while (await feedReader.Read()) { switch (feedReader.ElementType) { // Read category case SyndicationElementType.Category: ISyndicationCategory category = await feedReader.ReadCategory(); result.Add(category); break; // Read Image case SyndicationElementType.Image: ISyndicationImage image = await feedReader.ReadImage(); result.Add(image); break; // Read Item case SyndicationElementType.Item: ISyndicationItem item = await feedReader.ReadItem(); result.Add(item); break; // Read link case SyndicationElementType.Link: ISyndicationLink link = await feedReader.ReadLink(); result.Add(link); break; // Read Person case SyndicationElementType.Person: ISyndicationPerson person = await feedReader.ReadPerson(); result.Add(person); break; // Read content default: ISyndicationContent content = await feedReader.ReadContent(); result.Add(content); break; } } return(result); } }
public FeedItem(ISyndicationItem item, DateTimeOffset recent, Uri website, bool truncateDescription, IExecutionContext context) { Title = item.Title; ISyndicationLink firstLink = item.Links.FirstOrDefault(x => x.RelationshipType == RssLinkTypes.Alternate); if (firstLink != null) { Link = firstLink.Uri.IsAbsoluteUri ? firstLink.Uri.AbsoluteUri : new Uri(website, firstLink.Uri).AbsoluteUri; } else { Link = item.Id; } Published = item.Published != default ? item.Published : item.LastUpdated; Recent = Published > recent; Links = item.Links .Where(x => !string.IsNullOrEmpty(x.MediaType)) .GroupBy(x => x.MediaType) .Select(x => x.First()) .ToDictionary(x => x.MediaType, x => x.Uri.ToString()); ISyndicationPerson person = item.Contributors.FirstOrDefault(x => x.RelationshipType == "author"); if (person != null) { Author = person.Name ?? person.Email; } Description = item.Description; AtomEntry atom = item as AtomEntry; if (atom != null && !string.IsNullOrEmpty(atom.Summary)) { Description = atom.Summary; } if (truncateDescription) { try { // Use the first <p> if one is found HtmlParser htmlParser = new HtmlParser(); IHtmlDocument htmlDocument = htmlParser.ParseDocument(Description); IElement element = htmlDocument.QuerySelector("p"); if (element is object && !string.IsNullOrWhiteSpace(element.OuterHtml)) { Description = element.OuterHtml; } } catch (Exception ex) { context.LogWarning($"Error parsing HTML description for feed {Title}: {ex.Message}"); } } }
public RssContent() { using (var xmlReader = XmlReader.Create("C:/Users/Developer/Desktop.brisbane-city-council.rss", new XmlReaderSettings() { Async = true })) { var feedReader = new RssFeedReader(xmlReader); while (await feedReader.Read()) { switch (feedReader.ElementType) { // Read category case SyndicationElementType.Category: ISyndicationCategory category = await feedReader.ReadCategory(); break; // Read Image case SyndicationElementType.Image: ISyndicationImage image = await feedReader.ReadImage(); break; // Read Item case SyndicationElementType.Item: ISyndicationItem item = await feedReader.ReadItem(); break; // Read link case SyndicationElementType.Link: ISyndicationLink link = await feedReader.ReadLink(); break; // Read Person case SyndicationElementType.Person: ISyndicationPerson person = await feedReader.ReadPerson(); break; // Read content default: ISyndicationContent content = await feedReader.ReadContent(); break; } } } }
public void AddContributor(ISyndicationPerson person) { if (person == null) { throw new ArgumentNullException(nameof(person)); } if (_contributors.IsReadOnly) { _contributors = _contributors.ToList(); } _contributors.Add(person); }
public virtual ISyndicationContent CreateContent(ISyndicationPerson person) { if (person == null) { throw new ArgumentNullException(nameof(person)); } // // RSS requires Email if (string.IsNullOrEmpty(person.Email)) { throw new ArgumentNullException("Invalid person Email"); } return(new SyndicationContent(person.RelationshipType ?? RssElementNames.Author, person.Email)); }
private void ConvertUser(IList <User> users, UserMapping[] mappings, ISyndicationPerson user) { var map = mappings.Single(candidate => string.Equals(candidate.FromGooglePlusUrl, user.Uri)); var result = new User { Id = map?.Id ?? Guid.NewGuid().ToString(), Name = map?.Name ?? user.Name, Email = map.ToEmail, Slug = map.Slug }; result.CreatedBy = result.Id; result.UpdatedBy = result.Id; _userLookup.Add(map.FromGooglePlusUrl, result.Id); users.Add(result); }
public virtual ISyndicationContent CreateContent(ISyndicationPerson person) { if (person == null) { throw new ArgumentNullException(nameof(person)); } if (string.IsNullOrEmpty(person.Name)) { throw new ArgumentNullException("Name"); } string contributorType = person.RelationshipType ?? AtomContributorTypes.Author; if (contributorType != AtomContributorTypes.Author && contributorType != AtomContributorTypes.Contributor) { throw new ArgumentException("RelationshipType"); } var result = new SyndicationContent(contributorType); // // name result.AddField(new SyndicationContent(AtomElementNames.Name, person.Name)); // // email if (!string.IsNullOrEmpty(person.Email)) { result.AddField(new SyndicationContent(AtomElementNames.Email, person.Email)); } // // uri if (person.Uri != null) { result.AddField(new SyndicationContent(AtomElementNames.Uri, FormatValue(person.Uri))); } return(result); }
private async Task ReadWhile() { using (var xmlReader = XmlReader.Create(@"..\..\..\TestFeeds\rss20.xml", new XmlReaderSettings() { Async = true })) { var reader = new RssFeedReader(xmlReader); while (await reader.Read()) { switch (reader.ElementType) { case SyndicationElementType.Link: ISyndicationLink link = await reader.ReadLink(); break; case SyndicationElementType.Item: ISyndicationItem item = await reader.ReadItem(); break; case SyndicationElementType.Person: ISyndicationPerson person = await reader.ReadPerson(); break; case SyndicationElementType.Image: ISyndicationImage image = await reader.ReadImage(); break; default: ISyndicationContent content = await reader.ReadContent(); break; } } } }
public virtual ISyndicationContent CreateContent(ISyndicationPerson person) { if (person == null) { throw new ArgumentNullException(nameof(person)); } // // RSS requires Email if (string.IsNullOrEmpty(person.Email)) { throw new ArgumentNullException("Invalid person Email"); } // // Real name recommended with RSS e-mail addresses // Ex: <author>[email protected] (John Doe)</author> string value = string.IsNullOrEmpty(person.Name) ? person.Email : $"{person.Email} ({person.Name})"; return(new SyndicationContent(person.RelationshipType ?? RssElementNames.Author, value)); }
public async Task <Podcast> ParseRssFeed(Podcast podcastData, bool isUpdate = false) { string title = string.Empty, imageUrl = string.Empty, description = string.Empty, author = string.Empty; DateTime pubDate = DateTime.MinValue, lastUpdateDate = DateTime.MinValue; List <ISyndicationItem> items = new List <ISyndicationItem>(); IEnumerable <PodcastItem> podcastItems = Enumerable.Empty <PodcastItem>(); Podcast podcast = null; using (var xmlReader = XmlReader.Create(podcastData.Url, new XmlReaderSettings { Async = true })) { var feedReader = new RssFeedReader(xmlReader); while (await feedReader.Read()) { switch (feedReader.ElementType) { case SyndicationElementType.Category: ISyndicationCategory category = await feedReader.ReadCategory(); break; case SyndicationElementType.Content: ISyndicationContent content = await feedReader.ReadContent(); if (content.Name == "title") { title = content.Value; } if (content.Name == "description") { description = content.Value; } if (content.Name == "author") { author = content.Value; } break; case SyndicationElementType.Image: ISyndicationImage image = await feedReader.ReadImage(); imageUrl = image.Url?.AbsoluteUri; break; case SyndicationElementType.Item: ISyndicationItem item = await feedReader.ReadItem(); items.Add(item); break; case SyndicationElementType.Link: ISyndicationLink link = await feedReader.ReadLink(); break; case SyndicationElementType.Person: ISyndicationPerson person = await feedReader.ReadPerson(); break; case SyndicationElementType.None: default: break; } } pubDate = items.Max(item => item.Published.DateTime); if (isUpdate) { lastUpdateDate = podcastData.LastUpdateDate; podcastData.Author = author; podcastData.Title = title; podcastData.ImageUrl = imageUrl; podcastData.Description = description; podcastData.LastUpdateDate = pubDate; podcast = podcastData; await dataService.Update <Podcast>(podcast); } else { podcast = new Podcast(title, podcastData.Url, imageUrl, description, author) { LastUpdateDate = pubDate == DateTime.MinValue ? DateTime.Now : pubDate }; await dataService.Insert <Podcast>(podcast); } podcastItems = items.Select(item => new { item.Title, item.Description, Enclosure = item.Links.FirstOrDefault(linkItem => linkItem.RelationshipType == "enclosure"), PublishDate = item.Published.DateTime }).Select(data => new PodcastItem(data.Title, data.Description, data.Enclosure.Uri.OriginalString, data.Enclosure.Length, data.PublishDate, podcast.Id)) .Where(item => item.PublishDate > lastUpdateDate) .ToList(); await dataService.Insert(podcastItems); podcast.PodcastItems = podcastItems.ToList(); await dataService.Update(podcast); } return(podcast); }
void ComparePerson(ISyndicationPerson person1, ISyndicationPerson person2) { Assert.True(person1.Email == person2.Email); Assert.True(person1.RelationshipType == person2.RelationshipType); }
public string Format(ISyndicationPerson person) { return(Format(CreateContent(person))); }
public async Task <Channel9RssResult> Parse(Uri rssUri) { var result = new Channel9RssResult(); result.SourceUrl = rssUri; try { using (var client = new HttpClient()) { result.RawXml = await client.GetStringAsync(rssUri); using (var xmlReader = XmlReader.Create(new StringReader(result.RawXml), new XmlReaderSettings() { Async = false })) { var feedReader = new RssFeedReader(xmlReader); while (await feedReader.Read()) { switch (feedReader.ElementType) { // Read category case SyndicationElementType.Category: ISyndicationCategory category = await feedReader.ReadCategory(); break; // Read Image case SyndicationElementType.Image: ISyndicationImage image = await feedReader.ReadImage(); break; // Read Item case SyndicationElementType.Item: // parse the syndication item ISyndicationItem item = await feedReader.ReadItem(); result.SyndicationItems.Add(item); // then construct a session info var si = new SessionInfo(); si.SessionID = item.Id.Substring(item.Id.LastIndexOf("/", StringComparison.InvariantCultureIgnoreCase) + 1); si.Title = item.Title; si.SessionSite = new Uri(item.Id); si.PublishDate = item.Published.DateTime; si.Presenter = item.Contributors.FirstOrDefault()?.Name; result.Sessions.Add(si); foreach (var v in item.Links) { if (!string.IsNullOrWhiteSpace(v.MediaType)) { si.VideoRecordings.Add(new VideoRecording() { SessionInfo = si, Url = v.Uri, MediaType = v.MediaType, Length = v.Length }); } } break; // Read link case SyndicationElementType.Link: ISyndicationLink link = await feedReader.ReadLink(); break; // Read Person case SyndicationElementType.Person: ISyndicationPerson person = await feedReader.ReadPerson(); break; // Read content default: ISyndicationContent content = await feedReader.ReadContent(); break; } } } } } catch (Exception ex) { result.Exceptions = new List <Exception>(); result.Exceptions.Add(ex); } return(result); }
public static async Task ReadFeed() { using (XmlReader xmlReader = XmlReader.Create(@"..\DemoTest1\TestFeeds\rss20.xml", new XmlReaderSettings() { Async = true })) using (XmlWriter xmlWriter = XmlWriter.Create(Console.Out, new XmlWriterSettings() { Indent = true, Async = true })) { Rss20FeedReader reader = new Rss20FeedReader(xmlReader); Rss20FeedWriter writer = new Rss20FeedWriter(xmlWriter); while (await reader.Read()) { switch (reader.ElementType) { case SyndicationElementType.Category: ISyndicationCategory category = await reader.ReadCategory(); await writer.Write(category); break; case SyndicationElementType.Image: ISyndicationImage image = await reader.ReadImage(); await writer.Write(image); break; case SyndicationElementType.Item: ISyndicationItem item = await reader.ReadItem(); await writer.Write(item); break; case SyndicationElementType.Link: ISyndicationLink link = await reader.ReadLink(); await writer.Write(link); break; case SyndicationElementType.Person: ISyndicationPerson person = await reader.ReadPerson(); await writer.Write(person); break; default: ISyndicationContent content = await reader.ReadContent(); await writer.Write(content); break; } xmlWriter.Flush(); // Simulate a slow stream. Thread.Sleep(300); } } }
public string Format(ISyndicationPerson person) { ISyndicationContent content = CreateContent(person); return(Format(content)); }
internal void DisplayPerson(ISyndicationPerson person) { Console.WriteLine("--- Person Read ---"); Console.WriteLine("Email: " + person.Email); Console.WriteLine(); }
private async Task <List <BlogPost> > GetFeed() { var xmlReader = XmlReader.Create(_settings.Value.FeedUrl); var reader = new AtomFeedReader(xmlReader); var posts = new List <BlogPost>(); // // Read the feed while (await reader.Read()) { // // Check the type of the current element. switch (reader.ElementType) { // // Read category case SyndicationElementType.Category: ISyndicationCategory category = await reader.ReadCategory(); break; // // Read image case SyndicationElementType.Image: ISyndicationImage image = await reader.ReadImage(); break; // // Read entry case SyndicationElementType.Item: IAtomEntry entry = await reader.ReadEntry(); // these are the only ones we need for now posts.Add(new BlogPost { Title = entry.Title, PublishedDate = entry.Published.DateTime, Categories = entry.Categories?.Select(c => c.Name)?.ToList() }); break; // // Read link case SyndicationElementType.Link: ISyndicationLink link = await reader.ReadLink(); break; // // Read person case SyndicationElementType.Person: ISyndicationPerson person = await reader.ReadPerson(); break; // // Read content default: ISyndicationContent content = await reader.ReadContent(); break; } } return(posts); }
public virtual Task Write(ISyndicationPerson person) { return(WriteRaw(Formatter.Format(person ?? throw new ArgumentNullException(nameof(person))))); }
async static Task <List <Tuple <string, string, DateTimeOffset> > > Latest5PostsFromRssAsync(string filePath) { List <Tuple <string, string, DateTimeOffset> > result = new List <Tuple <string, string, DateTimeOffset> >(); using (var xmlReader = XmlReader.Create(filePath, new XmlReaderSettings() { Async = true })) { var feedReader = new RssFeedReader(xmlReader); while (await feedReader.Read()) { switch (feedReader.ElementType) { // Read category case SyndicationElementType.Category: ISyndicationCategory category = await feedReader.ReadCategory(); break; // Read Image case SyndicationElementType.Image: ISyndicationImage image = await feedReader.ReadImage(); break; // Read Item case SyndicationElementType.Item: ISyndicationItem item = await feedReader.ReadItem(); result.Add(new Tuple <string, string, DateTimeOffset>(item.Title, item.Links.First().Uri.ToString(), item.Published)); break; // Read link case SyndicationElementType.Link: ISyndicationLink link = await feedReader.ReadLink(); break; // Read Person case SyndicationElementType.Person: ISyndicationPerson person = await feedReader.ReadPerson(); break; // Read content default: ISyndicationContent content = await feedReader.ReadContent(); break; } if (result.Count == 5) { break; } } } return(result); }
public async Task ConsumeFeed(string filePath) { bool verbose = false; // Information to display. double size = new FileInfo(filePath).Length; double currentSize = 0; int itemsRead = 0; // Transform the size of the file to Kb - Mb - Gb. Tuple <double, string> sizeInfo = utils.ConvertBytesToSize(size); // Display the Size of the feed and ask for verbose. Console.ForegroundColor = ConsoleColor.DarkGreen; Console.WriteLine("Size of the Feed: {0:N2} {1}", sizeInfo.Item1, sizeInfo.Item2); Console.Write("Verbose Items (Y/N): "); Console.ForegroundColor = ConsoleColor.White; string input = Console.ReadLine(); verbose = utils.ValidateVerbose(input); Console.CursorVisible = false; Stopwatch stopWatch = null; using (var xmlReader = XmlReader.Create(filePath, new XmlReaderSettings() { Async = true })) { var reader = new Rss20FeedReader(xmlReader); stopWatch = new Stopwatch(); stopWatch.Start(); ElementDisplayer displayer = new ElementDisplayer(); while (await reader.Read()) { if (verbose) { utils.ClearInformation(); } switch (reader.ElementType) { case SyndicationElementType.Content: ISyndicationContent content = await reader.ReadContent(); if (verbose) { displayer.DisplayContent(content); } break; case SyndicationElementType.Item: ISyndicationItem item = await ReadItem(reader); itemsRead++; if (verbose) { displayer.DisplayItem(item); } currentSize += _sizeOfItem; break; case SyndicationElementType.Person: ISyndicationPerson person = await ReadPerson(reader); if (verbose) { displayer.DisplayPerson(person); } break; case SyndicationElementType.Image: ISyndicationImage image = await ReadImage(reader); if (verbose) { displayer.DisplayImage(image); } break; case SyndicationElementType.Link: ISyndicationLink link = await ReadLink(reader); if (verbose) { displayer.DisplayLink(link); } break; case SyndicationElementType.Category: ISyndicationCategory category = await ReadCategory(reader); if (verbose) { displayer.DisplayCategory(category); } break; } double percentage = ((currentSize * 100) / size); if (itemsRead % 200 == 0) { utils.WriteInformation(percentage, itemsRead, stopWatch.Elapsed); } } } utils.ClearInformation(); //Print end of reading Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("Finished Reading, press enter to close.\n\n"); utils.WriteInformation(100, itemsRead, stopWatch.Elapsed); Console.ReadLine(); }
public static async Task ReadAtomFeed(string filePath) { // // Create an XmlReader from file // Example: ..\tests\TestFeeds\simpleAtomFeed.xml using (XmlReader xmlReader = XmlReader.Create(filePath, new XmlReaderSettings() { Async = true })) { // // Create an AtomFeedReader var reader = new AtomFeedReader(xmlReader); // // Read the feed while (await reader.Read()) { // // Check the type of the current element. switch (reader.ElementType) { // // Read category case SyndicationElementType.Category: ISyndicationCategory category = await reader.ReadCategory(); break; // // Read image case SyndicationElementType.Image: ISyndicationImage image = await reader.ReadImage(); break; // // Read entry case SyndicationElementType.Item: IAtomEntry entry = await reader.ReadEntry(); break; // // Read link case SyndicationElementType.Link: ISyndicationLink link = await reader.ReadLink(); break; // // Read person case SyndicationElementType.Person: ISyndicationPerson person = await reader.ReadPerson(); break; // // Read content default: ISyndicationContent content = await reader.ReadContent(); break; } } } }
protected override async Task <IEnumerable <IDocument> > ExecuteInputAsync(IDocument input, IExecutionContext context) { string feed = input.GetString("Feed"); if (!string.IsNullOrEmpty(feed)) { try { // Download the feed context.LogInformation($"Getting feed for {feed}"); Uri website = null; string title = null; string author = null; string description = null; string image = null; List <ISyndicationItem> items = new List <ISyndicationItem>(); using (HttpClient httpClient = context.CreateHttpClient()) { httpClient.DefaultRequestHeaders.Add("User-Agent", "Wyam"); var response = await httpClient.GetAsync(feed); if (response.StatusCode == HttpStatusCode.Redirect || response.StatusCode == HttpStatusCode.MovedPermanently) { context.LogWarning($"Attempting to follow redirect for {feed}"); feed = response.Headers.Location.OriginalString; response = await httpClient.GetAsync(feed); } if (response.IsSuccessStatusCode) { using (Stream stream = await response.Content.ReadAsStreamAsync()) using (StreamReader streamReader = new XmlStreamReader(stream)) using (XmlReader xmlReader = XmlReader.Create(streamReader, new XmlReaderSettings { Async = true, DtdProcessing = DtdProcessing.Ignore })) { xmlReader.MoveToContent(); bool atom = xmlReader.Name == "feed"; context.LogInformation($"Reading {feed} as " + (atom ? "Atom" : "RSS")); XmlFeedReader feedReader = atom ? (XmlFeedReader) new AtomFeedReader(xmlReader, new FixedAtomParser()) : new RssFeedReader(xmlReader); while (await feedReader.Read()) { try { switch (feedReader.ElementType) { case SyndicationElementType.Person: ISyndicationPerson person = await feedReader.ReadPerson(); if (person.RelationshipType == "author") { author = person.Name ?? person.Email; } break; case SyndicationElementType.Image: ISyndicationImage img = await feedReader.ReadImage(); image = img.Url.ToString(); break; case SyndicationElementType.Link: ISyndicationLink link = await feedReader.ReadLink(); website = link.Uri; break; case SyndicationElementType.Item: ISyndicationItem item = await feedReader.ReadItem(); items.Add(item); break; case SyndicationElementType.None: break; default: ISyndicationContent content = await feedReader.ReadContent(); if (string.Equals(content.Name, "title", StringComparison.OrdinalIgnoreCase)) { title = content.Value; } else if (string.Equals(content.Name, "description", StringComparison.OrdinalIgnoreCase) || string.Equals(content.Name, "subtitle", StringComparison.OrdinalIgnoreCase)) { description = content.Value; } break; } } catch (Exception ex) { context.LogWarning($"Exception while processing {feedReader.ElementType} in {feed}: {ex.Message}"); } } } } // Get a new document with feed metadata MetadataItems metadata = new MetadataItems(); if (items.Count > 0) { BlogFeedItem[] feedItems = items .Select(x => new BlogFeedItem(x, _recent, website)) .OrderByDescending(x => x.Published) .Take(50) // Only take the 50 most recent items .ToArray(); metadata.Add(SiteKeys.BlogFeedItems, feedItems); return(input.Clone(metadata).Yield()); } } } catch (Exception ex) { context.LogWarning($"Error getting feed for {feed}: {ex.Message}"); } } else { var speaker = input.GetString("Title"); if (!string.IsNullOrWhiteSpace(speaker)) { context.LogWarning($"No RSS specified for {speaker}."); } else { context.LogWarning($"No RSS specified for unknown speaker."); } } return(input.Yield()); }
protected override async Task <IEnumerable <IDocument> > ExecuteInputAsync(IDocument input, IExecutionContext context) { // Don't get data if we're just validating if (context.Settings.GetBool(SiteKeys.Validate)) { return(null); } string feed = input.GetString(SiteKeys.Feed); if (!string.IsNullOrEmpty(feed)) { try { // Download the feed context.LogInformation($"Getting feed for {feed}"); Uri website = null; string title = null; string author = null; string description = null; string image = null; List <ISyndicationItem> items = new List <ISyndicationItem>(); using (HttpClient httpClient = context.CreateHttpClient()) { httpClient.DefaultRequestHeaders.Add("User-Agent", nameof(Statiq)); using (Stream stream = await httpClient.GetStreamAsync(feed)) { using (StreamReader streamReader = new XmlStreamReader(stream)) { using (XmlReader xmlReader = XmlReader.Create(streamReader, new XmlReaderSettings { Async = true, DtdProcessing = DtdProcessing.Ignore })) { xmlReader.MoveToContent(); bool atom = xmlReader.Name == "feed"; context.LogInformation($"Reading {feed} as " + (atom ? "Atom" : "RSS")); XmlFeedReader feedReader = atom ? (XmlFeedReader) new AtomFeedReader(xmlReader, new DiscoverAtomParser()) : new RssFeedReader(xmlReader); while (await feedReader.Read()) { try { switch (feedReader.ElementType) { case SyndicationElementType.Person: ISyndicationPerson person = await feedReader.ReadPerson(); if (person.RelationshipType == "author") { author = person.Name ?? person.Email; } break; case SyndicationElementType.Image: ISyndicationImage img = await feedReader.ReadImage(); image = img.Url.ToString(); break; case SyndicationElementType.Link: ISyndicationLink link = await feedReader.ReadLink(); website = link.Uri; break; case SyndicationElementType.Item: ISyndicationItem item = await feedReader.ReadItem(); items.Add(item); break; case SyndicationElementType.None: break; default: ISyndicationContent content = await feedReader.ReadContent(); if (string.Equals(content.Name, "title", StringComparison.OrdinalIgnoreCase)) { title = content.Value; } else if (string.Equals(content.Name, "description", StringComparison.OrdinalIgnoreCase) || string.Equals(content.Name, "subtitle", StringComparison.OrdinalIgnoreCase)) { description = content.Value; } break; } } catch (Exception ex) { context.LogWarning($"Exception while processing {feedReader.ElementType} in {feed}: {ex.Message}"); } } } } } } // Get a new document with feed metadata MetadataItems metadata = new MetadataItems(); if (items.Count > 0) { FeedItem[] feedItems = items .Select(x => new FeedItem(x, _recent, website, _truncateDescription, context)) .OrderByDescending(x => x.Published) .Take(50) // Only take the 50 most recent items .ToArray(); metadata.Add(SiteKeys.FeedItems, feedItems); metadata.Add(SiteKeys.LastPublished, feedItems.First().Published); metadata.Add(SiteKeys.NewestFeedItem, feedItems[0]); } if (!input.ContainsKey(SiteKeys.Website) && website != null) { metadata.Add(SiteKeys.Website, website.ToString()); } if (!input.ContainsKey(SiteKeys.Title)) { if (!string.IsNullOrEmpty(title)) { metadata.Add(SiteKeys.Title, title); } else { string generatedTitle = GenerateTitleFromFeedName(feed); metadata.Add(SiteKeys.Title, generatedTitle); } } if (!input.ContainsKey(SiteKeys.Author) && !string.IsNullOrEmpty(author)) { metadata.Add(SiteKeys.Author, author); } if (!input.ContainsKey(SiteKeys.Description) && !string.IsNullOrEmpty(description)) { metadata.Add(SiteKeys.Description, description); } if (!input.ContainsKey(SiteKeys.Image) && !string.IsNullOrEmpty(image)) { metadata.Add(SiteKeys.Image, image); } return(input.Clone(metadata).Yield()); } catch (Exception ex) { context.LogWarning($"Error getting feed for {feed}: {ex.Message}"); } } // Return null so this feed is not included return(null); }