void CompareImage(ISyndicationImage image1, ISyndicationImage image2)
{
Assert.True(image1.RelationshipType == image2.RelationshipType);
Assert.True(image1.Url == image2.Url);
Assert.True(image1.Link.Uri.ToString() == image2.Link.Uri.ToString());
Assert.True(image1.Description == image2.Description);
}
public async Task ReadImage()
{
using (XmlReader xmlReader = XmlReader.Create(@"..\..\..\TestFeeds\simpleAtomFeed.xml", new XmlReaderSettings {
Async = true
}))
{
var reader = new AtomFeedReader(xmlReader);
int imagesRead = 0;
List <String> contentsOfImages = new List <string>();
while (await reader.Read())
{
if (reader.ElementType == SyndicationElementType.Image)
{
ISyndicationImage image = await reader.ReadImage();
imagesRead++;
contentsOfImages.Add(image.Url.OriginalString);
}
}
Assert.True(imagesRead == 2);
Assert.True(contentsOfImages[0] == "/icon.jpg");
Assert.True(contentsOfImages[1] == "/logo.jpg");
}
}
private async Task <ISyndicationImage> ReadImage(Rss20FeedReader reader)
{
ISyndicationImage image = null;
image = await reader.ReadImage();
return(image);
}
// Read an RssFeed
public static async Task CreateRssFeedReaderExample(string filePath)
{
// Create an XmlReader
// Example: ..\tests\TestFeeds\rss20-2items.xml
using (var xmlReader = XmlReader.Create(filePath, new XmlReaderSettings()
{
Async = true
}))
{
// Instantiate an Rss20FeedReader using the XmlReader.
// This will assign as default an Rss20FeedParser as the parser.
var feedReader = new RssFeedReader(xmlReader);
//
// Read the feed
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await feedReader.ReadCategory();
break;
// Read Image
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
break;
// Read Item
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
break;
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
break;
// Read Person
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
break;
// Read content
default:
ISyndicationContent content = await feedReader.ReadContent();
break;
}
}
}
}
public static async Task TestReadFeedElements(XmlReader outerXmlReader)
{
using (var xmlReader = outerXmlReader)
{
var reader = new RssFeedReader(xmlReader);
int items = 0;
while (await reader.Read())
{
switch (reader.ElementType)
{
case SyndicationElementType.Person:
ISyndicationPerson person = await reader.ReadPerson();
Assert.True(person.Email == "John Smith");
break;
case SyndicationElementType.Link:
ISyndicationLink link = await reader.ReadLink();
Assert.True(link.Length == 123);
Assert.True(link.MediaType == "testType");
Assert.True(link.Uri.OriginalString == "http://example.com/");
break;
case SyndicationElementType.Image:
ISyndicationImage image = await reader.ReadImage();
Assert.True(image.Title == "Microsoft News");
Assert.True(image.Description == "Test description");
Assert.True(image.Url.OriginalString == "http://2.bp.blogspot.com/-NA5Jb-64eUg/URx8CSdcj_I/AAAAAAAAAUo/eCx0irI0rq0/s1600/bg_Microsoft_logo3-20120824073001907469-620x349.jpg");
break;
case SyndicationElementType.Item:
items++;
ISyndicationItem item = await reader.ReadItem();
if (items == 1)
{
Assert.True(item.Title == "Lorem ipsum 2017-07-06T20:25:00+00:00");
Assert.True(item.Description == "Exercitation sit dolore mollit et est eiusmod veniam aute officia veniam ipsum.");
Assert.True(item.Links.Count() == 3);
}
else if (items == 2)
{
Assert.True(item.Title == "Lorem ipsum 2017-07-06T20:24:00+00:00");
Assert.True(item.Description == "Do ipsum dolore veniam minim est cillum aliqua ea.");
Assert.True(item.Links.Count() == 3);
}
break;
default:
break;
}
}
}
}
private async Task <bool> retrievePosts(Feed feed)
{
using (var xmlReader = XmlReader.Create("https://azurecomcdn.azureedge.net/en-us/blog/feed/", new XmlReaderSettings()
{
Async = true
}))
{
// Instantiate an Rss20FeedReader using the XmlReader.
// This will assign as default an Rss20FeedParser as the parser.
var feedReader = new RssFeedReader(xmlReader);
//
// Read the feed
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await feedReader.ReadCategory();
break;
// Read Image
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
break;
// Read Item
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
break;
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
break;
// Read Person
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
break;
// Read content
default:
ISyndicationContent content = await feedReader.ReadContent();
break;
}
}
}
return(true);
}
private static async Task <List <object> > ParseExport(XmlReader xmlReader)
{
List <object> result = new List <object>();
{
var feedReader = new AtomFeedReader(xmlReader);
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await feedReader.ReadCategory();
result.Add(category);
break;
// Read Image
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
result.Add(image);
break;
// Read Item
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
result.Add(item);
break;
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
result.Add(link);
break;
// Read Person
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
result.Add(person);
break;
// Read content
default:
ISyndicationContent content = await feedReader.ReadContent();
result.Add(content);
break;
}
}
return(result);
}
}
public RssContent()
{
using (var xmlReader = XmlReader.Create("C:/Users/Developer/Desktop.brisbane-city-council.rss", new XmlReaderSettings()
{
Async = true
}))
{
var feedReader = new RssFeedReader(xmlReader);
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await feedReader.ReadCategory();
break;
// Read Image
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
break;
// Read Item
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
break;
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
break;
// Read Person
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
break;
// Read content
default:
ISyndicationContent content = await feedReader.ReadContent();
break;
}
}
}
}
public virtual ISyndicationContent CreateContent(ISyndicationImage image)
{
if (image == null)
{
throw new ArgumentNullException(nameof(image));
}
if (image.Url == null)
{
throw new ArgumentNullException("Url");
}
return(new SyndicationContent(!string.IsNullOrEmpty(image.RelationshipType) ? image.RelationshipType : AtomImageTypes.Icon,
FormatValue(image.Url)));
}
private async Task ReadWhile()
{
using (var xmlReader = XmlReader.Create(@"..\..\..\TestFeeds\rss20.xml", new XmlReaderSettings()
{
Async = true
}))
{
var reader = new RssFeedReader(xmlReader);
while (await reader.Read())
{
switch (reader.ElementType)
{
case SyndicationElementType.Link:
ISyndicationLink link = await reader.ReadLink();
break;
case SyndicationElementType.Item:
ISyndicationItem item = await reader.ReadItem();
break;
case SyndicationElementType.Person:
ISyndicationPerson person = await reader.ReadPerson();
break;
case SyndicationElementType.Image:
ISyndicationImage image = await reader.ReadImage();
break;
default:
ISyndicationContent content = await reader.ReadContent();
break;
}
}
}
}
public virtual ISyndicationContent CreateContent(ISyndicationImage image)
{
if (image == null)
{
throw new ArgumentNullException(nameof(image));
}
// Required URL - Title - Link
if (string.IsNullOrEmpty(image.Title))
{
throw new ArgumentNullException("Image requires a title");
}
if (image.Link == null)
{
throw new ArgumentNullException("Image requires a link");
}
if (image.Url == null)
{
throw new ArgumentNullException("Image requires an url");
}
var content = new SyndicationContent(RssElementNames.Image);
// Write required contents of image
content.AddField(new SyndicationContent(RssElementNames.Url, FormatValue(image.Url)));
content.AddField(new SyndicationContent(RssElementNames.Title, image.Title));
content.AddField(CreateContent(image.Link));
// Write optional elements
if (!string.IsNullOrEmpty(image.Description))
{
content.AddField(new SyndicationContent(RssElementNames.Description, image.Description));
}
return(content);
}
public async Task <EmbedBuilder> TrackFeeds()
{
while (true)
{
foreach (var kvp in _subs)
{
if (kvp.Value.Count == 0)
{
continue;
}
if (!_lastPosts.TryGetValue(kvp.Key, out DateTime lastTime))
{
lastTime = _lastPosts.AddOrUpdate(kvp.Key, DateTime.UtcNow, (k, old) => DateTime.UtcNow);
}
var rssUrl = kvp.Key;
try
{
using (var xmlReader = XmlReader.Create(rssUrl, new XmlReaderSettings()
{
Async = true
}))
{
var feedReader = new RssFeedReader(xmlReader);
var embed = new EmbedBuilder()
.WithAuthor(kvp.Key)
.WithOkColor();
while (await feedReader.Read() && feedReader.ElementType != SyndicationElementType.Item)
{
switch (feedReader.ElementType)
{
case SyndicationElementType.Link:
var uri = await feedReader.ReadLink();
embed.WithAuthor(kvp.Key, url: uri.Uri.AbsoluteUri);
break;
case SyndicationElementType.Content:
var content = await feedReader.ReadContent();
break;
case SyndicationElementType.Category:
break;
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
embed.WithThumbnailUrl(image.Url.AbsoluteUri);
break;
default:
break;
}
}
ISyndicationItem item = await feedReader.ReadItem();
if (item.Published.UtcDateTime <= lastTime)
{
continue;
}
var desc = item.Description.StripHTML();
lastTime = item.Published.UtcDateTime;
var title = string.IsNullOrWhiteSpace(item.Title) ? "-" : item.Title;
desc = Format.Code(item.Published.ToString()) + Environment.NewLine + desc;
var link = item.Links.FirstOrDefault();
if (link != null)
{
desc = $"[link]({link.Uri}) " + desc;
}
var img = item.Links.FirstOrDefault(x => x.RelationshipType == "enclosure")?.Uri.AbsoluteUri
?? Regex.Match(item.Description, @"src=""(?<src>.*?)""").Groups["src"].ToString();
if (!string.IsNullOrWhiteSpace(img) && Uri.IsWellFormedUriString(img, UriKind.Absolute))
{
embed.WithImageUrl(img);
}
embed.AddField(title, desc);
//send the created embed to all subscribed channels
var sendTasks = kvp.Value
.Where(x => x.GuildConfig != null)
.Select(x => _client.GetGuild(x.GuildConfig.GuildId)
?.GetTextChannel(x.ChannelId))
.Where(x => x != null)
.Select(x => x.EmbedAsync(embed));
_lastPosts.AddOrUpdate(kvp.Key, item.Published.UtcDateTime, (k, old) => item.Published.UtcDateTime);
await Task.WhenAll(sendTasks).ConfigureAwait(false);
}
}
catch { }
}
await Task.Delay(10000);
}
}
public async Task <Channel9RssResult> Parse(Uri rssUri)
{
var result = new Channel9RssResult();
result.SourceUrl = rssUri;
try
{
using (var client = new HttpClient())
{
result.RawXml = await client.GetStringAsync(rssUri);
using (var xmlReader = XmlReader.Create(new StringReader(result.RawXml), new XmlReaderSettings()
{
Async = false
}))
{
var feedReader = new RssFeedReader(xmlReader);
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await feedReader.ReadCategory();
break;
// Read Image
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
break;
// Read Item
case SyndicationElementType.Item:
// parse the syndication item
ISyndicationItem item = await feedReader.ReadItem();
result.SyndicationItems.Add(item);
// then construct a session info
var si = new SessionInfo();
si.SessionID = item.Id.Substring(item.Id.LastIndexOf("/", StringComparison.InvariantCultureIgnoreCase) + 1);
si.Title = item.Title;
si.SessionSite = new Uri(item.Id);
si.PublishDate = item.Published.DateTime;
si.Presenter = item.Contributors.FirstOrDefault()?.Name;
result.Sessions.Add(si);
foreach (var v in item.Links)
{
if (!string.IsNullOrWhiteSpace(v.MediaType))
{
si.VideoRecordings.Add(new VideoRecording()
{
SessionInfo = si, Url = v.Uri, MediaType = v.MediaType, Length = v.Length
});
}
}
break;
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
break;
// Read Person
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
break;
// Read content
default:
ISyndicationContent content = await feedReader.ReadContent();
break;
}
}
}
}
}
catch (Exception ex)
{
result.Exceptions = new List <Exception>();
result.Exceptions.Add(ex);
}
return(result);
}
public string Format(ISyndicationImage image)
{
ISyndicationContent content = CreateContent(image);
return(Format(content));
}
public static async Task ReadAtomFeed(string filePath)
{
//
// Create an XmlReader from file
// Example: ..\tests\TestFeeds\simpleAtomFeed.xml
using (XmlReader xmlReader = XmlReader.Create(filePath, new XmlReaderSettings()
{
Async = true
}))
{
//
// Create an AtomFeedReader
var reader = new AtomFeedReader(xmlReader);
//
// Read the feed
while (await reader.Read())
{
//
// Check the type of the current element.
switch (reader.ElementType)
{
//
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await reader.ReadCategory();
break;
//
// Read image
case SyndicationElementType.Image:
ISyndicationImage image = await reader.ReadImage();
break;
//
// Read entry
case SyndicationElementType.Item:
IAtomEntry entry = await reader.ReadEntry();
break;
//
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await reader.ReadLink();
break;
//
// Read person
case SyndicationElementType.Person:
ISyndicationPerson person = await reader.ReadPerson();
break;
//
// Read content
default:
ISyndicationContent content = await reader.ReadContent();
break;
}
}
}
}
public async Task <Podcast> ParseRssFeed(Podcast podcastData, bool isUpdate = false)
{
string title = string.Empty,
imageUrl = string.Empty,
description = string.Empty,
author = string.Empty;
DateTime pubDate = DateTime.MinValue,
lastUpdateDate = DateTime.MinValue;
List <ISyndicationItem> items = new List <ISyndicationItem>();
IEnumerable <PodcastItem> podcastItems = Enumerable.Empty <PodcastItem>();
Podcast podcast = null;
using (var xmlReader = XmlReader.Create(podcastData.Url, new XmlReaderSettings {
Async = true
}))
{
var feedReader = new RssFeedReader(xmlReader);
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
case SyndicationElementType.Category:
ISyndicationCategory category = await feedReader.ReadCategory();
break;
case SyndicationElementType.Content:
ISyndicationContent content = await feedReader.ReadContent();
if (content.Name == "title")
{
title = content.Value;
}
if (content.Name == "description")
{
description = content.Value;
}
if (content.Name == "author")
{
author = content.Value;
}
break;
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
imageUrl = image.Url?.AbsoluteUri;
break;
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
items.Add(item);
break;
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
break;
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
break;
case SyndicationElementType.None:
default:
break;
}
}
pubDate = items.Max(item => item.Published.DateTime);
if (isUpdate)
{
lastUpdateDate = podcastData.LastUpdateDate;
podcastData.Author = author;
podcastData.Title = title;
podcastData.ImageUrl = imageUrl;
podcastData.Description = description;
podcastData.LastUpdateDate = pubDate;
podcast = podcastData;
await dataService.Update <Podcast>(podcast);
}
else
{
podcast = new Podcast(title, podcastData.Url, imageUrl, description, author)
{
LastUpdateDate = pubDate == DateTime.MinValue ? DateTime.Now : pubDate
};
await dataService.Insert <Podcast>(podcast);
}
podcastItems = items.Select(item => new
{
item.Title,
item.Description,
Enclosure = item.Links.FirstOrDefault(linkItem => linkItem.RelationshipType == "enclosure"),
PublishDate = item.Published.DateTime
}).Select(data => new PodcastItem(data.Title, data.Description, data.Enclosure.Uri.OriginalString,
data.Enclosure.Length, data.PublishDate, podcast.Id))
.Where(item => item.PublishDate > lastUpdateDate)
.ToList();
await dataService.Insert(podcastItems);
podcast.PodcastItems = podcastItems.ToList();
await dataService.Update(podcast);
}
return(podcast);
}
internal void DisplayImage(ISyndicationImage image)
{
Console.WriteLine("--- Image Read ---");
Console.WriteLine("Image Link: " + image.Link.Uri.AbsoluteUri);
Console.WriteLine();
}
public static async Task ReadFeed()
{
using (XmlReader xmlReader = XmlReader.Create(@"..\DemoTest1\TestFeeds\rss20.xml", new XmlReaderSettings()
{
Async = true
}))
using (XmlWriter xmlWriter = XmlWriter.Create(Console.Out, new XmlWriterSettings()
{
Indent = true, Async = true
}))
{
Rss20FeedReader reader = new Rss20FeedReader(xmlReader);
Rss20FeedWriter writer = new Rss20FeedWriter(xmlWriter);
while (await reader.Read())
{
switch (reader.ElementType)
{
case SyndicationElementType.Category:
ISyndicationCategory category = await reader.ReadCategory();
await writer.Write(category);
break;
case SyndicationElementType.Image:
ISyndicationImage image = await reader.ReadImage();
await writer.Write(image);
break;
case SyndicationElementType.Item:
ISyndicationItem item = await reader.ReadItem();
await writer.Write(item);
break;
case SyndicationElementType.Link:
ISyndicationLink link = await reader.ReadLink();
await writer.Write(link);
break;
case SyndicationElementType.Person:
ISyndicationPerson person = await reader.ReadPerson();
await writer.Write(person);
break;
default:
ISyndicationContent content = await reader.ReadContent();
await writer.Write(content);
break;
}
xmlWriter.Flush();
// Simulate a slow stream.
Thread.Sleep(300);
}
}
}
public string Format(ISyndicationImage image)
{
return(Format(CreateContent(image)));
}
private async Task <List <BlogPost> > GetFeed()
{
var xmlReader = XmlReader.Create(_settings.Value.FeedUrl);
var reader = new AtomFeedReader(xmlReader);
var posts = new List <BlogPost>();
//
// Read the feed
while (await reader.Read())
{
//
// Check the type of the current element.
switch (reader.ElementType)
{
//
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await reader.ReadCategory();
break;
//
// Read image
case SyndicationElementType.Image:
ISyndicationImage image = await reader.ReadImage();
break;
//
// Read entry
case SyndicationElementType.Item:
IAtomEntry entry = await reader.ReadEntry();
// these are the only ones we need for now
posts.Add(new BlogPost
{
Title = entry.Title,
PublishedDate = entry.Published.DateTime,
Categories = entry.Categories?.Select(c => c.Name)?.ToList()
});
break;
//
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await reader.ReadLink();
break;
//
// Read person
case SyndicationElementType.Person:
ISyndicationPerson person = await reader.ReadPerson();
break;
//
// Read content
default:
ISyndicationContent content = await reader.ReadContent();
break;
}
}
return(posts);
}
public async Task ConsumeFeed(string filePath)
{
bool verbose = false;
// Information to display.
double size = new FileInfo(filePath).Length;
double currentSize = 0;
int itemsRead = 0;
// Transform the size of the file to Kb - Mb - Gb.
Tuple <double, string> sizeInfo = utils.ConvertBytesToSize(size);
// Display the Size of the feed and ask for verbose.
Console.ForegroundColor = ConsoleColor.DarkGreen;
Console.WriteLine("Size of the Feed: {0:N2} {1}", sizeInfo.Item1, sizeInfo.Item2);
Console.Write("Verbose Items (Y/N): ");
Console.ForegroundColor = ConsoleColor.White;
string input = Console.ReadLine();
verbose = utils.ValidateVerbose(input);
Console.CursorVisible = false;
Stopwatch stopWatch = null;
using (var xmlReader = XmlReader.Create(filePath, new XmlReaderSettings()
{
Async = true
}))
{
var reader = new Rss20FeedReader(xmlReader);
stopWatch = new Stopwatch();
stopWatch.Start();
ElementDisplayer displayer = new ElementDisplayer();
while (await reader.Read())
{
if (verbose)
{
utils.ClearInformation();
}
switch (reader.ElementType)
{
case SyndicationElementType.Content:
ISyndicationContent content = await reader.ReadContent();
if (verbose)
{
displayer.DisplayContent(content);
}
break;
case SyndicationElementType.Item:
ISyndicationItem item = await ReadItem(reader);
itemsRead++;
if (verbose)
{
displayer.DisplayItem(item);
}
currentSize += _sizeOfItem;
break;
case SyndicationElementType.Person:
ISyndicationPerson person = await ReadPerson(reader);
if (verbose)
{
displayer.DisplayPerson(person);
}
break;
case SyndicationElementType.Image:
ISyndicationImage image = await ReadImage(reader);
if (verbose)
{
displayer.DisplayImage(image);
}
break;
case SyndicationElementType.Link:
ISyndicationLink link = await ReadLink(reader);
if (verbose)
{
displayer.DisplayLink(link);
}
break;
case SyndicationElementType.Category:
ISyndicationCategory category = await ReadCategory(reader);
if (verbose)
{
displayer.DisplayCategory(category);
}
break;
}
double percentage = ((currentSize * 100) / size);
if (itemsRead % 200 == 0)
{
utils.WriteInformation(percentage, itemsRead, stopWatch.Elapsed);
}
}
}
utils.ClearInformation();
//Print end of reading
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Finished Reading, press enter to close.\n\n");
utils.WriteInformation(100, itemsRead, stopWatch.Elapsed);
Console.ReadLine();
}
protected override async Task <IEnumerable <IDocument> > ExecuteInputAsync(IDocument input, IExecutionContext context)
{
// Don't get data if we're just validating
if (context.Settings.GetBool(SiteKeys.Validate))
{
return(null);
}
string feed = input.GetString(SiteKeys.Feed);
if (!string.IsNullOrEmpty(feed))
{
try
{
// Download the feed
context.LogInformation($"Getting feed for {feed}");
Uri website = null;
string title = null;
string author = null;
string description = null;
string image = null;
List <ISyndicationItem> items = new List <ISyndicationItem>();
using (HttpClient httpClient = context.CreateHttpClient())
{
httpClient.DefaultRequestHeaders.Add("User-Agent", nameof(Statiq));
using (Stream stream = await httpClient.GetStreamAsync(feed))
{
using (StreamReader streamReader = new XmlStreamReader(stream))
{
using (XmlReader xmlReader = XmlReader.Create(streamReader, new XmlReaderSettings {
Async = true, DtdProcessing = DtdProcessing.Ignore
}))
{
xmlReader.MoveToContent();
bool atom = xmlReader.Name == "feed";
context.LogInformation($"Reading {feed} as " + (atom ? "Atom" : "RSS"));
XmlFeedReader feedReader = atom
? (XmlFeedReader) new AtomFeedReader(xmlReader, new DiscoverAtomParser())
: new RssFeedReader(xmlReader);
while (await feedReader.Read())
{
try
{
switch (feedReader.ElementType)
{
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
if (person.RelationshipType == "author")
{
author = person.Name ?? person.Email;
}
break;
case SyndicationElementType.Image:
ISyndicationImage img = await feedReader.ReadImage();
image = img.Url.ToString();
break;
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
website = link.Uri;
break;
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
items.Add(item);
break;
case SyndicationElementType.None:
break;
default:
ISyndicationContent content = await feedReader.ReadContent();
if (string.Equals(content.Name, "title", StringComparison.OrdinalIgnoreCase))
{
title = content.Value;
}
else if (string.Equals(content.Name, "description", StringComparison.OrdinalIgnoreCase) ||
string.Equals(content.Name, "subtitle", StringComparison.OrdinalIgnoreCase))
{
description = content.Value;
}
break;
}
}
catch (Exception ex)
{
context.LogWarning($"Exception while processing {feedReader.ElementType} in {feed}: {ex.Message}");
}
}
}
}
}
}
// Get a new document with feed metadata
MetadataItems metadata = new MetadataItems();
if (items.Count > 0)
{
FeedItem[] feedItems = items
.Select(x => new FeedItem(x, _recent, website, _truncateDescription, context))
.OrderByDescending(x => x.Published)
.Take(50) // Only take the 50 most recent items
.ToArray();
metadata.Add(SiteKeys.FeedItems, feedItems);
metadata.Add(SiteKeys.LastPublished, feedItems.First().Published);
metadata.Add(SiteKeys.NewestFeedItem, feedItems[0]);
}
if (!input.ContainsKey(SiteKeys.Website) && website != null)
{
metadata.Add(SiteKeys.Website, website.ToString());
}
if (!input.ContainsKey(SiteKeys.Title))
{
if (!string.IsNullOrEmpty(title))
{
metadata.Add(SiteKeys.Title, title);
}
else
{
string generatedTitle = GenerateTitleFromFeedName(feed);
metadata.Add(SiteKeys.Title, generatedTitle);
}
}
if (!input.ContainsKey(SiteKeys.Author) && !string.IsNullOrEmpty(author))
{
metadata.Add(SiteKeys.Author, author);
}
if (!input.ContainsKey(SiteKeys.Description) && !string.IsNullOrEmpty(description))
{
metadata.Add(SiteKeys.Description, description);
}
if (!input.ContainsKey(SiteKeys.Image) && !string.IsNullOrEmpty(image))
{
metadata.Add(SiteKeys.Image, image);
}
return(input.Clone(metadata).Yield());
}
catch (Exception ex)
{
context.LogWarning($"Error getting feed for {feed}: {ex.Message}");
}
}
// Return null so this feed is not included
return(null);
}
async static Task <List <Tuple <string, string, DateTimeOffset> > > Latest5PostsFromRssAsync(string filePath)
{
List <Tuple <string, string, DateTimeOffset> > result = new List <Tuple <string, string, DateTimeOffset> >();
using (var xmlReader = XmlReader.Create(filePath, new XmlReaderSettings()
{
Async = true
}))
{
var feedReader = new RssFeedReader(xmlReader);
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await feedReader.ReadCategory();
break;
// Read Image
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
break;
// Read Item
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
result.Add(new Tuple <string, string, DateTimeOffset>(item.Title, item.Links.First().Uri.ToString(), item.Published));
break;
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
break;
// Read Person
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
break;
// Read content
default:
ISyndicationContent content = await feedReader.ReadContent();
break;
}
if (result.Count == 5)
{
break;
}
}
}
return(result);
}
protected override async Task <IEnumerable <IDocument> > ExecuteInputAsync(IDocument input, IExecutionContext context)
{
string feed = input.GetString("Feed");
if (!string.IsNullOrEmpty(feed))
{
try
{
// Download the feed
context.LogInformation($"Getting feed for {feed}");
Uri website = null;
string title = null;
string author = null;
string description = null;
string image = null;
List <ISyndicationItem> items = new List <ISyndicationItem>();
using (HttpClient httpClient = context.CreateHttpClient())
{
httpClient.DefaultRequestHeaders.Add("User-Agent", "Wyam");
var response = await httpClient.GetAsync(feed);
if (response.StatusCode == HttpStatusCode.Redirect ||
response.StatusCode == HttpStatusCode.MovedPermanently)
{
context.LogWarning($"Attempting to follow redirect for {feed}");
feed = response.Headers.Location.OriginalString;
response = await httpClient.GetAsync(feed);
}
if (response.IsSuccessStatusCode)
{
using (Stream stream = await response.Content.ReadAsStreamAsync())
using (StreamReader streamReader = new XmlStreamReader(stream))
using (XmlReader xmlReader = XmlReader.Create(streamReader, new XmlReaderSettings {
Async = true, DtdProcessing = DtdProcessing.Ignore
}))
{
xmlReader.MoveToContent();
bool atom = xmlReader.Name == "feed";
context.LogInformation($"Reading {feed} as " + (atom ? "Atom" : "RSS"));
XmlFeedReader feedReader = atom
? (XmlFeedReader) new AtomFeedReader(xmlReader, new FixedAtomParser())
: new RssFeedReader(xmlReader);
while (await feedReader.Read())
{
try
{
switch (feedReader.ElementType)
{
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
if (person.RelationshipType == "author")
{
author = person.Name ?? person.Email;
}
break;
case SyndicationElementType.Image:
ISyndicationImage img = await feedReader.ReadImage();
image = img.Url.ToString();
break;
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
website = link.Uri;
break;
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
items.Add(item);
break;
case SyndicationElementType.None:
break;
default:
ISyndicationContent content = await feedReader.ReadContent();
if (string.Equals(content.Name, "title", StringComparison.OrdinalIgnoreCase))
{
title = content.Value;
}
else if (string.Equals(content.Name, "description", StringComparison.OrdinalIgnoreCase) ||
string.Equals(content.Name, "subtitle", StringComparison.OrdinalIgnoreCase))
{
description = content.Value;
}
break;
}
}
catch (Exception ex)
{
context.LogWarning($"Exception while processing {feedReader.ElementType} in {feed}: {ex.Message}");
}
}
}
}
// Get a new document with feed metadata
MetadataItems metadata = new MetadataItems();
if (items.Count > 0)
{
BlogFeedItem[] feedItems = items
.Select(x => new BlogFeedItem(x, _recent, website))
.OrderByDescending(x => x.Published)
.Take(50) // Only take the 50 most recent items
.ToArray();
metadata.Add(SiteKeys.BlogFeedItems, feedItems);
return(input.Clone(metadata).Yield());
}
}
}
catch (Exception ex)
{
context.LogWarning($"Error getting feed for {feed}: {ex.Message}");
}
}
else
{
var speaker = input.GetString("Title");
if (!string.IsNullOrWhiteSpace(speaker))
{
context.LogWarning($"No RSS specified for {speaker}.");
}
else
{
context.LogWarning($"No RSS specified for unknown speaker.");
}
}
return(input.Yield());
}
public virtual Task Write(ISyndicationImage image)
{
return(WriteRaw(Formatter.Format(image ?? throw new ArgumentNullException(nameof(image)))));
}
