public FeedItem(ISyndicationItem item, DateTimeOffset recent)
{
Title = item.Title;
Link = item.Links.FirstOrDefault(x => x.RelationshipType == RssLinkTypes.Alternate)?.Uri.ToString() ?? item.Id;
Published = item.Published != default ? item.Published : item.LastUpdated;
Recent = Published > recent;
Description = item.Description;
Links = item.Links
.Where(x => !string.IsNullOrEmpty(x.MediaType))
.GroupBy(x => x.MediaType)
.Select(x => x.First())
.ToDictionary(x => x.MediaType, x => x.Uri.ToString());
ISyndicationPerson person = item.Contributors.FirstOrDefault(x => x.RelationshipType == "author");
if (person != null)
{
Author = person.Name ?? person.Email;
}
AtomEntry atom = item as AtomEntry;
if (atom != null && !string.IsNullOrEmpty(atom.Summary))
{
Description = atom.Summary;
}
}
public async Task ReadPerson()
{
using (XmlReader xmlReader = XmlReader.Create(@"..\..\..\TestFeeds\simpleAtomFeed.xml", new XmlReaderSettings {
Async = true
}))
{
var persons = new List <ISyndicationPerson>();
var reader = new AtomFeedReader(xmlReader);
while (await reader.Read())
{
if (reader.ElementType == SyndicationElementType.Person)
{
ISyndicationPerson person = await reader.ReadPerson();
persons.Add(person);
}
}
Assert.True(persons.Count() == 2);
Assert.True(persons[0].Name == "Mark Pilgrim");
Assert.True(persons[0].Email == "*****@*****.**");
Assert.True(persons[0].Uri == "http://example.org/");
Assert.True(persons[1].Name == "Sam Ruby");
}
}
private async Task <ISyndicationPerson> ReadPerson(Rss20FeedReader reader)
{
ISyndicationPerson person = null;
person = await reader.ReadPerson();
return(person);
}
// Read an RssFeed
public static async Task CreateRssFeedReaderExample(string filePath)
{
// Create an XmlReader
// Example: ..\tests\TestFeeds\rss20-2items.xml
using (var xmlReader = XmlReader.Create(filePath, new XmlReaderSettings()
{
Async = true
}))
{
// Instantiate an Rss20FeedReader using the XmlReader.
// This will assign as default an Rss20FeedParser as the parser.
var feedReader = new RssFeedReader(xmlReader);
//
// Read the feed
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await feedReader.ReadCategory();
break;
// Read Image
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
break;
// Read Item
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
break;
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
break;
// Read Person
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
break;
// Read content
default:
ISyndicationContent content = await feedReader.ReadContent();
break;
}
}
}
}
public static async Task TestReadFeedElements(XmlReader outerXmlReader)
{
using (var xmlReader = outerXmlReader)
{
var reader = new RssFeedReader(xmlReader);
int items = 0;
while (await reader.Read())
{
switch (reader.ElementType)
{
case SyndicationElementType.Person:
ISyndicationPerson person = await reader.ReadPerson();
Assert.True(person.Email == "John Smith");
break;
case SyndicationElementType.Link:
ISyndicationLink link = await reader.ReadLink();
Assert.True(link.Length == 123);
Assert.True(link.MediaType == "testType");
Assert.True(link.Uri.OriginalString == "http://example.com/");
break;
case SyndicationElementType.Image:
ISyndicationImage image = await reader.ReadImage();
Assert.True(image.Title == "Microsoft News");
Assert.True(image.Description == "Test description");
Assert.True(image.Url.OriginalString == "http://2.bp.blogspot.com/-NA5Jb-64eUg/URx8CSdcj_I/AAAAAAAAAUo/eCx0irI0rq0/s1600/bg_Microsoft_logo3-20120824073001907469-620x349.jpg");
break;
case SyndicationElementType.Item:
items++;
ISyndicationItem item = await reader.ReadItem();
if (items == 1)
{
Assert.True(item.Title == "Lorem ipsum 2017-07-06T20:25:00+00:00");
Assert.True(item.Description == "Exercitation sit dolore mollit et est eiusmod veniam aute officia veniam ipsum.");
Assert.True(item.Links.Count() == 3);
}
else if (items == 2)
{
Assert.True(item.Title == "Lorem ipsum 2017-07-06T20:24:00+00:00");
Assert.True(item.Description == "Do ipsum dolore veniam minim est cillum aliqua ea.");
Assert.True(item.Links.Count() == 3);
}
break;
default:
break;
}
}
}
}
private async Task <bool> retrievePosts(Feed feed)
{
using (var xmlReader = XmlReader.Create("https://azurecomcdn.azureedge.net/en-us/blog/feed/", new XmlReaderSettings()
{
Async = true
}))
{
// Instantiate an Rss20FeedReader using the XmlReader.
// This will assign as default an Rss20FeedParser as the parser.
var feedReader = new RssFeedReader(xmlReader);
//
// Read the feed
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await feedReader.ReadCategory();
break;
// Read Image
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
break;
// Read Item
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
break;
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
break;
// Read Person
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
break;
// Read content
default:
ISyndicationContent content = await feedReader.ReadContent();
break;
}
}
}
return(true);
}
private static async Task <List <object> > ParseExport(XmlReader xmlReader)
{
List <object> result = new List <object>();
{
var feedReader = new AtomFeedReader(xmlReader);
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await feedReader.ReadCategory();
result.Add(category);
break;
// Read Image
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
result.Add(image);
break;
// Read Item
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
result.Add(item);
break;
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
result.Add(link);
break;
// Read Person
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
result.Add(person);
break;
// Read content
default:
ISyndicationContent content = await feedReader.ReadContent();
result.Add(content);
break;
}
}
return(result);
}
}
public FeedItem(ISyndicationItem item, DateTimeOffset recent, Uri website, bool truncateDescription, IExecutionContext context)
{
Title = item.Title;
ISyndicationLink firstLink = item.Links.FirstOrDefault(x => x.RelationshipType == RssLinkTypes.Alternate);
if (firstLink != null)
{
Link = firstLink.Uri.IsAbsoluteUri ? firstLink.Uri.AbsoluteUri : new Uri(website, firstLink.Uri).AbsoluteUri;
}
else
{
Link = item.Id;
}
Published = item.Published != default ? item.Published : item.LastUpdated;
Recent = Published > recent;
Links = item.Links
.Where(x => !string.IsNullOrEmpty(x.MediaType))
.GroupBy(x => x.MediaType)
.Select(x => x.First())
.ToDictionary(x => x.MediaType, x => x.Uri.ToString());
ISyndicationPerson person = item.Contributors.FirstOrDefault(x => x.RelationshipType == "author");
if (person != null)
{
Author = person.Name ?? person.Email;
}
Description = item.Description;
AtomEntry atom = item as AtomEntry;
if (atom != null && !string.IsNullOrEmpty(atom.Summary))
{
Description = atom.Summary;
}
if (truncateDescription)
{
try
{
// Use the first <p> if one is found
HtmlParser htmlParser = new HtmlParser();
IHtmlDocument htmlDocument = htmlParser.ParseDocument(Description);
IElement element = htmlDocument.QuerySelector("p");
if (element is object && !string.IsNullOrWhiteSpace(element.OuterHtml))
{
Description = element.OuterHtml;
}
}
catch (Exception ex)
{
context.LogWarning($"Error parsing HTML description for feed {Title}: {ex.Message}");
}
}
}
public RssContent()
{
using (var xmlReader = XmlReader.Create("C:/Users/Developer/Desktop.brisbane-city-council.rss", new XmlReaderSettings()
{
Async = true
}))
{
var feedReader = new RssFeedReader(xmlReader);
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await feedReader.ReadCategory();
break;
// Read Image
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
break;
// Read Item
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
break;
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
break;
// Read Person
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
break;
// Read content
default:
ISyndicationContent content = await feedReader.ReadContent();
break;
}
}
}
}
public void AddContributor(ISyndicationPerson person)
{
if (person == null)
{
throw new ArgumentNullException(nameof(person));
}
if (_contributors.IsReadOnly)
{
_contributors = _contributors.ToList();
}
_contributors.Add(person);
}
public virtual ISyndicationContent CreateContent(ISyndicationPerson person)
{
if (person == null)
{
throw new ArgumentNullException(nameof(person));
}
//
// RSS requires Email
if (string.IsNullOrEmpty(person.Email))
{
throw new ArgumentNullException("Invalid person Email");
}
return(new SyndicationContent(person.RelationshipType ?? RssElementNames.Author, person.Email));
}
private void ConvertUser(IList <User> users, UserMapping[] mappings, ISyndicationPerson user)
{
var map = mappings.Single(candidate => string.Equals(candidate.FromGooglePlusUrl, user.Uri));
var result = new User
{
Id = map?.Id ?? Guid.NewGuid().ToString(),
Name = map?.Name ?? user.Name,
Email = map.ToEmail,
Slug = map.Slug
};
result.CreatedBy = result.Id;
result.UpdatedBy = result.Id;
_userLookup.Add(map.FromGooglePlusUrl, result.Id);
users.Add(result);
}
public virtual ISyndicationContent CreateContent(ISyndicationPerson person)
{
if (person == null)
{
throw new ArgumentNullException(nameof(person));
}
if (string.IsNullOrEmpty(person.Name))
{
throw new ArgumentNullException("Name");
}
string contributorType = person.RelationshipType ?? AtomContributorTypes.Author;
if (contributorType != AtomContributorTypes.Author &&
contributorType != AtomContributorTypes.Contributor)
{
throw new ArgumentException("RelationshipType");
}
var result = new SyndicationContent(contributorType);
//
// name
result.AddField(new SyndicationContent(AtomElementNames.Name, person.Name));
//
// email
if (!string.IsNullOrEmpty(person.Email))
{
result.AddField(new SyndicationContent(AtomElementNames.Email, person.Email));
}
//
// uri
if (person.Uri != null)
{
result.AddField(new SyndicationContent(AtomElementNames.Uri, FormatValue(person.Uri)));
}
return(result);
}
private async Task ReadWhile()
{
using (var xmlReader = XmlReader.Create(@"..\..\..\TestFeeds\rss20.xml", new XmlReaderSettings()
{
Async = true
}))
{
var reader = new RssFeedReader(xmlReader);
while (await reader.Read())
{
switch (reader.ElementType)
{
case SyndicationElementType.Link:
ISyndicationLink link = await reader.ReadLink();
break;
case SyndicationElementType.Item:
ISyndicationItem item = await reader.ReadItem();
break;
case SyndicationElementType.Person:
ISyndicationPerson person = await reader.ReadPerson();
break;
case SyndicationElementType.Image:
ISyndicationImage image = await reader.ReadImage();
break;
default:
ISyndicationContent content = await reader.ReadContent();
break;
}
}
}
}
public virtual ISyndicationContent CreateContent(ISyndicationPerson person)
{
if (person == null)
{
throw new ArgumentNullException(nameof(person));
}
//
// RSS requires Email
if (string.IsNullOrEmpty(person.Email))
{
throw new ArgumentNullException("Invalid person Email");
}
//
// Real name recommended with RSS e-mail addresses
// Ex: <author>[email protected] (John Doe)</author>
string value = string.IsNullOrEmpty(person.Name) ? person.Email : $"{person.Email} ({person.Name})";
return(new SyndicationContent(person.RelationshipType ?? RssElementNames.Author, value));
}
public async Task <Podcast> ParseRssFeed(Podcast podcastData, bool isUpdate = false)
{
string title = string.Empty,
imageUrl = string.Empty,
description = string.Empty,
author = string.Empty;
DateTime pubDate = DateTime.MinValue,
lastUpdateDate = DateTime.MinValue;
List <ISyndicationItem> items = new List <ISyndicationItem>();
IEnumerable <PodcastItem> podcastItems = Enumerable.Empty <PodcastItem>();
Podcast podcast = null;
using (var xmlReader = XmlReader.Create(podcastData.Url, new XmlReaderSettings {
Async = true
}))
{
var feedReader = new RssFeedReader(xmlReader);
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
case SyndicationElementType.Category:
ISyndicationCategory category = await feedReader.ReadCategory();
break;
case SyndicationElementType.Content:
ISyndicationContent content = await feedReader.ReadContent();
if (content.Name == "title")
{
title = content.Value;
}
if (content.Name == "description")
{
description = content.Value;
}
if (content.Name == "author")
{
author = content.Value;
}
break;
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
imageUrl = image.Url?.AbsoluteUri;
break;
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
items.Add(item);
break;
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
break;
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
break;
case SyndicationElementType.None:
default:
break;
}
}
pubDate = items.Max(item => item.Published.DateTime);
if (isUpdate)
{
lastUpdateDate = podcastData.LastUpdateDate;
podcastData.Author = author;
podcastData.Title = title;
podcastData.ImageUrl = imageUrl;
podcastData.Description = description;
podcastData.LastUpdateDate = pubDate;
podcast = podcastData;
await dataService.Update <Podcast>(podcast);
}
else
{
podcast = new Podcast(title, podcastData.Url, imageUrl, description, author)
{
LastUpdateDate = pubDate == DateTime.MinValue ? DateTime.Now : pubDate
};
await dataService.Insert <Podcast>(podcast);
}
podcastItems = items.Select(item => new
{
item.Title,
item.Description,
Enclosure = item.Links.FirstOrDefault(linkItem => linkItem.RelationshipType == "enclosure"),
PublishDate = item.Published.DateTime
}).Select(data => new PodcastItem(data.Title, data.Description, data.Enclosure.Uri.OriginalString,
data.Enclosure.Length, data.PublishDate, podcast.Id))
.Where(item => item.PublishDate > lastUpdateDate)
.ToList();
await dataService.Insert(podcastItems);
podcast.PodcastItems = podcastItems.ToList();
await dataService.Update(podcast);
}
return(podcast);
}
void ComparePerson(ISyndicationPerson person1, ISyndicationPerson person2)
{
Assert.True(person1.Email == person2.Email);
Assert.True(person1.RelationshipType == person2.RelationshipType);
}
public string Format(ISyndicationPerson person)
{
return(Format(CreateContent(person)));
}
public async Task <Channel9RssResult> Parse(Uri rssUri)
{
var result = new Channel9RssResult();
result.SourceUrl = rssUri;
try
{
using (var client = new HttpClient())
{
result.RawXml = await client.GetStringAsync(rssUri);
using (var xmlReader = XmlReader.Create(new StringReader(result.RawXml), new XmlReaderSettings()
{
Async = false
}))
{
var feedReader = new RssFeedReader(xmlReader);
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await feedReader.ReadCategory();
break;
// Read Image
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
break;
// Read Item
case SyndicationElementType.Item:
// parse the syndication item
ISyndicationItem item = await feedReader.ReadItem();
result.SyndicationItems.Add(item);
// then construct a session info
var si = new SessionInfo();
si.SessionID = item.Id.Substring(item.Id.LastIndexOf("/", StringComparison.InvariantCultureIgnoreCase) + 1);
si.Title = item.Title;
si.SessionSite = new Uri(item.Id);
si.PublishDate = item.Published.DateTime;
si.Presenter = item.Contributors.FirstOrDefault()?.Name;
result.Sessions.Add(si);
foreach (var v in item.Links)
{
if (!string.IsNullOrWhiteSpace(v.MediaType))
{
si.VideoRecordings.Add(new VideoRecording()
{
SessionInfo = si, Url = v.Uri, MediaType = v.MediaType, Length = v.Length
});
}
}
break;
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
break;
// Read Person
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
break;
// Read content
default:
ISyndicationContent content = await feedReader.ReadContent();
break;
}
}
}
}
}
catch (Exception ex)
{
result.Exceptions = new List <Exception>();
result.Exceptions.Add(ex);
}
return(result);
}
public static async Task ReadFeed()
{
using (XmlReader xmlReader = XmlReader.Create(@"..\DemoTest1\TestFeeds\rss20.xml", new XmlReaderSettings()
{
Async = true
}))
using (XmlWriter xmlWriter = XmlWriter.Create(Console.Out, new XmlWriterSettings()
{
Indent = true, Async = true
}))
{
Rss20FeedReader reader = new Rss20FeedReader(xmlReader);
Rss20FeedWriter writer = new Rss20FeedWriter(xmlWriter);
while (await reader.Read())
{
switch (reader.ElementType)
{
case SyndicationElementType.Category:
ISyndicationCategory category = await reader.ReadCategory();
await writer.Write(category);
break;
case SyndicationElementType.Image:
ISyndicationImage image = await reader.ReadImage();
await writer.Write(image);
break;
case SyndicationElementType.Item:
ISyndicationItem item = await reader.ReadItem();
await writer.Write(item);
break;
case SyndicationElementType.Link:
ISyndicationLink link = await reader.ReadLink();
await writer.Write(link);
break;
case SyndicationElementType.Person:
ISyndicationPerson person = await reader.ReadPerson();
await writer.Write(person);
break;
default:
ISyndicationContent content = await reader.ReadContent();
await writer.Write(content);
break;
}
xmlWriter.Flush();
// Simulate a slow stream.
Thread.Sleep(300);
}
}
}
public string Format(ISyndicationPerson person)
{
ISyndicationContent content = CreateContent(person);
return(Format(content));
}
internal void DisplayPerson(ISyndicationPerson person)
{
Console.WriteLine("--- Person Read ---");
Console.WriteLine("Email: " + person.Email);
Console.WriteLine();
}
private async Task <List <BlogPost> > GetFeed()
{
var xmlReader = XmlReader.Create(_settings.Value.FeedUrl);
var reader = new AtomFeedReader(xmlReader);
var posts = new List <BlogPost>();
//
// Read the feed
while (await reader.Read())
{
//
// Check the type of the current element.
switch (reader.ElementType)
{
//
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await reader.ReadCategory();
break;
//
// Read image
case SyndicationElementType.Image:
ISyndicationImage image = await reader.ReadImage();
break;
//
// Read entry
case SyndicationElementType.Item:
IAtomEntry entry = await reader.ReadEntry();
// these are the only ones we need for now
posts.Add(new BlogPost
{
Title = entry.Title,
PublishedDate = entry.Published.DateTime,
Categories = entry.Categories?.Select(c => c.Name)?.ToList()
});
break;
//
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await reader.ReadLink();
break;
//
// Read person
case SyndicationElementType.Person:
ISyndicationPerson person = await reader.ReadPerson();
break;
//
// Read content
default:
ISyndicationContent content = await reader.ReadContent();
break;
}
}
return(posts);
}
public virtual Task Write(ISyndicationPerson person)
{
return(WriteRaw(Formatter.Format(person ?? throw new ArgumentNullException(nameof(person)))));
}
async static Task <List <Tuple <string, string, DateTimeOffset> > > Latest5PostsFromRssAsync(string filePath)
{
List <Tuple <string, string, DateTimeOffset> > result = new List <Tuple <string, string, DateTimeOffset> >();
using (var xmlReader = XmlReader.Create(filePath, new XmlReaderSettings()
{
Async = true
}))
{
var feedReader = new RssFeedReader(xmlReader);
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await feedReader.ReadCategory();
break;
// Read Image
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
break;
// Read Item
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
result.Add(new Tuple <string, string, DateTimeOffset>(item.Title, item.Links.First().Uri.ToString(), item.Published));
break;
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
break;
// Read Person
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
break;
// Read content
default:
ISyndicationContent content = await feedReader.ReadContent();
break;
}
if (result.Count == 5)
{
break;
}
}
}
return(result);
}
public async Task ConsumeFeed(string filePath)
{
bool verbose = false;
// Information to display.
double size = new FileInfo(filePath).Length;
double currentSize = 0;
int itemsRead = 0;
// Transform the size of the file to Kb - Mb - Gb.
Tuple <double, string> sizeInfo = utils.ConvertBytesToSize(size);
// Display the Size of the feed and ask for verbose.
Console.ForegroundColor = ConsoleColor.DarkGreen;
Console.WriteLine("Size of the Feed: {0:N2} {1}", sizeInfo.Item1, sizeInfo.Item2);
Console.Write("Verbose Items (Y/N): ");
Console.ForegroundColor = ConsoleColor.White;
string input = Console.ReadLine();
verbose = utils.ValidateVerbose(input);
Console.CursorVisible = false;
Stopwatch stopWatch = null;
using (var xmlReader = XmlReader.Create(filePath, new XmlReaderSettings()
{
Async = true
}))
{
var reader = new Rss20FeedReader(xmlReader);
stopWatch = new Stopwatch();
stopWatch.Start();
ElementDisplayer displayer = new ElementDisplayer();
while (await reader.Read())
{
if (verbose)
{
utils.ClearInformation();
}
switch (reader.ElementType)
{
case SyndicationElementType.Content:
ISyndicationContent content = await reader.ReadContent();
if (verbose)
{
displayer.DisplayContent(content);
}
break;
case SyndicationElementType.Item:
ISyndicationItem item = await ReadItem(reader);
itemsRead++;
if (verbose)
{
displayer.DisplayItem(item);
}
currentSize += _sizeOfItem;
break;
case SyndicationElementType.Person:
ISyndicationPerson person = await ReadPerson(reader);
if (verbose)
{
displayer.DisplayPerson(person);
}
break;
case SyndicationElementType.Image:
ISyndicationImage image = await ReadImage(reader);
if (verbose)
{
displayer.DisplayImage(image);
}
break;
case SyndicationElementType.Link:
ISyndicationLink link = await ReadLink(reader);
if (verbose)
{
displayer.DisplayLink(link);
}
break;
case SyndicationElementType.Category:
ISyndicationCategory category = await ReadCategory(reader);
if (verbose)
{
displayer.DisplayCategory(category);
}
break;
}
double percentage = ((currentSize * 100) / size);
if (itemsRead % 200 == 0)
{
utils.WriteInformation(percentage, itemsRead, stopWatch.Elapsed);
}
}
}
utils.ClearInformation();
//Print end of reading
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Finished Reading, press enter to close.\n\n");
utils.WriteInformation(100, itemsRead, stopWatch.Elapsed);
Console.ReadLine();
}
public static async Task ReadAtomFeed(string filePath)
{
//
// Create an XmlReader from file
// Example: ..\tests\TestFeeds\simpleAtomFeed.xml
using (XmlReader xmlReader = XmlReader.Create(filePath, new XmlReaderSettings()
{
Async = true
}))
{
//
// Create an AtomFeedReader
var reader = new AtomFeedReader(xmlReader);
//
// Read the feed
while (await reader.Read())
{
//
// Check the type of the current element.
switch (reader.ElementType)
{
//
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await reader.ReadCategory();
break;
//
// Read image
case SyndicationElementType.Image:
ISyndicationImage image = await reader.ReadImage();
break;
//
// Read entry
case SyndicationElementType.Item:
IAtomEntry entry = await reader.ReadEntry();
break;
//
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await reader.ReadLink();
break;
//
// Read person
case SyndicationElementType.Person:
ISyndicationPerson person = await reader.ReadPerson();
break;
//
// Read content
default:
ISyndicationContent content = await reader.ReadContent();
break;
}
}
}
}
protected override async Task <IEnumerable <IDocument> > ExecuteInputAsync(IDocument input, IExecutionContext context)
{
string feed = input.GetString("Feed");
if (!string.IsNullOrEmpty(feed))
{
try
{
// Download the feed
context.LogInformation($"Getting feed for {feed}");
Uri website = null;
string title = null;
string author = null;
string description = null;
string image = null;
List <ISyndicationItem> items = new List <ISyndicationItem>();
using (HttpClient httpClient = context.CreateHttpClient())
{
httpClient.DefaultRequestHeaders.Add("User-Agent", "Wyam");
var response = await httpClient.GetAsync(feed);
if (response.StatusCode == HttpStatusCode.Redirect ||
response.StatusCode == HttpStatusCode.MovedPermanently)
{
context.LogWarning($"Attempting to follow redirect for {feed}");
feed = response.Headers.Location.OriginalString;
response = await httpClient.GetAsync(feed);
}
if (response.IsSuccessStatusCode)
{
using (Stream stream = await response.Content.ReadAsStreamAsync())
using (StreamReader streamReader = new XmlStreamReader(stream))
using (XmlReader xmlReader = XmlReader.Create(streamReader, new XmlReaderSettings {
Async = true, DtdProcessing = DtdProcessing.Ignore
}))
{
xmlReader.MoveToContent();
bool atom = xmlReader.Name == "feed";
context.LogInformation($"Reading {feed} as " + (atom ? "Atom" : "RSS"));
XmlFeedReader feedReader = atom
? (XmlFeedReader) new AtomFeedReader(xmlReader, new FixedAtomParser())
: new RssFeedReader(xmlReader);
while (await feedReader.Read())
{
try
{
switch (feedReader.ElementType)
{
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
if (person.RelationshipType == "author")
{
author = person.Name ?? person.Email;
}
break;
case SyndicationElementType.Image:
ISyndicationImage img = await feedReader.ReadImage();
image = img.Url.ToString();
break;
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
website = link.Uri;
break;
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
items.Add(item);
break;
case SyndicationElementType.None:
break;
default:
ISyndicationContent content = await feedReader.ReadContent();
if (string.Equals(content.Name, "title", StringComparison.OrdinalIgnoreCase))
{
title = content.Value;
}
else if (string.Equals(content.Name, "description", StringComparison.OrdinalIgnoreCase) ||
string.Equals(content.Name, "subtitle", StringComparison.OrdinalIgnoreCase))
{
description = content.Value;
}
break;
}
}
catch (Exception ex)
{
context.LogWarning($"Exception while processing {feedReader.ElementType} in {feed}: {ex.Message}");
}
}
}
}
// Get a new document with feed metadata
MetadataItems metadata = new MetadataItems();
if (items.Count > 0)
{
BlogFeedItem[] feedItems = items
.Select(x => new BlogFeedItem(x, _recent, website))
.OrderByDescending(x => x.Published)
.Take(50) // Only take the 50 most recent items
.ToArray();
metadata.Add(SiteKeys.BlogFeedItems, feedItems);
return(input.Clone(metadata).Yield());
}
}
}
catch (Exception ex)
{
context.LogWarning($"Error getting feed for {feed}: {ex.Message}");
}
}
else
{
var speaker = input.GetString("Title");
if (!string.IsNullOrWhiteSpace(speaker))
{
context.LogWarning($"No RSS specified for {speaker}.");
}
else
{
context.LogWarning($"No RSS specified for unknown speaker.");
}
}
return(input.Yield());
}
protected override async Task <IEnumerable <IDocument> > ExecuteInputAsync(IDocument input, IExecutionContext context)
{
// Don't get data if we're just validating
if (context.Settings.GetBool(SiteKeys.Validate))
{
return(null);
}
string feed = input.GetString(SiteKeys.Feed);
if (!string.IsNullOrEmpty(feed))
{
try
{
// Download the feed
context.LogInformation($"Getting feed for {feed}");
Uri website = null;
string title = null;
string author = null;
string description = null;
string image = null;
List <ISyndicationItem> items = new List <ISyndicationItem>();
using (HttpClient httpClient = context.CreateHttpClient())
{
httpClient.DefaultRequestHeaders.Add("User-Agent", nameof(Statiq));
using (Stream stream = await httpClient.GetStreamAsync(feed))
{
using (StreamReader streamReader = new XmlStreamReader(stream))
{
using (XmlReader xmlReader = XmlReader.Create(streamReader, new XmlReaderSettings {
Async = true, DtdProcessing = DtdProcessing.Ignore
}))
{
xmlReader.MoveToContent();
bool atom = xmlReader.Name == "feed";
context.LogInformation($"Reading {feed} as " + (atom ? "Atom" : "RSS"));
XmlFeedReader feedReader = atom
? (XmlFeedReader) new AtomFeedReader(xmlReader, new DiscoverAtomParser())
: new RssFeedReader(xmlReader);
while (await feedReader.Read())
{
try
{
switch (feedReader.ElementType)
{
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
if (person.RelationshipType == "author")
{
author = person.Name ?? person.Email;
}
break;
case SyndicationElementType.Image:
ISyndicationImage img = await feedReader.ReadImage();
image = img.Url.ToString();
break;
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
website = link.Uri;
break;
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
items.Add(item);
break;
case SyndicationElementType.None:
break;
default:
ISyndicationContent content = await feedReader.ReadContent();
if (string.Equals(content.Name, "title", StringComparison.OrdinalIgnoreCase))
{
title = content.Value;
}
else if (string.Equals(content.Name, "description", StringComparison.OrdinalIgnoreCase) ||
string.Equals(content.Name, "subtitle", StringComparison.OrdinalIgnoreCase))
{
description = content.Value;
}
break;
}
}
catch (Exception ex)
{
context.LogWarning($"Exception while processing {feedReader.ElementType} in {feed}: {ex.Message}");
}
}
}
}
}
}
// Get a new document with feed metadata
MetadataItems metadata = new MetadataItems();
if (items.Count > 0)
{
FeedItem[] feedItems = items
.Select(x => new FeedItem(x, _recent, website, _truncateDescription, context))
.OrderByDescending(x => x.Published)
.Take(50) // Only take the 50 most recent items
.ToArray();
metadata.Add(SiteKeys.FeedItems, feedItems);
metadata.Add(SiteKeys.LastPublished, feedItems.First().Published);
metadata.Add(SiteKeys.NewestFeedItem, feedItems[0]);
}
if (!input.ContainsKey(SiteKeys.Website) && website != null)
{
metadata.Add(SiteKeys.Website, website.ToString());
}
if (!input.ContainsKey(SiteKeys.Title))
{
if (!string.IsNullOrEmpty(title))
{
metadata.Add(SiteKeys.Title, title);
}
else
{
string generatedTitle = GenerateTitleFromFeedName(feed);
metadata.Add(SiteKeys.Title, generatedTitle);
}
}
if (!input.ContainsKey(SiteKeys.Author) && !string.IsNullOrEmpty(author))
{
metadata.Add(SiteKeys.Author, author);
}
if (!input.ContainsKey(SiteKeys.Description) && !string.IsNullOrEmpty(description))
{
metadata.Add(SiteKeys.Description, description);
}
if (!input.ContainsKey(SiteKeys.Image) && !string.IsNullOrEmpty(image))
{
metadata.Add(SiteKeys.Image, image);
}
return(input.Clone(metadata).Yield());
}
catch (Exception ex)
{
context.LogWarning($"Error getting feed for {feed}: {ex.Message}");
}
}
// Return null so this feed is not included
return(null);
}
