private Article mapArticle(ISyndicationContent item)
{
var a = new Article();
a.Id = item.Fields.FirstOrDefault(f => f.Name == "guid")?.Value;
a.Title = item.Fields.FirstOrDefault(f => f.Name == "title")?.Value;
a.Content = item.Fields.FirstOrDefault(f => f.Name == "encoded")?.Value;
string pubDate = item.Fields.FirstOrDefault(f => f.Name == "pubDate")?.Value;
if (DateTime.TryParse(pubDate, out DateTime date))
{
a.DatePublished = date;
}
a.Creator = item.Fields.FirstOrDefault(f => f.Name == "creator")?.Value;
a.Description = item.Fields.FirstOrDefault(f => f.Name == "description")?.Value;
Regex regex;
regex = new Regex("<a[^>]+>Read more</a>");
a.Description = regex.Replace(a.Description, "");
regex = new Regex(@"(<img[^>])src=""(https?://[^""]+)""", RegexOptions.Multiline);
foreach (Match match in regex.Matches(a.Content))
{
var url = $"/Home/Resource?ResourceId={match.Groups[2].Value}";
a.Content = a.Content.Replace(match.Value, $@"{match.Groups[1].Value}src=""{url}""");
a.ResourcesUrl.Add(url);
}
return(a);
}
public async Task WriteContent()
{
ISyndicationContent content = null;
//
// Read
using (var xmlReader = XmlReader.Create(@"..\..\..\TestFeeds\CustomXml.xml"))
{
RssFeedReader reader = new RssFeedReader(xmlReader);
content = await reader.ReadContent();
}
//
// Write
StringBuilder sb = new StringBuilder();
using (var xmlWriter = XmlWriter.Create(sb))
{
var writer = new RssFeedWriter(xmlWriter);
await writer.Write(content);
await writer.Flush();
}
string res = sb.ToString();
Assert.True(res == "<?xml version=\"1.0\" encoding=\"utf-16\"?><rss version=\"2.0\"><channel><NewItem><enclosure url=\"http://www.scripting.com/mp3s/weatherReportSuite.mp3\" length=\"12216320\" type=\"audio/mpeg\" /><title>Lorem ipsum 2017-07-06T20:25:00+00:00</title><description>Exercitation sit dolore mollit et est eiusmod veniam aute officia veniam ipsum.</description><link>http://example.com/test/1499372700</link><guid isPermaLink=\"true\">http://example.com/test/1499372700</guid><pubDate>Thu, 06 Jul 2017 20:25:00 GMT</pubDate></NewItem></channel></rss>");
}
public static async Task <List <ISyndicationContent> > RssFeedReader()
{
///Wired Sitesindeki Rss beslemesinden son haberlerin linklerini ve başlıklarını çekiyoruz
string feedUri = "https://www.wired.com/feed/rss";
List <ISyndicationContent> contentList = new List <ISyndicationContent>();
using (var xmlReader = XmlReader.Create(feedUri, new XmlReaderSettings()
{
Async = true
}))
{
var feedReader = new RssFeedReader(xmlReader);
while (await feedReader.Read())
{
ISyndicationContent content = await feedReader.ReadContent();
if (content.Name == "item")
{
contentList.Add(content);
}
}
}
return(contentList);
}
public async Task <FeedDto> RetrieveFeed(Uri uri)
{
string title = "";
var items = new List <ISyndicationItem>();
using (var client = new HttpClient())
using (var stream = await client.GetStreamAsync(uri).ConfigureAwait(false))
using (var xmlReader = XmlReader.Create(stream, new XmlReaderSettings {
Async = true
}))
{
var feedReader = GetReader(xmlReader);
while (await feedReader.Read().ConfigureAwait(false))
{
switch (feedReader.ElementType)
{
case SyndicationElementType.Item:
items.Add(await feedReader.ReadItem().ConfigureAwait(false));
break;
case SyndicationElementType.Content:
ISyndicationContent content = await feedReader.ReadContent().ConfigureAwait(false);
if (content.Name == "title")
{
title = content.Value;
}
break;
}
}
}
return(new FeedDto(title, items.Select(x => new FeedItemDto(x.Links.FirstOrDefault()?.Uri, x.Published,
RemoveHtmlTags(x.Title), RemoveHtmlTags(x.Description))).ToArray()));
}
public static void WriteSyndicationContent(this XmlWriter writer, ISyndicationContent content, string defaultNs)
{
string ns = content.Namespace ?? defaultNs;
//
// Write opening
if (ns != null)
{
XmlUtils.SplitName(content.Name, out string prefix, out string localName);
prefix = writer.LookupPrefix(ns) ?? prefix;
if (prefix != null)
{
writer.WriteStartElement(prefix, localName, ns);
}
else
{
writer.WriteStartElement(localName, ns);
}
}
else
{
writer.WriteStartElement(content.Name);
}
//
// Write attributes
if (content.Attributes != null)
{
foreach (var a in content.Attributes)
{
writer.WriteSyndicationAttribute(a);
}
}
//
// Write value
if (content.Value != null)
{
writer.WriteString(content.Value);
}
//
// Write Fields
else
{
if (content.Fields != null)
{
foreach (var field in content.Fields)
{
writer.WriteSyndicationContent(field, defaultNs);
}
}
}
//
// Write closing
writer.WriteEndElement();
}
public static async Task <List <ComplexSyndication> > Get(params Uri[] url)
{
var parser = new RssParser();
var toBeProcessed = new List <Task <HttpResponseMessage> >();
var httpClient = HttpClientFactory.Get();
foreach (var u in url)
{
var t = httpClient.GetAsync(u);
toBeProcessed.Add(t);
}
Task.WaitAll(toBeProcessed.ToArray());
var syndications = new List <ComplexSyndication>();
foreach (var result in toBeProcessed)
{
var res = result.Result;
var resultContent = await res.Content.ReadAsStringAsync();
using (var xmlReader = XmlReader.Create(new MemoryStream(Encoding.UTF8.GetBytes(resultContent))))
{
var feedReader = new RssFeedReader(xmlReader);
var syndication = new ComplexSyndication();
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
case SyndicationElementType.Item:
//ISyndicationContent is a raw representation of the feed
ISyndicationContent content = await feedReader.ReadContent();
ISyndicationItem item = parser.CreateItem(content);
ISyndicationContent outline = content.Fields.FirstOrDefault(f => f.Name == "source:outline");
var i = new ComplexSyndicationItem(item, outline);
Replay.OnNext(i);
syndication.Items.Add(i);
break;
default:
break;
}
}
syndications.Add(syndication);
}
}
Replay.OnCompleted();
return(syndications);
}
// Read an RssFeed
public static async Task CreateRssFeedReaderExample(string filePath)
{
// Create an XmlReader
// Example: ..\tests\TestFeeds\rss20-2items.xml
using (var xmlReader = XmlReader.Create(filePath, new XmlReaderSettings()
{
Async = true
}))
{
// Instantiate an Rss20FeedReader using the XmlReader.
// This will assign as default an Rss20FeedParser as the parser.
var feedReader = new RssFeedReader(xmlReader);
//
// Read the feed
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await feedReader.ReadCategory();
break;
// Read Image
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
break;
// Read Item
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
break;
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
break;
// Read Person
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
break;
// Read content
default:
ISyndicationContent content = await feedReader.ReadContent();
break;
}
}
}
}
public async Task ReadItemContent()
{
using (XmlReader xmlReader = XmlReader.Create(@"..\..\..\TestFeeds\simpleAtomFeed.xml", new XmlReaderSettings {
Async = true
}))
{
var reader = new AtomFeedReader(xmlReader);
while (await reader.Read())
{
if (reader.ElementType == SyndicationElementType.Item)
{
ISyndicationContent content = await reader.ReadContent();
var fields = content.Fields.ToArray();
Assert.True(fields.Length == 12);
Assert.True(fields[0].Name == "title");
Assert.False(string.IsNullOrEmpty(fields[0].Value));
Assert.True(fields[1].Name == "link");
Assert.True(fields[1].Attributes.Count() > 0);
Assert.True(fields[2].Name == "link");
Assert.True(fields[2].Attributes.Count() > 0);
Assert.True(fields[3].Name == "id");
Assert.False(string.IsNullOrEmpty(fields[3].Value));
Assert.True(fields[4].Name == "updated");
Assert.False(string.IsNullOrEmpty(fields[4].Value));
Assert.True(fields[5].Name == "published");
Assert.False(string.IsNullOrEmpty(fields[5].Value));
Assert.True(fields[6].Name == "source");
Assert.True(fields[6].Fields.Count() > 0);
Assert.True(fields[7].Name == "author");
Assert.True(fields[7].Fields.Count() > 0);
Assert.True(fields[8].Name == "contributor");
Assert.True(fields[8].Fields.Count() > 0);
Assert.True(fields[9].Name == "contributor");
Assert.True(fields[9].Fields.Count() > 0);
Assert.True(fields[10].Name == "rights");
Assert.False(string.IsNullOrEmpty(fields[10].Value));
Assert.True(fields[11].Name == "content");
Assert.False(string.IsNullOrEmpty(fields[11].Value));
}
}
}
}
private async Task <bool> retrievePosts(Feed feed)
{
using (var xmlReader = XmlReader.Create("https://azurecomcdn.azureedge.net/en-us/blog/feed/", new XmlReaderSettings()
{
Async = true
}))
{
// Instantiate an Rss20FeedReader using the XmlReader.
// This will assign as default an Rss20FeedParser as the parser.
var feedReader = new RssFeedReader(xmlReader);
//
// Read the feed
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await feedReader.ReadCategory();
break;
// Read Image
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
break;
// Read Item
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
break;
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
break;
// Read Person
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
break;
// Read content
default:
ISyndicationContent content = await feedReader.ReadContent();
break;
}
}
}
return(true);
}
private static async Task <List <object> > ParseExport(XmlReader xmlReader)
{
List <object> result = new List <object>();
{
var feedReader = new AtomFeedReader(xmlReader);
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await feedReader.ReadCategory();
result.Add(category);
break;
// Read Image
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
result.Add(image);
break;
// Read Item
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
result.Add(item);
break;
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
result.Add(link);
break;
// Read Person
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
result.Add(person);
break;
// Read content
default:
ISyndicationContent content = await feedReader.ReadContent();
result.Add(content);
break;
}
}
return(result);
}
}
private void SetFeedNameIfNotSet(Feed feed, ISyndicationContent content)
{
if (string.IsNullOrWhiteSpace(feed.Name))
{
if (content.Name == "title")
{
feed.Name = content.Value;
}
}
}
public ISyndicationImage ParseImage(string value)
{
ISyndicationContent content = ParseContent(value);
if (content.Name != AtomElementNames.Logo && content.Name != AtomElementNames.Icon)
{
throw new FormatException("Invalid Atom Image");
}
return(CreateImage(content));
}
public IAtomEntry ParseEntry(string value)
{
ISyndicationContent content = ParseContent(value);
if (content.Name != AtomElementNames.Entry)
{
throw new FormatException("Invalid Atom feed");
}
return(CreateEntry(content));
}
public virtual async Task <T> ReadValue <T>()
{
ISyndicationContent content = await ReadContent();
if (!Parser.TryParseValue(content.Value, out T value))
{
throw new FormatException();
}
return(value);
}
public ISyndicationLink ParseLink(string value)
{
ISyndicationContent content = ParseContent(value);
if (content.Name != AtomElementNames.Link)
{
throw new FormatException("Invalid Atom Link");
}
return(CreateLink(content));
}
public ComplexSyndicationItem(ISyndicationItem item, ISyndicationContent outline = null)
{
Item = item;
if (outline != null)
{
Outline = new Outline
{
Attributes = outline.Attributes.ToDictionary(x => x.Name, x => x.Value)
};
}
}
public virtual ISyndicationLink CreateLink(ISyndicationContent content)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
//
// Title
string title = content.Value;
//
// Url
Uri uri = null;
string url = content.Attributes.GetRss("url");
if (url != null)
{
if (!TryParseValue(url, out uri))
{
throw new FormatException("Invalid url attribute");
}
}
else
{
if (!TryParseValue(content.Value, out uri))
{
throw new FormatException("Invalid url");
}
title = null;
}
//
// Length
long length = 0;
TryParseValue(content.Attributes.GetRss("length"), out length);
//
// Type
string type = content.Attributes.GetRss("type");
//
// rel
string rel = (content.Name == RssElementNames.Link) ? RssLinkTypes.Alternate : content.Name;
return(new SyndicationLink(uri, rel)
{
Title = title,
Length = length,
MediaType = type
});
}
public ISyndicationPerson ParsePerson(string value)
{
ISyndicationContent content = ParseContent(value);
if (content.Name != AtomContributorTypes.Author && content.Name != AtomContributorTypes.Contributor)
{
throw new FormatException("Invalid Atom person");
}
return(CreatePerson(content));
}
public OutlineSyndicationItem(ISyndicationItem basic, ISyndicationContent outline)
{
Item = basic;
if (outline != null)
{
Outline = new Outline
{
Text = outline.Attributes.FirstOrDefault(x => x.Name.Equals("text", StringComparison.OrdinalIgnoreCase))?.Value,
Attributes = outline.Attributes.ToDictionary(x => x.Name, x => x.Value)
};
}
}
public ISyndicationImage ParseImage(string value)
{
ISyndicationContent content = ParseContent(value);
if (content.Name != RssElementNames.Image ||
content.Namespace != RssConstants.Rss20Namespace)
{
throw new FormatException("Invalid Rss Image");
}
return(CreateImage(content));
}
public ISyndicationLink ParseLink(string value)
{
ISyndicationContent content = ParseContent(value);
if (content.Name != RssElementNames.Link ||
content.Namespace != RssConstants.Rss20Namespace)
{
throw new FormatException("Invalid Rss link");
}
return(CreateLink(content));
}
public ISyndicationCategory ParseCategory(string value)
{
ISyndicationContent content = ParseContent(value);
if (content.Name != RssElementNames.Category ||
content.Namespace != RssConstants.Rss20Namespace)
{
throw new FormatException("Invalid Rss category");
}
return(CreateCategory(content));
}
public RssContent()
{
using (var xmlReader = XmlReader.Create("C:/Users/Developer/Desktop.brisbane-city-council.rss", new XmlReaderSettings()
{
Async = true
}))
{
var feedReader = new RssFeedReader(xmlReader);
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
// Read category
case SyndicationElementType.Category:
ISyndicationCategory category = await feedReader.ReadCategory();
break;
// Read Image
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
break;
// Read Item
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
break;
// Read link
case SyndicationElementType.Link:
ISyndicationLink link = await feedReader.ReadLink();
break;
// Read Person
case SyndicationElementType.Person:
ISyndicationPerson person = await feedReader.ReadPerson();
break;
// Read content
default:
ISyndicationContent content = await feedReader.ReadContent();
break;
}
}
}
}
public ISyndicationPerson ParsePerson(string value)
{
ISyndicationContent content = ParseContent(value);
if ((content.Name != RssElementNames.Author &&
content.Name != RssElementNames.ManagingEditor) ||
content.Namespace != RssConstants.Rss20Namespace)
{
throw new FormatException("Invalid Rss Person");
}
return(CreatePerson(content));
}
private async Task <List <ChapterUpdateBucket> > GetUpdatedChapterBucketList()
{
List <ChapterUpdateBucket> allUpdatedChapterBuckets = new();
DbResult <IEnumerable <NovelInfo> > result = await this.mediator.Send(new NovelInfos.GetAllNovelsInfo());
if (!result.TryGetValue(out IEnumerable <NovelInfo>?allNovelsInfo))
{
return(new List <ChapterUpdateBucket>());
}
// Retrieve all updated chapters
foreach (NovelInfo novelInfo in allNovelsInfo)
{
ChapterUpdateBucket temp = new(novelInfo);
using XmlReader reader = XmlReader.Create(novelInfo.SyndicationUri, new XmlReaderSettings { Async = true, });
RssFeedReader feedReader = new(reader);
while (await feedReader.Read())
{
if (feedReader.ElementType == SyndicationElementType.Item)
{
ISyndicationItem item = await feedReader.ReadItem();
ChapterUpdateItem chapterUpdateItem = item.ToChapterUpdateItem();
if (chapterUpdateItem.Id == novelInfo.MostRecentChapterId || chapterUpdateItem.Id == null)
{
break;
}
chapterUpdateItem.Description = chapterUpdateItem.Description.FromHtmlToDiscordMarkdown();
temp.ChapterUpdateItems.Add(chapterUpdateItem);
}
else if (feedReader.ElementType == SyndicationElementType.Content)
{
ISyndicationContent content = await feedReader.ReadContent();
if (content.Name.Equals("title") && !content.Value.Equals(temp.Novel.Name))
{
temp.NewTitle = content.Value;
}
}
}
if (temp.ChapterUpdateItems.Any())
{
allUpdatedChapterBuckets.Add(temp);
}
}
return(allUpdatedChapterBuckets);
}
public virtual ISyndicationPerson CreatePerson(ISyndicationContent content)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
string name = null;
string email = null;
string uri = null;
foreach (var field in content.Fields)
{
// content does not contain atom's namespace. So if we receibe a different namespace we will ignore it.
if (field.Namespace != AtomConstants.Atom10Namespace)
{
continue;
}
switch (field.Name)
{
//
// Name
case AtomElementNames.Name:
name = field.Value;
break;
//
// Email
case AtomElementNames.Email:
email = field.Value;
break;
//
// Uri
case AtomElementNames.Uri:
uri = field.Value;
break;
//
// Unrecognized field
default:
break;
}
}
return(new SyndicationPerson(name, email, content.Name)
{
Uri = uri
});
}
public void AddField(ISyndicationContent field)
{
if (field == null)
{
throw new ArgumentNullException(nameof(field));
}
if (_children.IsReadOnly)
{
_children = _children.ToList();
}
_children.Add(field);
}
public virtual ISyndicationImage CreateImage(ISyndicationContent content)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
if (!TryParseValue(content.Value, out Uri uri))
{
throw new FormatException("Invalid Atom image url");
}
return(new SyndicationImage(uri, content.Name));
}
public virtual ISyndicationCategory CreateCategory(ISyndicationContent content)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
if (content.Value == null)
{
throw new FormatException("Invalid Rss category name");
}
return(new SyndicationCategory(content.Value));
}
public virtual ISyndicationPerson CreatePerson(ISyndicationContent content)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
if (string.IsNullOrEmpty(content.Value))
{
throw new ArgumentNullException("Content value is required");
}
return(new SyndicationPerson(null, content.Value, content.Name));
}
