public async Task ParseWebResponse(FeedSource source, WebResponse response)
{
await Console.Error.WriteLineAsync("[Builder/Html] Parsing Html");
// Parse the HTML
HtmlDocument html = new HtmlDocument();
using (StreamReader reader = new StreamReader(response.GetResponseStream()))
html.LoadHtml(await reader.ReadToEndAsync());
HtmlNode document = html.DocumentNode;
document.AbsolutifyUris(new Uri(source.Feed.Url));
await Console.Error.WriteLineAsync("[Builder/Html] Generating feed content");
// Add the title
await feed.WriteTitle(ReferenceSubstitutor.Replace(source.Feed.Title, document));
await feed.WriteSubtitle(ReferenceSubstitutor.Replace(source.Feed.Subtitle, document));
// Add the logo
if (source.Feed.Logo != null)
{
HtmlNode logoNode = document.QuerySelector(source.Feed.Logo.Selector);
xml.WriteElementString("logo", logoNode.Attributes[source.Feed.Logo.Attribute].Value);
}
// Add the feed entries
foreach (HtmlNode nextNode in document.QuerySelectorAll(source.Entries.Selector))
{
await addEntry(source, nextNode);
}
}
public async Task <string> WriteAtomAsync()
{
var feed = GetItemCollection(FeedItemCollection);
var sw = new StringWriterWithEncoding(Encoding.UTF8);
await using (var xmlWriter = XmlWriter.Create(sw, new() { Async = true }))
{
var writer = new AtomFeedWriter(xmlWriter);
await writer.WriteTitle(HeadTitle);
await writer.WriteSubtitle(HeadDescription);
await writer.WriteRights(Copyright);
await writer.WriteUpdated(DateTime.UtcNow);
await writer.WriteGenerator(Generator, HostUrl, GeneratorVersion);
foreach (var item in feed)
{
await writer.Write(item);
}
await xmlWriter.FlushAsync();
xmlWriter.Close();
}
var xml = sw.ToString();
return(xml);
}
private async Task <ISyndicationFeedWriter> GetWriter(string type, XmlWriter xmlWriter, DateTime updated)
{
var host = Request.Scheme + "://" + Request.Host + "/";
if (type.Equals("rss", StringComparison.OrdinalIgnoreCase))
{
var rss = new RssFeedWriter(xmlWriter);
await rss.WriteTitle(_settings.Brand);
await rss.WriteDescription(_settings.Description);
await rss.WriteGenerator("Bagombo Blog Engine");
await rss.WriteValue("link", host);
return(rss);
}
var atom = new AtomFeedWriter(xmlWriter);
await atom.WriteTitle(_settings.Brand);
await atom.WriteId(host);
await atom.WriteSubtitle(_settings.Description);
await atom.WriteGenerator("Bagombo Blog Engine", "https://github.com/tylerlrhodes/bagobo", "0.2.5a");
await atom.WriteValue("updated", updated.ToString("yyyy-MM-ddTHH:mm:ssZ"));
return(atom);
}
public async Task WriteValue()
{
const string title = "Example Feed";
Guid id = Guid.NewGuid();
DateTimeOffset updated = DateTimeOffset.UtcNow.AddDays(-21);
var sw = new StringWriterWithEncoding(Encoding.UTF8);
using (var xmlWriter = XmlWriter.Create(sw))
{
var writer = new AtomFeedWriter(xmlWriter);
await writer.WriteTitle(title);
await writer.WriteId(id.ToString());
await writer.WriteUpdated(updated);
await writer.Flush();
}
string res = sw.ToString();
Assert.True(CheckResult(res, $"<title>{title}</title><id>{id}</id><updated>{updated.ToString("r")}</updated>"));
}
public async Task WriteAtom10FileAsync(string absolutePath)
{
var feed = GetSyndicationItemCollection(FeedItemCollection);
var settings = new XmlWriterSettings
{
Async = true,
Encoding = Encoding.UTF8,
Indent = true
};
using var xmlWriter = XmlWriter.Create(absolutePath, settings);
var writer = new AtomFeedWriter(xmlWriter);
await writer.WriteTitle(HeadTitle);
await writer.WriteSubtitle(HeadDescription);
await writer.WriteRights(Copyright);
await writer.WriteUpdated(DateTime.UtcNow);
await writer.WriteGenerator(Generator, HostUrl, GeneratorVersion);
foreach (var item in feed)
{
await writer.Write(item);
}
await xmlWriter.FlushAsync();
xmlWriter.Close();
}
private async Task <ISyndicationFeedWriter> GetWriter(string type, XmlWriter xmlWriter, DateTime updated)
{
string host = Request.Scheme + "://" + Request.Host + "/";
if (type.Equals("rss", StringComparison.OrdinalIgnoreCase))
{
var rss = new RssFeedWriter(xmlWriter);
await rss.WriteTitle(_settings.Value.Name);
await rss.WriteDescription(_settings.Value.Description);
await rss.WriteGenerator("Miniblog.Core");
await rss.WriteValue("link", host);
return(rss);
}
var atom = new AtomFeedWriter(xmlWriter);
await atom.WriteTitle(_settings.Value.Name);
await atom.WriteId(host);
await atom.WriteSubtitle(_settings.Value.Description);
await atom.WriteGenerator("Miniblog.Core", "https://github.com/madskristensen/Miniblog.Core", "1.0");
await atom.WriteValue("updated", updated.ToString("yyyy-MM-ddTHH:mm:ssZ"));
return(atom);
}
private async Task <ISyndicationFeedWriter> GetWriter(string?type, XmlWriter xmlWriter, DateTime updated)
{
var host = $"{this.Request.Scheme}://{this.Request.Host}/";
if (type?.Equals("rss", StringComparison.OrdinalIgnoreCase) ?? false)
{
var rss = new RssFeedWriter(xmlWriter);
await rss.WriteTitle(this.manifest.Name).ConfigureAwait(false);
await rss.WriteDescription(this.manifest.Description).ConfigureAwait(false);
await rss.WriteGenerator("Miniblog.Core").ConfigureAwait(false);
await rss.WriteValue("link", host).ConfigureAwait(false);
return(rss);
}
var atom = new AtomFeedWriter(xmlWriter);
await atom.WriteTitle(this.manifest.Name).ConfigureAwait(false);
await atom.WriteId(host).ConfigureAwait(false);
await atom.WriteSubtitle(this.manifest.Description).ConfigureAwait(false);
await atom.WriteGenerator("Miniblog.Core", "https://github.com/madskristensen/Miniblog.Core", "1.0").ConfigureAwait(false);
await atom.WriteValue("updated", updated.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture)).ConfigureAwait(false);
return(atom);
}
private async Task <ISyndicationFeedWriter> GetWriter(string type, XmlWriter xmlWriter, DateTime updated)
{
string host = Request.Scheme + "://" + Request.Host + "/";
if (type.Equals("rss", StringComparison.OrdinalIgnoreCase))
{
var rss = new RssFeedWriter(xmlWriter);
await rss.WriteTitle(_manifest.Name);
await rss.WriteDescription(_manifest.Description);
await rss.WriteGenerator("BrickApp");
await rss.WriteValue("link", host);
return(rss);
}
var atom = new AtomFeedWriter(xmlWriter);
await atom.WriteTitle(_manifest.Name);
await atom.WriteId(host);
await atom.WriteSubtitle(_manifest.Description);
await atom.WriteValue("updated", updated.ToString("yyyy-MM-ddTHH:mm:ssZ"));
return(atom);
}
public static async Task <string> ToAtom(this IList <Page> pages)
{
using (var stringWriter = new StringWriter())
{
using (var xmlWriter = XmlWriter.Create(stringWriter,
new XmlWriterSettings
{
Async = true,
Indent = true,
Encoding = Encoding.UTF8,
WriteEndDocumentOnClose = true
}))
{
var atomWriter = new AtomFeedWriter(xmlWriter);
var tasks = new List <Task>
{
atomWriter.WriteTitle(Settings.Title),
atomWriter.WriteSubtitle(Settings.Description),
atomWriter.WriteId(Settings.Domain),
atomWriter.Write(new SyndicationLink(new Uri(Settings.Domain))),
atomWriter.WriteUpdated(pages.First().Timestamp.ToLocalTime())
};
tasks.AddRange(pages.Select(p => atomWriter.Write(p.ToSyndicationItem())));
Task.WaitAll(tasks.ToArray());
await atomWriter.Flush();
}
return(stringWriter.ToString());
}
}
/// <summary>
/// Write default meta data for an atom feed
/// </summary>
/// <param name="atomFeedWriter">the AtomFeedWriter</param>
/// <param name="id">the id of the feed</param>
/// <param name="title">the title of the feed</param>
/// <param name="version">the version of the feed</param>
/// <param name="selfUri">the self referencing uri of the feed</param>
/// <param name="relatedUrls">optional related urls</param>
/// <returns></returns>
public static async Task WriteDefaultMetadata(
this AtomFeedWriter atomFeedWriter,
string id,
string title,
string version,
Uri selfUri,
params string[] relatedUrls)
{
await atomFeedWriter.WriteId(id);
await atomFeedWriter.WriteTitle(title);
await atomFeedWriter.WriteSubtitle("Basisregisters Vlaanderen stelt u in staat om alles te weten te komen rond: de Belgische gemeenten; de Belgische postcodes; de Vlaamse straatnamen; de Vlaamse adressen; de Vlaamse gebouwen en gebouweenheden; de Vlaamse percelen; de Vlaamse organisaties en organen; de Vlaamse dienstverlening.");
await atomFeedWriter.WriteGenerator("Basisregisters Vlaanderen", "https://basisregisters.vlaanderen.be", version);
await atomFeedWriter.WriteRights($"Copyright (c) 2017-{DateTime.Now.Year}, Informatie Vlaanderen");
await atomFeedWriter.WriteUpdated(DateTimeOffset.UtcNow);
await atomFeedWriter.Write(new SyndicationPerson("agentschap Informatie Vlaanderen", "*****@*****.**", AtomContributorTypes.Author));
await atomFeedWriter.Write(new SyndicationLink(selfUri, AtomLinkTypes.Self));
foreach (var relatedUrl in relatedUrls)
{
await atomFeedWriter.Write(new SyndicationLink(new Uri(relatedUrl), AtomLinkTypes.Related));
}
}
public override async Task ExecuteResultAsync(ActionContext context)
{
using (XmlWriter xmlWriter = XmlWriter.Create(context.HttpContext.Response.Body, new XmlWriterSettings {
Async = true, Indent = true
}))
{
var writer = new AtomFeedWriter(xmlWriter);
var uniqueFeedIdBuilder = new UriBuilder(m_feedRequestUrl)
{
Scheme = Uri.UriSchemeHttp,
Query = string.Empty
};
var uniqueFeedId = uniqueFeedIdBuilder.ToString();
await writer.WriteId(uniqueFeedId);
await writer.WriteTitle(m_feedTitle);
//await writer.WriteDescription(m_feedTitle);
await writer.Write(new SyndicationLink(m_feedRequestUrl));
//await writer.WriteUpdated(DateTimeOffset.UtcNow);
foreach (var syndicationItem in m_feedItems)
{
syndicationItem.Id = $"{uniqueFeedId}?itemId={syndicationItem.Id}";
await writer.Write(syndicationItem);
}
xmlWriter.Flush();
}
}
/// <summary>
/// Write default meta data for an atom feed
/// </summary>
/// <param name="atomFeedWriter">the AtomFeedWriter</param>
/// <param name="atomFeedConfiguration">the configuration of the atom feed</param>
/// <returns></returns>
public static async Task WriteDefaultMetadata(
this AtomFeedWriter atomFeedWriter,
AtomFeedConfiguration atomFeedConfiguration)
{
await atomFeedWriter.WriteId(atomFeedConfiguration.Id);
await atomFeedWriter.WriteTitle(atomFeedConfiguration.Title);
await atomFeedWriter.WriteSubtitle(atomFeedConfiguration.Subtitle);
await atomFeedWriter.WriteGenerator(atomFeedConfiguration.GeneratorTitle, atomFeedConfiguration.GeneratorUri, atomFeedConfiguration.GeneratorVersion);
await atomFeedWriter.WriteRights(atomFeedConfiguration.Rights);
await atomFeedWriter.WriteUpdated(atomFeedConfiguration.Updated);
await atomFeedWriter.Write(atomFeedConfiguration.Author);
await atomFeedWriter.Write(atomFeedConfiguration.SelfUri);
foreach (var alternateUri in atomFeedConfiguration.AlternateUris)
{
await atomFeedWriter.Write(alternateUri);
}
foreach (var relatedUri in atomFeedConfiguration.RelatedUris)
{
await atomFeedWriter.Write(relatedUri);
}
}
public async Task ExecuteResultAsync(ActionContext context)
{
var response = context.HttpContext.Response;
var request = context.HttpContext.Request;
response.ContentType = "application/atom+xml";
var host = request.Scheme + "://" + request.Host + "/";
using (var xmlWriter = XmlWriter.Create(response.Body, new XmlWriterSettings()
{
Async = true, Indent = true, Encoding = new UTF8Encoding(true)
}))
{
var atom = new AtomFeedWriter(xmlWriter);
await atom.WriteTitle(_options.Value.Title);
await atom.WriteId(host);
await atom.WriteSubtitle(_options.Value.Description);
await atom.WriteRaw($"\n <link rel=\"self\" type=\"application/atom+xml\" href=\"{host}feed/atom\"/>");
await atom.WriteGenerator(_options.Value.GeneratorDescription, _options.Value.SourceCodeLink, "1.0");
var lastPost = _page.Items.FirstOrDefault();
if (lastPost != null)
{
await atom.WriteValue("updated", lastPost.PublishedDate.Value.ToString("yyyy-MM-ddTHH:mm:ssZ"));
}
foreach (var post in _page.Items)
{
var item = new AtomEntry
{
Title = post.Title,
Description = post.GetContentWithoutDataSrc(),
Id = $"{host}/{post.Alias}",
Published = post.PublishedDate.Value,
LastUpdated = post.ModifiedDate,
ContentType = "html",
};
foreach (var tag in post.BlogStoryTags)
{
item.AddCategory(new SyndicationCategory(tag.Tag.Name));
}
item.AddContributor(new SyndicationPerson(_options.Value.AuthorName, _options.Value.Email, "author"));
// item.AddLink(new SyndicationLink(new Uri($"{host}/{post.Alias}")));
await atom.Write(item);
}
}
}
private XmlFeedWriter InitializeAtomFeedWriter(XmlWriter xmlWriter)
{
var result = new AtomFeedWriter(xmlWriter);
result.WriteTitle(_siteSettings.SiteName);
result.WriteSubtitle(_siteSettings.Description);
result.WriteRights(_siteSettings.Copyright);
result.WriteUpdated(_blogPostsConfig.Blogs.Where(x => x.Published).First().CreateDate.Date);
result.WriteGenerator(_siteSettings.Nickname, _siteSettings.PersonalSiteURL, _siteSettings.Version);
return(result);
}
public async Task <string> GenerateFeed(IEnumerable <ReleaseViewModel> releases)
{
const string title = "Changelog Feed";
var id = new UniqueId();
var updated = DateTimeOffset.UtcNow;
var sw = new StringWriterWithEncoding(Encoding.UTF8);
using (var xmlWriter = XmlWriter.Create(sw, new XmlWriterSettings()
{
Async = true, Indent = true
}))
{
var writer = new AtomFeedWriter(xmlWriter);
await writer.WriteTitle(title);
await writer.WriteId(id.ToString());
await writer.WriteUpdated(updated);
foreach (var release in releases)
{
var item = new SyndicationItem()
{
Title = release.ReleaseVersion,
Id = new UniqueId(release.ReleaseId).ToString(),
LastUpdated = release.ReleaseDate
};
release.Authors.ForEach(x =>
item.AddContributor(new SyndicationPerson(x, $"{x}@test.com"))
);
var sb = new StringBuilder();
foreach (var workItemViewModel in release.WorkItems)
{
sb.Append(workItemViewModel.WorkItemTypeString)
.Append(": ")
.AppendLine(workItemViewModel.Description)
;
}
item.Description = sb.ToString();
await writer.Write(item);
}
xmlWriter.Flush();
}
return(sw.GetStringBuilder().ToString());
}
/// <summary>
/// Creates the feed writer and writes the initial headers.
/// </summary>
/// <param name="writer">The current xml writer</param>
/// <param name="blog">The blog service</param>
/// <param name="url">The currently requested url</param>
/// <param name="host">The host name</param>
/// <returns>The feed writer</returns>
private async Task <ISyndicationFeedWriter> GetWriter(XmlWriter writer, IBlogService blog, string url, string host, DateTime?latest)
{
var segments = url.Substring(1).Split('/');
if (latest.HasValue && segments.Length == 2)
{
if (segments[1] == "rss")
{
var rss = new RssFeedWriter(writer);
// Write feed headers
await rss.WriteTitle(blog.Settings.Title);
await rss.WriteDescription(blog.Settings.Description);
await rss.WriteGenerator("RazorBlog");
await rss.WriteValue("link", host);
return(rss);
}
else if (segments[1] == "atom")
{
var atom = new AtomFeedWriter(writer);
// Write feed headers
await atom.WriteTitle(blog.Settings.Title);
await atom.WriteId(host);
await atom.WriteSubtitle(blog.Settings.Description);
await atom.WriteGenerator("RazorBlog", "https://github.com/tidyui/razorblog", "0.1");
await atom.WriteValue("updated", latest.Value.ToString("yyyy-MM-ddTHH:mm:ssZ"));
return(atom);
}
}
return(null);
}
public async Task WriteCDATAValue()
{
var sw = new StringWriterWithEncoding(Encoding.UTF8);
string title = "Title & Markup";
using (var xmlWriter = XmlWriter.Create(sw))
{
var writer = new AtomFeedWriter(xmlWriter, null, new AtomFormatter()
{
UseCDATA = true
});
await writer.WriteTitle(title);
await writer.Flush();
}
var res = sw.ToString();
Assert.True(CheckResult(res, $"<title><![CDATA[{title}]]></title>"));
}
public async Task <string> WriteAtomAsync()
{
var feed = GetItemCollection(FeedItemCollection);
var settings = new XmlWriterSettings
{
Async = true
};
var sb = new StringBuilder();
await using (var xmlWriter = XmlWriter.Create(sb, settings))
{
var writer = new AtomFeedWriter(xmlWriter);
await writer.WriteTitle(HeadTitle);
await writer.WriteSubtitle(HeadDescription);
await writer.WriteRights(Copyright);
await writer.WriteUpdated(DateTime.UtcNow);
await writer.WriteGenerator(Generator, HostUrl, GeneratorVersion);
foreach (var item in feed)
{
await writer.Write(item);
}
await xmlWriter.FlushAsync();
xmlWriter.Close();
}
var xml = sb.ToString();
return(xml);
}
public async Task <ISyndicationFeedWriter> GetWriter(string type, string host, XmlWriter xmlWriter)
{
var lastPost = _db.BlogPosts.All().OrderByDescending(p => p.Published).FirstOrDefault();
var blog = await _db.CustomFields.GetBlogSettings();
if (lastPost == null)
{
return(null);
}
if (type.Equals("rss", StringComparison.OrdinalIgnoreCase))
{
var rss = new RssFeedWriter(xmlWriter);
await rss.WriteTitle(blog.Title);
await rss.WriteDescription(blog.Description);
await rss.WriteGenerator("Blogifier");
await rss.WriteValue("link", host);
return(rss);
}
var atom = new AtomFeedWriter(xmlWriter);
await atom.WriteTitle(blog.Title);
await atom.WriteId(host);
await atom.WriteSubtitle(blog.Description);
await atom.WriteGenerator("Blogifier", "https://github.com/blogifierdotnet/Blogifier", "1.0");
await atom.WriteValue("updated", lastPost.Published.ToString("yyyy-MM-ddTHH:mm:ssZ"));
return(atom);
}
public async Task WritePrefixedAtomNs()
{
const string title = "Example Feed";
const string uri = "https://contoso.com/generator";
const string generator = "Example Toolkit";
var sw = new StringWriterWithEncoding(Encoding.UTF8);
using (var xmlWriter = XmlWriter.Create(sw))
{
var writer = new AtomFeedWriter(xmlWriter,
new ISyndicationAttribute[] { new SyndicationAttribute("xmlns:atom", "http://www.w3.org/2005/Atom") });
await writer.WriteTitle(title);
await writer.WriteGenerator(generator, uri, null);
await writer.Flush();
}
string res = sw.ToString();
Assert.True(CheckResult(res, $"<atom:title>{title}</atom:title><atom:generator uri=\"{uri}\">{generator}</atom:generator>", "atom"));
}
private async Task <string> GetSyndicationItems(string id)
{
// See if we already have the items in the cache
if (_cache.TryGetValue($"{id}_items", out string s))
{
Log.Information("CACHE HIT: Returning {bytes} bytes", s.Length);
return(s);
}
Log.Information("CACHE MISS: Loading feed items for {id}", id);
var sb = new StringBuilder();
var stringWriter = new StringWriterWithEncoding(sb, Encoding.UTF8);
int days = 5;
var feed = GetFeed(id);
using (XmlWriter xmlWriter = XmlWriter.Create(stringWriter, new XmlWriterSettings()
{
Async = true, Indent = true, Encoding = Encoding.UTF8
}))
{
var rssWriter = new AtomFeedWriter(xmlWriter);
await rssWriter.WriteTitle(feed.title);
await rssWriter.Write(new SyndicationLink(new Uri(feed.url)));
await rssWriter.WriteUpdated(DateTimeOffset.UtcNow);
// Add Items
foreach (var item in await GetFeedItems(id.ToLowerInvariant(), days))
{
try
{
var si = new SyndicationItem()
{
Id = item.Id,
Title = item.Title.Replace("\u0008", "").Replace("\u0003", "").Replace("\u0010", "").Replace("\u0012", "").Replace("\u0002", "").Replace("\u001f", ""),
Description = item.ArticleText.Replace("\u0008", "").Replace("\u0003", "").Replace("\u0010", "").Replace("\u0012", "").Replace("\u0002", "").Replace("\u001f", ""),
Published = item.DateAdded,
LastUpdated = item.DateAdded
};
si.AddLink(new SyndicationLink(new Uri(item.Url)));
si.AddContributor(new SyndicationPerson(string.IsNullOrWhiteSpace(item.SiteName) ? item.HostName : item.SiteName, feed.authoremail, AtomContributorTypes.Author));
await rssWriter.Write(si);
}
catch (Exception ex)
{
Log.Error(ex, "Error building item {urlHash}:{url}", item.UrlHash, item.Url);
}
}
xmlWriter.Flush();
}
// Add the items to the cache before returning
s = stringWriter.ToString();
_cache.Set <string>($"{id}_items", s, TimeSpan.FromMinutes(60));
Log.Information("CACHE SET: Storing feed items for {id} for {minutes} minutes", id, 60);
return(s);
}