public async Task MakesLinksAbsoluteInDescription()
{
// Given
TestExecutionContext context = new TestExecutionContext();
context.Settings[Keys.Host] = "statiq.dev";
TestDocument document = new TestDocument(new NormalizedPath("a.txt"))
{
{ FeedKeys.Title, "A" },
{ FeedKeys.Description, "<p>Foo <a href=\"/fizz/buzz\">Fizzbuzz</a> bar <img src=\"/abc/def.png\"> baz</p>" }
};
GenerateFeeds module = new GenerateFeeds();
// When
IReadOnlyList <TestDocument> results = await ExecuteAsync(document, context, module);
// Then
TestDocument result = results.Single(x => x.Destination.FullPath == "feed.rss");
result.Content.ShouldContain(@"<description><p>Foo <a href=""http://statiq.dev/fizz/buzz"">Fizzbuzz</a> bar <img src=""http://statiq.dev/abc/def.png""> baz</p></description>");
}
public async Task DoesNotChangeImageDomain()
{
// Given
TestExecutionContext context = new TestExecutionContext();
context.Settings[Keys.Host] = "buzz.com";
TestDocument document = new TestDocument(
new FilePath("/input/fizz/buzz"),
new FilePath("fizz/buzz"),
new Dictionary <string, object>
{
{ FeedKeys.Image, new Uri("http://foo.com/bar/baz.png") }
});
GenerateFeeds module = new GenerateFeeds();
// When
IReadOnlyList <TestDocument> results = await ExecuteAsync(document, context, module);
// Then
results.Select(x => x.Destination.FullPath).ShouldBe(new[] { "feed.rss", "feed.atom" }, true);
results[0].Content.ShouldContain("http://foo.com/bar/baz.png");
}
public async Task PreservesInputOrder()
{
// Given
TestDocument a = new TestDocument(new NormalizedPath("a.txt"))
{
{ FeedKeys.Title, "A" },
{ FeedKeys.Published, new DateTime(2010, 1, 1) }
};
TestDocument b = new TestDocument(new NormalizedPath("b.txt"))
{
{ FeedKeys.Title, "B" },
{ FeedKeys.Published, new DateTime(2010, 2, 1) }
};
GenerateFeeds module = new GenerateFeeds().PreserveOrdering();
// When
IReadOnlyList <TestDocument> results = await ExecuteAsync(new[] { a, b }, module);
// Then
TestDocument result = results.Single(x => x.Destination.FullPath == "feed.rss");
result.Content.IndexOf("<title>A</title>").ShouldBeLessThan(result.Content.IndexOf("<title>B</title>"));
}
public void DoesNotChangeImageDomain()
{
// Given
TestExecutionContext context = new TestExecutionContext();
context.AddTypeConversion <string, Uri>(x => new Uri(x));
context.Settings[Keys.Host] = "buzz.com";
TestDocument document = new TestDocument(new Dictionary <string, object>
{
{ Keys.RelativeFilePath, new FilePath("fizz/buzz") },
{ FeedKeys.Image, new Uri("http://foo.com/bar/baz.png") }
});
document.AddTypeConversion <Uri, string>(x => x.ToString());
document.AddTypeConversion <FilePath, string>(x => x.FullPath);
GenerateFeeds module = new GenerateFeeds();
// When
IList <IDocument> results = module.Execute(new[] { document }, context).ToList(); // Make sure to materialize the result list
// Then
results.Select(x => x.FilePath(Keys.WritePath).FullPath).ShouldBe(new[] { "feed.rss", "feed.atom" }, true);
results[0].Content.ShouldContain("http://foo.com/bar/baz.png");
}
public Feeds()
{
Dependencies.AddRange(nameof(Content), nameof(Archives), nameof(Data));
InputModules = new ModuleList
{
new ReadFiles("**/{!_,}*.{json,yaml,yml}")
};
ProcessModules = new ModuleList
{
new ExecuteSwitch(Config.FromDocument(doc => doc.ContentProvider.MediaType))
.Case(MediaTypes.Json, new ParseJson())
.Case(MediaTypes.Yaml, new ParseYaml()),
new FilterDocuments(Config.FromDocument(IsFeed)),
new ForEachDocument
{
new ExecuteConfig(Config.FromDocument(feedDoc =>
{
ModuleList modules = new ModuleList();
// Get outputs from the pipeline(s)
modules.Add(
new ReplaceDocuments(feedDoc.GetList(WebKeys.FeedPipelines, new[] { nameof(Content) }).ToArray()),
new MergeMetadata(Config.FromValue(feedDoc.Yield())).KeepExisting());
// Filter by document source
if (feedDoc.ContainsKey(WebKeys.FeedSources))
{
modules.Add(new FilterSources(feedDoc.GetList <string>(WebKeys.FeedSources)));
}
// Filter by metadata
if (feedDoc.ContainsKey(WebKeys.FeedFilter))
{
modules.Add(new FilterDocuments(Config.FromDocument(doc => doc.GetBool(WebKeys.FeedFilter))));
}
// Order the documents
if (feedDoc.ContainsKey(WebKeys.FeedOrderKey))
{
modules.Add(
new OrderDocuments(feedDoc.GetString(WebKeys.FeedOrderKey))
.Descending(feedDoc.GetBool(WebKeys.FeedOrderDescending)));
}
// Limit the count
if (feedDoc.ContainsKey(WebKeys.FeedSize))
{
modules.Add(new TakeDocuments(feedDoc.GetInt(WebKeys.FeedSize)));
}
// Generate the feed(s)
GenerateFeeds generateFeeds = new GenerateFeeds()
.MaximumItems(feedDoc.GetInt(WebKeys.FeedSize))
.WithRssPath(feedDoc.GetBool(WebKeys.FeedRss, false) ? feedDoc.Destination.ChangeExtension("rss") : null)
.WithAtomPath(feedDoc.GetBool(WebKeys.FeedAtom, false) ? feedDoc.Destination.ChangeExtension("atom") : null)
.WithRdfPath(feedDoc.GetBool(WebKeys.FeedRdf, false) ? feedDoc.Destination.ChangeExtension("rdf") : null)
.WithFeedId(feedDoc.Get <Uri>(WebKeys.FeedId))
.WithFeedTitle(feedDoc.GetString(WebKeys.FeedTitle))
.WithFeedDescription(feedDoc.GetString(WebKeys.FeedDescription))
.WithFeedAuthor(feedDoc.GetString(WebKeys.FeedAuthor))
.WithFeedPublished(feedDoc.ContainsKey(WebKeys.FeedPublished) ? feedDoc.GetDateTime(WebKeys.FeedPublished) : (DateTime?)null)
.WithFeedUpdated(feedDoc.ContainsKey(WebKeys.FeedUpdated) ? feedDoc.GetDateTime(WebKeys.FeedUpdated) : (DateTime?)null)
.WithFeedLink(feedDoc.Get <Uri>(WebKeys.FeedLink))
.WithFeedImageLink(feedDoc.Get <Uri>(WebKeys.FeedImageLink))
.WithFeedCopyright(feedDoc.GetString(WebKeys.Copyright));
// Set the per-item delegates (these would have been copied down to each document from the feed document in the MergeMetadata up above)
if (feedDoc.ContainsKey(WebKeys.FeedItemId))
{
generateFeeds = generateFeeds.WithItemId(Config.FromDocument(doc => doc.Get <Uri>(WebKeys.FeedItemId)));
}
if (feedDoc.ContainsKey(WebKeys.FeedItemTitle))
{
generateFeeds = generateFeeds.WithItemTitle(Config.FromDocument(doc => doc.GetString(WebKeys.FeedItemTitle)));
}
if (feedDoc.ContainsKey(WebKeys.FeedItemDescription))
{
generateFeeds = generateFeeds.WithItemDescription(Config.FromDocument(doc => doc.GetString(WebKeys.FeedItemDescription)));
}
if (feedDoc.ContainsKey(WebKeys.FeedItemAuthor))
{
generateFeeds = generateFeeds.WithItemAuthor(Config.FromDocument(doc => doc.GetString(WebKeys.FeedItemAuthor)));
}
if (feedDoc.ContainsKey(WebKeys.FeedItemPublished))
{
generateFeeds = generateFeeds.WithItemPublished(Config.FromDocument(doc => doc.Get <DateTime?>(WebKeys.FeedItemPublished)));
}
if (feedDoc.ContainsKey(WebKeys.FeedItemUpdated))
{
generateFeeds = generateFeeds.WithItemUpdated(Config.FromDocument(doc => doc.Get <DateTime?>(WebKeys.FeedItemUpdated)));
}
if (feedDoc.ContainsKey(WebKeys.FeedItemLink))
{
generateFeeds = generateFeeds.WithItemLink(Config.FromDocument(doc => doc.Get <Uri>(WebKeys.FeedItemLink)));
}
if (feedDoc.ContainsKey(WebKeys.FeedItemImageLink))
{
generateFeeds = generateFeeds.WithItemImageLink(Config.FromDocument(doc => doc.Get <Uri>(WebKeys.FeedItemImageLink)));
}
if (feedDoc.ContainsKey(WebKeys.FeedItemContent))
{
generateFeeds = generateFeeds.WithItemContent(Config.FromDocument(doc => doc.GetString(WebKeys.FeedItemContent)));
}
if (feedDoc.ContainsKey(WebKeys.FeedItemThreadLink))
{
generateFeeds = generateFeeds.WithItemThreadLink(Config.FromDocument(doc => doc.Get <Uri>(WebKeys.FeedItemThreadLink)));
}
if (feedDoc.ContainsKey(WebKeys.FeedItemThreadCount))
{
generateFeeds = generateFeeds.WithItemThreadCount(Config.FromDocument(doc => doc.GetInt(WebKeys.FeedItemThreadCount)));
}
if (feedDoc.ContainsKey(WebKeys.FeedItemThreadUpdated))
{
generateFeeds = generateFeeds.WithItemThreadUpdated(Config.FromDocument(doc => doc.Get <DateTime?>(WebKeys.FeedItemThreadUpdated)));
}
modules.Add(generateFeeds);
return(modules);
}))
}
};
OutputModules = new ModuleList
{
new WriteFiles()
};
}
public EmailFeedPipeline()
{
Dependencies.AddRange(nameof(Inputs), nameof(Content), nameof(Archives), nameof(Data));
ProcessModules = new ModuleList
{
new GetPipelineDocuments(ContentType.Data),
// Filter to feeds
new FilterDocuments(Config.FromDocument(IsFeed)),
// Generate the feeds
new ForEachDocument
{
new ExecuteConfig(Config.FromDocument(feedDoc =>
{
ModuleList modules = new ModuleList();
// Get outputs from the pipeline(s)
modules.Add(
new ReplaceDocuments(feedDoc.GetList(WebKeys.FeedPipelines, new[] { nameof(Content) }).ToArray()),
new MergeMetadata(Config.FromValue(feedDoc.Yield())).KeepExisting());
// Filter by document source
if (feedDoc.ContainsKey(WebKeys.FeedSources))
{
modules.Add(new FilterSources(feedDoc.GetList <string>(WebKeys.FeedSources)));
}
// Filter by metadata
if (feedDoc.ContainsKey(WebKeys.FeedFilter))
{
modules.Add(new FilterDocuments(Config.FromDocument(doc => doc.GetBool(WebKeys.FeedFilter))));
}
// Order the documents
if (feedDoc.ContainsKey(WebKeys.FeedOrderKey))
{
modules.Add(
new OrderDocuments(feedDoc.GetString(WebKeys.FeedOrderKey))
.Descending(feedDoc.GetBool(WebKeys.FeedOrderDescending)));
}
// Generate the feed(s)
GenerateFeeds generateFeeds = new GenerateFeeds()
.WithRssPath("feed/index.xml")
.WithFeedId(feedDoc.GetString(WebKeys.FeedId))
.WithFeedTitle(feedDoc.GetString(WebKeys.FeedTitle))
.WithFeedDescription(feedDoc.GetString(WebKeys.FeedDescription))
.WithFeedAuthor(feedDoc.GetString(WebKeys.FeedAuthor))
.WithFeedPublished(feedDoc.ContainsKey(WebKeys.FeedPublished) ? feedDoc.GetDateTime(WebKeys.FeedPublished) : (DateTime?)null)
.WithFeedUpdated(feedDoc.ContainsKey(WebKeys.FeedUpdated) ? feedDoc.GetDateTime(WebKeys.FeedUpdated) : (DateTime?)null)
.WithFeedLink(feedDoc.Get <Uri>(WebKeys.FeedLink))
.WithFeedImageLink(feedDoc.Get <Uri>(WebKeys.FeedImageLink))
.WithFeedCopyright(feedDoc.GetString(WebKeys.Copyright));
// Set the per-item delegates (these would have been copied down to each document from the feed document in the MergeMetadata up above)
if (feedDoc.ContainsKey(WebKeys.FeedItemId))
{
generateFeeds = generateFeeds.WithItemId(Config.FromDocument(doc => doc.GetString(WebKeys.FeedItemId)));
}
if (feedDoc.ContainsKey(WebKeys.FeedItemTitle))
{
generateFeeds = generateFeeds.WithItemTitle(Config.FromDocument(doc => doc.GetString(WebKeys.FeedItemTitle)));
}
generateFeeds = generateFeeds.WithItemDescription(Config.FromDocument(doc => doc.GetString("Excerpt", WebKeys.FeedItemDescription)));
if (feedDoc.ContainsKey(WebKeys.FeedItemAuthor))
{
generateFeeds = generateFeeds.WithItemAuthor(Config.FromDocument(doc => doc.GetString(WebKeys.FeedItemAuthor)));
}
if (feedDoc.ContainsKey(WebKeys.FeedItemPublished))
{
generateFeeds = generateFeeds.WithItemPublished(Config.FromDocument(doc => doc.Get <DateTime?>(WebKeys.FeedItemPublished)));
}
if (feedDoc.ContainsKey(WebKeys.FeedItemUpdated))
{
generateFeeds = generateFeeds.WithItemUpdated(Config.FromDocument(doc => doc.Get <DateTime?>(WebKeys.FeedItemUpdated)));
}
if (feedDoc.ContainsKey(WebKeys.FeedItemLink))
{
//generateFeeds = generateFeeds.WithItemLink(Config.FromDocument(doc => doc.Get<Uri>(WebKeys.FeedItemLink)));
}
generateFeeds = generateFeeds.WithItemImageLink(Config.FromDocument((doc, ctx) => {
return(TypeHelper.Convert <Uri>(ctx.GetLink(doc, "Card", true)) ?? TypeHelper.Convert <Uri>(ctx.GetLink("/images/ogcard.jpeg", true)));
}
));
if (feedDoc.ContainsKey(WebKeys.FeedItemContent))
{
generateFeeds = generateFeeds.WithItemContent(Config.FromDocument(doc => doc.GetString(WebKeys.FeedItemContent)));
}
if (feedDoc.ContainsKey(WebKeys.FeedItemThreadLink))
{
generateFeeds = generateFeeds.WithItemThreadLink(Config.FromDocument(doc => doc.Get <Uri>(WebKeys.FeedItemThreadLink)));
}
if (feedDoc.ContainsKey(WebKeys.FeedItemThreadCount))
{
generateFeeds = generateFeeds.WithItemThreadCount(Config.FromDocument(doc => doc.GetInt(WebKeys.FeedItemThreadCount)));
}
if (feedDoc.ContainsKey(WebKeys.FeedItemThreadUpdated))
{
generateFeeds = generateFeeds.WithItemThreadUpdated(Config.FromDocument(doc => doc.Get <DateTime?>(WebKeys.FeedItemThreadUpdated)));
}
modules.Add(generateFeeds);
return(modules);
}))
}
};
OutputModules = new ModuleList
{
new FilterDocuments(Config.FromDocument(WebKeys.ShouldOutput, true)),
new WriteFiles()
};
}
