private async Task CompareFeeds(ISyndicationFeedReader f1, ISyndicationFeedReader f2)
{
while (await f1.Read() && await f2.Read())
{
Assert.True(f1.ElementType == f2.ElementType);
switch (f1.ElementType)
{
case SyndicationElementType.Item:
CompareItem(await f1.ReadItem(), await f2.ReadItem());
break;
case SyndicationElementType.Person:
ComparePerson(await f1.ReadPerson(), await f2.ReadPerson());
break;
case SyndicationElementType.Image:
CompareImage(await f1.ReadImage(), await f2.ReadImage());
break;
default:
CompareContent(await f1.ReadContent(), await f2.ReadContent());
break;
}
}
}
public RandomPosts(ISyndicationFeedReader syndicationFeedReader,
IRandomPostSettings randomPostSettings,
ILogger <RandomPosts> logger,
TelemetryClient telemetryClient)
{
_syndicationFeedReader = syndicationFeedReader;
_randomPostSettings = randomPostSettings;
_logger = logger;
_telemetryClient = telemetryClient;
}
private async Task ProcessBlog(Blog blog)
{
using (var reader = XmlReader.Create(blog.FeedUrl, new XmlReaderSettings {
Async = true, DtdProcessing = DtdProcessing.Parse
}))
{
ISyndicationFeedReader feedReader = null;
if (blog.FeedType == "atom")
{
feedReader = new AtomFeedReader(reader);
}
else
{
feedReader = new RssFeedReader(reader);
}
while (await feedReader.Read())
{
switch (feedReader.ElementType)
{
case SyndicationElementType.Item:
ISyndicationItem item = await feedReader.ReadItem();
var entity = new RssItem()
{
Title = item.Title,
Link = item.Id,
};
if (!entity.Link.StartsWith("http"))
{
entity.Link = blog.Url + item.Id;
}
using (var txn = this.StateManager.CreateTransaction())
{
var items = await this.StateManager.GetOrAddAsync <IReliableDictionary <string, RssItem> >(string.Format(rssItems, blog.Id.ToString("N")));
await items.AddAsync(txn, Guid.NewGuid().ToString("N"), entity);
await txn.CommitAsync();
}
break;
default:
break;
}
}
}
}
public CheckFeedForUpdates(ISyndicationFeedReader syndicationFeedReader,
ISettings settings,
ConfigurationRepository configurationRepository,
SourceDataRepository sourceDataRepository,
IUrlShortener urlShortener,
IEventPublisher eventPublisher,
ILogger <CheckFeedForUpdates> logger,
TelemetryClient telemetryClient)
{
_syndicationFeedReader = syndicationFeedReader;
_settings = settings;
_configurationRepository = configurationRepository;
_sourceDataRepository = sourceDataRepository;
_urlShortener = urlShortener;
_eventPublisher = eventPublisher;
_logger = logger;
_telemetryClient = telemetryClient;
}
public static async Task <ISyndicationItem> ReadFeedItemAsync(this ISyndicationFeedReader @this)
{
while (await @this.Read())
{
if (@this.ElementType == SyndicationElementType.Item)
{
var feedItem = await @this.ReadItem();
if (feedItem != null)
{
return(feedItem);
}
}
else
{
await @this.Skip();
}
}
return(null);
}
private async Task ParseReadElement(ApplicationDbContext context, Feed feed, ISyndicationFeedReader feedreader)
{
_logger.LogTrace($"Parsing element {feedreader.ElementName}");
try
{
switch (feedreader.ElementType)
{
case SyndicationElementType.Item:
var item = await feedreader.ReadItem();
var itemidentifier = item.Links.FirstOrDefault().Uri.AbsoluteUri;
if (await context.Articles
.Where(x => x.UniqueId == itemidentifier)
.AsNoTracking()
.CountAsync() > 0)
{
break;
}
var article = CreateArticleFromItem(feed, item, itemidentifier);
context.Articles.Add(article);
await context.SaveChangesAsync();
break;
default:
SetFeedNameIfNotSet(feed, await feedreader.ReadContent());
break;
}
}
catch (Exception ex)
{
_logger.LogError(ex, $"An error occurred while refreshing \"{feed.Name}\"");
}
}
public SyndicationFeedReaderTests(ISyndicationFeedReader syndicationFeedReader, IRandomPostSettings randomPostSettings, ITestOutputHelper testOutputHelper)
{
_syndicationFeedReader = syndicationFeedReader;
_randomPostSettings = randomPostSettings;
_testOutputHelper = testOutputHelper;
}
// Handles RSS2.0 and Atom feeds
private async Task <bool> HandleXmlFeed(Feed feed, bool populateCacheOnly = false)
{
using (log.BeginScope($"{ nameof(FeedService) }->{ nameof(HandleXmlFeed) } with feed: {feed}"))
{
try
{
// Create async reader
using (XmlReader xmlReader = XmlReader.Create(feed.ConnectionString, new XmlReaderSettings()
{
Async = true
}))
{
ISyndicationFeedReader reader = null;
// Special reader for every type
switch (feed.Type)
{
case FeedType.RSS:
log.LogDebug("Creating RssFeedReader");
reader = new RssFeedReader(xmlReader);
break;
case FeedType.Atom:
log.LogDebug("Creating AtomFeedReader");
reader = new AtomFeedReader(xmlReader);
break;
default:
log.LogError("Invalid FeedType found.");
return(false);
}
log.LogDebug("Starting feed reading");
// Read the entries async
while (await reader.Read())
{
log.LogDebug("Found elementType '{ElementType}'", reader.ElementType);
// We only need the entrytypes (yet)
if (reader.ElementType == SyndicationElementType.Item)
{
FeedEntry feedEntry = null;
// Create my feedEntry
switch (feed.Type)
{
case FeedType.RSS:
ISyndicationItem rssEntry = await(reader as RssFeedReader).ReadItem();
log.LogDebug("ISyndicationItem: {Item}", rssEntry);
feedEntry = CreateFeedEntry(rssEntry, feed);
break;
case FeedType.Atom:
IAtomEntry atomEntry = await(reader as AtomFeedReader).ReadEntry();
log.LogDebug("IAtomEntry: {Entry}", atomEntry);
feedEntry = CreateFeedEntry(atomEntry, feed);
break;
default:
continue;
}
// Check if we already had this one
if (await cache.IsCached(feedEntry.Id))
{
log.LogDebug("ID '{Id}' is in cache", feedEntry.Id);
}
// Check if the entry is older then MaxAge to prevent double posting
else if (feedEntry?.Published <= (DateTime.Now - cfg.Application.Cache.MaxAge))
{
log.LogInformation("Entry '{Id}' is too old.", feedEntry.Id);
}
else
{
// Hand over to instance service
log.LogDebug("Tooting id '{Id}' ", feedEntry.Id);
await TootTheFeedEntry(feedEntry, populateCacheOnly, feed);
}
}
}
return(true);
}
}
catch (Exception ex)
{
log.LogError(ex, "HandleXmlFeed Exception - {feed}", feed);
return(false);
}
}
}
