/// <summary>
/// Parses the given <see cref="FeedType"/> and returns a <see cref="IList<Item>"/>.
/// </summary>
/// <returns></returns>
public IList<Item> Parse(Feed feed)
{
try
{
_cleaner = FeedCleaner.ResolveCleaner(feed);
// Make sure we fill the original state object, not the 'info' variable which is a copy.
switch (feed.FeedType)
{
case FeedType.RSS:
return ParseRss(feed);
case FeedType.RDF:
return ParseRdf(feed);
case FeedType.Atom:
return ParseAtom(feed);
default:
throw new NotSupportedException(string.Format("{0} is not supported", feed.FeedType.ToString()));
}
}
catch (Exception e)
{
Logger.Info("An Exception occured with FeedFetcher.Parse:\n\n{0}", e);
return new List<Item>();
}
}
public static FeedCleaner ResolveCleaner(Feed feed)
{
switch (feed.Cleaner)
{
case "AolNewsCleaner":
return new AolNewsCleaner();
case "SlashdotCleaner":
return new SlashdotCleaner();
default:
return new FeedCleaner();
}
}
/// <summary>
/// All <see cref="Feed"/>s in the database. The Category and Site properties are null.
/// </summary>
/// <returns></returns>
public static List<Feed> AllFeeds()
{
List<Feed> list = new List<Feed>();
try
{
using (SqlConnection connection = new SqlConnection(ConnectionString))
{
connection.Open();
using (SqlCommand command = new SqlCommand())
{
// Ignore read items if it's setup.
string sql = "SELECT * FROM feeds ";
command.Connection = connection;
command.CommandText = sql;
using (SqlDataReader reader = command.ExecuteReader())
{
while (reader.Read())
{
Feed feed = new Feed();
feed.Id = (Guid)reader["id"];
feed.Cleaner = (string)reader["cleaner"];
feed.Url = (string)reader["url"];
feed.FeedType = (FeedType) ((byte)reader["type"]);
// Not needed
feed.Category = null;
feed.Site = null;
list.Add(feed);
}
}
}
}
}
catch (SqlException e)
{
Logger.Warn("A SqlException occured getting AllFeeds from the database:\n\n{0}", e);
}
return list;
}
public IList<Feed> ListFeeds()
{
if (FeedCache.Current.IsCached())
return FeedCache.Current.Items;
IList<Feed> list = new List<Feed>();
try
{
using (SqliteConnection connection = new SqliteConnection(FeedConnectionString))
{
connection.Open();
using (SqliteCommand command = new SqliteCommand(connection))
{
command.CommandText = @"SELECT f.*,c.title as categoryTitle,s.title as siteTitle, s.url as siteUrl " +
"FROM feeds f INNER JOIN categories c ON UPPER(c.id) = UPPER(f.categoryid) " +
"INNER JOIN sites s on UPPER(s.id) = UPPER(f.siteid)";
using (SqliteDataReader reader = command.ExecuteReader())
{
while (reader.Read())
{
Category category = new Category();
category.Id = (Guid)reader["categoryid"];
category.Title = (string)reader["categoryTitle"];
Site site = new Site();
site.Id = (Guid)reader["siteid"];
site.Title = (string)reader["siteTitle"];
site.Url = (string)reader["siteUrl"];
Feed feed = new Feed();
feed.Id = (Guid)reader["id"];
feed.Category = category;
feed.Site = site;
feed.Cleaner = (string)reader["cleaner"];
feed.Url = (string)reader["url"];
long type = (long)reader["type"];
feed.FeedType = (FeedType)type;
list.Add(feed);
}
}
}
}
}
catch (SqliteException e)
{
Logger.Warn("SqliteException occured while listing feeds: \n{0}", e);
}
FeedCache.Current.Update(list);
return list;
}
/// <summary>
/// Parses an Atom feed and returns a <see cref="IList<Item>"/>.
/// </summary>
public virtual IList<Item> ParseAtom(Feed feed)
{
try
{
string xml = LoadXml(feed.Url);
using (StringReader reader = new StringReader(xml))
{
XDocument doc = XDocument.Load(reader);
// Feed/Entry
var entries = from item in doc.Root.Elements().Where(i => i.Name.LocalName == "entry")
select item;
List<Item> list = FillList(entries, feed, FeedType.Atom);
ParseImages(list);
return list;
}
}
catch (Exception e)
{
Logger.Info("An error occured with FeedFetcher.ParseAtom '{0}':\n\n{1}", feed.Url, e);
return new List<Item>();
}
}
private List<Item> FillList(IEnumerable<XElement> elements, Feed feed, FeedType feedType)
{
List<Item> list = new List<Item>();
int count = 0;
foreach (XElement element in elements)
{
Item item = new Item();
item.Id = Guid.NewGuid();
item.Feed = feed;
if (feedType == FeedType.Atom)
{
string html = element.Elements().First(i => i.Name.LocalName == "content").Value;
item.Content = CleanContent(html);
item.RawHtml = html;
item.Link = element.Elements().First(i => i.Name.LocalName == "link").Attribute("href").Value;
item.PublishDate = ParseDate(element.Elements().First(i => i.Name.LocalName == "published").Value);
item.Title = CleanTitle(element.Elements().First(i => i.Name.LocalName == "title").Value);
}
else
{
string html = element.Elements().First(i => i.Name.LocalName == "description").Value;
// <content:encoded> contains richer content
if (element.Elements().Any(i => i.Name.LocalName == "encoded"))
html = element.Elements().First(i => i.Name.LocalName == "encoded").Value;
// <enclosure> tag (RSS 2).
if (element.Elements().Any(i => i.Name.LocalName == "enclosure"))
{
XElement enclosure = element.Elements().First(i => i.Name.LocalName == "enclosure");
XAttribute enclosureUrl = enclosure.Attribute("url");
if (enclosureUrl != null)
{
item.ImageUrl = NormaliseImageUrl(feed.Url, enclosureUrl.Value);
}
}
item.Content = CleanContent(html);
item.RawHtml = html;
item.Link = element.Elements().First(i => i.Name.LocalName == "link").Value;
item.Title = CleanTitle(element.Elements().First(i => i.Name.LocalName == "title").Value);
XElement dateElement = null;
// The only difference we care about with RSS and RDF:
if (feedType == FeedType.RSS)
dateElement = element.Elements().FirstOrDefault(i => i.Name.LocalName == "pubDate");
else
dateElement = element.Elements().FirstOrDefault(i => i.Name.LocalName == "date");
if (dateElement != null)
{
item.PublishDate = ParseDate(dateElement.Value);
// Some naughty feeds set their publish date to the future
if (item.PublishDate > DateTime.UtcNow)
item.PublishDate = DateTime.UtcNow;
}
else
{
Logger.Info("Warning: {0} is set as {1} but no pubDate/date element was found for the PublishDate.", feed.Url, feed.FeedType);
}
}
// Check it has all the fields so it's not garbage
if (!string.IsNullOrEmpty(item.Title) && !string.IsNullOrEmpty(item.Content) && !string.IsNullOrEmpty(item.Link))
{
// Check it's not already in our list.
if (!AllItems.Exists(i => i.GetHashCode() == item.GetHashCode()))
{
// Restrict the database size over time to something sane
if (count >= _settings.MaximumItemsPerFeed)
break;
list.Add(item);
}
count++;
}
}
return list;
}
