public async Task <Feed> SubscribeFeedAsync(string feedOriginalUri, string customGroup, User user)
{
Feed feed = null;
string feedUriHash = null;
var db = _dbFactory.CreateDbContext();
for (var i = 0; i < 10; ++i)
{
var feedUri = feedOriginalUri.Trim().ToLower();
feedUriHash = Utils.Sha256(feedUri);
// Do we have this feed already?
var feedInDb = await db.Feeds.FindAsync(feedUriHash);
if (feedInDb == null)
{
// Get information of this feed.
feed = await RefreshFeedAsync(feedOriginalUri, noItems : true);
if (feed.Error != null)
{
// This uri may not be the feed uri. It maybe the webpage, let's try to find out the potential feed uri in this webpage.
feedOriginalUri = await DiscoverFeedUriAsync(feedOriginalUri);
if (string.IsNullOrWhiteSpace(feedOriginalUri))
{
break;
}
continue;
}
// Save to feed table.
feedInDb = new ServerCore.Models.Feed
{
Description = feed.Description,
IconUri = feed.IconUri,
Id = feedUriHash,
UriHash = feedUriHash,
Name = feed.Name,
RegistrationTimeInUtc = DateTime.UtcNow,
WebSiteUri = feed.WebsiteLink,
Uri = feed.OriginalUri,
};
db.Feeds.Add(feedInDb);
var tryTimes = 0;
for (; tryTimes < 10; ++tryTimes)
{
if (tryTimes == 0)
{
feedInDb.ShortId = Utils.Crc32(feedInDb.Id);
}
else
{
feedInDb.ShortId = Utils.Crc32(feedInDb.Id + tryTimes);
}
try
{
await db.SaveChangesAsync();
feed.ShortId = feedInDb.ShortId;
break;
}
catch (DbUpdateException ex) when(ex.IsUniqueConstraintException())
{
if (await _dbFactory.CreateDbContext().Feeds.FindAsync(feedUriHash) != null)
{
// Save to ignore.
break;
}
}
}
if (tryTimes < 10)
{
break;
}
else
{
throw new ExternalErrorExcepiton($"Save feed {feed.OriginalUri} failed.");
}
}
else
{
feed = new Feed
{
ShortId = feedInDb.ShortId,
Description = feedInDb.Description,
IconUri = feedInDb.IconUri,
Name = feedInDb.Name,
OriginalUri = feedInDb.Uri,
Uri = feedUri,
WebsiteLink = feedInDb.WebSiteUri,
};
break;
}
}
Debug.Assert(feed != null);
Debug.Assert(feedUriHash != null);
// User user customized group.
feed.Group = customGroup;
// Save to usersfeeds table.
db.UserFeeds.Add(new ServerCore.Models.UserFeed
{
UserId = user.Id,
FeedId = feedUriHash,
Group = customGroup
});
try
{
await db.SaveChangesAsync();
}
catch (DbUpdateException ex) when(ex.IsUniqueConstraintException())
{
// Save to ignore.
}
// Return
return(feed);
}
public async Task <Feed> GetFeedItemsAsync(string feedUri, string feedShortId, int page, User user)
{
if (string.IsNullOrEmpty(feedUri) && string.IsNullOrEmpty(feedShortId))
{
throw new ExternalErrorExcepiton($"FeedUri and FeedShortId can't be empty in both.");
}
string feedId = null;
var db = _dbFactory.CreateDbContext();
if (string.IsNullOrEmpty(feedShortId))
{
var originalUri = feedUri;
feedUri = feedUri.Trim().ToLower();
feedId = Utils.Sha256(feedUri);
}
// Get feed information.
// If the user blob is not null, we will get the feed from the user blob because user might customize the group and name on this feed.
ServerCore.Models.UserFeed userFeed = null;
ServerCore.Models.Feed feedInDb = null;
if (user != null)
{
userFeed = await db.UserFeeds.Include(f => f.Feed).FirstOrDefaultAsync(u => u.UserId == user.Id && (u.FeedId == feedId || u.Feed.ShortId == feedShortId));
feedInDb = userFeed?.Feed;
}
// If we didn't get the feed, two possibility:
// 1. The feed is not subscribed by user yet.
// 2. Anonymous user.
// No matter for which case, we will try to get the feed info from feed info table directly.
if (feedInDb == null)
{
feedInDb = await db.Feeds.FirstOrDefaultAsync(f => f.Id == feedId || f.ShortId == feedShortId);
}
if (feedInDb == null)
{
throw new ExternalErrorExcepiton($"Feed '{feedUri ?? feedShortId}' is not found.");
}
feedId = feedInDb.Id;
var feed = new Feed
{
ShortId = feedInDb.ShortId,
Description = feedInDb.Description,
IconUri = feedInDb.IconUri,
Name = feedInDb.Name,
OriginalUri = feedInDb.Uri,
Uri = feedUri,
WebsiteLink = feedInDb.WebSiteUri,
};
if (userFeed != null)
{
feed.Group = userFeed.Group;
}
// Get feed items.
var feedItems = await db.FeedItems
.Where(f => f.FeedId == feedId)
.OrderByDescending(f => f.PublishTimeInUtc)
.Skip(page * 50)
.Take(50).ToListAsync();
if (feedItems.Count > 0)
{
feed.Items = feedItems.Select(f => new FeedItem
{
Id = f.Id,
Summary = f.Summary,
Content = f.Content,
PermentLink = f.Uri,
PubDate = f.PublishTimeInUtc,
Title = f.Title,
TopicPictureUri = f.TopicPictureUri,
}).ToList();
}
// Mark readed or not.
if (userFeed != null && userFeed.LastReadedTimeInUtc.Ticks != 0)
{
foreach (var feedItem in feed.Items)
{
if (feedItem.PubDate <= userFeed.LastReadedTimeInUtc)
{
feedItem.IsReaded = true;
}
}
}
// Mark stared or not.
if (user != null)
{
// Mark stared or not
var favorites = db.UserFeedItems.Where(f => f.UserId == user.Id && f.IsFavorite).Select(f => f.FeedItemId).ToList();
if (favorites.Count > 0)
{
foreach (var feedItem in feed.Items)
{
if (favorites.Find(id => id == feedItem.PermentLink.Sha256()) != null)
{
feedItem.IsStared = true;
}
}
}
}
// All done.
return(feed);
}
public static async Task UpdateFeed(IDbContextFactory <FeedReaderDbContext> dbFactory, ISolrOperations <Solrs.FeedItem> solrFeedItems, ServerCore.Models.Feed feed, ILogger log, HttpClient httpClient = null)
{
log.LogInformation($"UpdateFeed: {feed.Uri}");
// Get original uri.
var feedUriHash = feed.Id;
var feedOriginalUri = feed.Uri;
// Get feed content.
if (httpClient == null)
{
var handler = new HttpClientHandler()
{
AutomaticDecompression = System.Net.DecompressionMethods.All, AllowAutoRedirect = true
};
httpClient = new HttpClient(handler);
httpClient.DefaultRequestHeaders.Add("User-Agent", "FeedReader");
httpClient.DefaultRequestHeaders.CacheControl = new CacheControlHeaderValue {
NoCache = true
};
}
var feedContent = await httpClient.GetStringAsync(feedOriginalUri);
// Create parser.
var parser = FeedParser.FeedParser.Create(feedContent);
// Parse feed info.
var feedInfo = parser.ParseFeedInfo();
// Parse html content to get icon uri.
if (string.IsNullOrWhiteSpace(feedInfo.IconUri))
{
string uri = null;
try
{
if (!string.IsNullOrWhiteSpace(feedInfo.WebsiteLink))
{
uri = feedInfo.WebsiteLink;
}
else
{
uri = new Uri(feedOriginalUri).GetLeftPart(UriPartial.Authority);
}
var response = await httpClient.GetAsync(uri);
feedInfo.IconUri = new HtmlParser(await response.Content.ReadAsStringAsync(), response.RequestMessage.RequestUri.ToString()).ShortcurtIcon;
}
catch (HttpRequestException ex)
{
log.LogError($"Try download the content of {uri} failed. Ex: {ex.Message}");
}
catch (Exception ex)
{
log.LogError($"Parse html content of {uri} failed. Ex: {ex.Message}");
}
}
// Try to get the favicon.ico file directly, like https://www.cagle.com, we can't get any icon info from the index page, but it has favicon.ico file.
if (string.IsNullOrWhiteSpace(feedInfo.IconUri))
{
string uri = null;
try
{
var uriBuilder = new UriBuilder(string.IsNullOrWhiteSpace(feedInfo.WebsiteLink) ? feedOriginalUri : feedInfo.WebsiteLink);
uriBuilder.Path = "/favicon.ico";
uri = uriBuilder.Uri.ToString();
var response = await httpClient.GetAsync(uriBuilder.Uri);
if (response.IsSuccessStatusCode)
{
feedInfo.IconUri = uri;
}
}
catch (HttpRequestException ex)
{
log.LogError($"Try download the content of {uri} failed. Ex: {ex.Message}");
}
}
// Save feed info to table.
feed.Description = feedInfo.Description;
feed.IconUri = feedInfo.IconUri;
feed.LastUpdateTimeInUtc = DateTime.UtcNow;
feed.Name = feedInfo.Name;
feed.WebSiteUri = feedInfo.WebsiteLink;
// Parse feed items.
var feedItems = parser.ParseFeedItems();
// Save feed items to db.
var db = dbFactory.CreateDbContext();
foreach (var item in feedItems)
{
var feedItemId = item.PermentLink.Sha256();
if (await db.FeedItems.FindAsync(feedItemId) == null)
{
var feedItem = new ServerCore.Models.FeedItem
{
Content = item.Content,
FeedId = feed.Id,
Id = feedItemId,
PublishTimeInUtc = item.PubDate.ToUniversalTime(),
Summary = item.Summary,
Title = item.Title,
TopicPictureUri = item.TopicPictureUri,
Uri = item.PermentLink
};
db.FeedItems.Add(feedItem);
solrFeedItems.Add(new Solrs.FeedItem(feedItem, feed));
}
}
await db.SaveChangesAsync();
// Save to solrs
try
{
await solrFeedItems.CommitAsync();
}
catch (Exception ex)
{
log.LogError($"Save to solrs failed. ex: {ex.Message}");
}
log.LogInformation($"UpdateFeed: {feed.Uri} finished");
}
