async Task <River> WriteRiver(Uri uri, River river)
{
river = await MaybeArchiveRiver(uri.AbsoluteUri, river);
await feedStore.WriteRiver(uri, river);
return(river);
}
public async Task <River> FetchRiver(Uri uri)
{
River river = await feedStore.LoadRiverForFeed(uri);
while (river.Metadata.LastStatus == HttpStatusCode.MovedPermanently)
{
river = await feedStore.LoadRiverForFeed(river.Metadata.OriginUrl);
}
return(river);
}
public async Task <River> UpdateAsync(River river)
{
FetchResult fetchResult = await FetchAsync(
river.Metadata.OriginUrl,
river.Metadata.Etag,
river.Metadata.LastModified
);
var updatedFeeds = river.UpdatedFeeds;
if (fetchResult.Feed != null)
{
var feed = fetchResult.Feed;
Item[] newItems = await this.feedItemStore.StoreItems(
river.Metadata.OriginUrl, feed.Items.ToArray());
// TODO: Filter this out once we've loaded a bit.
var existingItems = new HashSet <string>(
from existingFeed in river.UpdatedFeeds.Feeds
from item in existingFeed.Items
where item.Id != null
select item.Id
);
newItems = newItems.Where(item => !existingItems.Contains(item.Id)).ToArray();
if (newItems.Length > 0)
{
Uri baseUri = SyndicationUtil.TryParseAbsoluteUrl(feed.WebsiteUrl) ?? feed.FeedUrl;
for (int i = 0; i < newItems.Length; i++)
{
newItems[i] = Rebase(newItems[i], baseUri);
}
newItems = await this.thumbnailExtractor.LoadItemThumbnailsAsync(baseUri, newItems);
await this.feedItemStore.UpdateItemThumbs(river.Metadata.OriginUrl, newItems);
feed = feed.With(items: newItems);
updatedFeeds = river.UpdatedFeeds.With(feeds: river.UpdatedFeeds.Feeds.Insert(0, feed));
}
}
var metadata = river.Metadata.With(
etag: fetchResult.Etag,
lastModified: fetchResult.LastModified,
originUrl: fetchResult.FeedUrl,
lastStatus: fetchResult.Status);
return(river.With(updatedFeeds: updatedFeeds, metadata: metadata));
}
public async Task <River> RefreshAggregateRiverWithFeeds(string id, IList <Uri> feedUrls)
{
Stopwatch aggregateTimer = Stopwatch.StartNew();
River river = await this.aggregateStore.LoadAggregate(id);
Log.AggregateRefreshStart(id, feedUrls.Count);
DateTimeOffset lastUpdated = river.UpdatedFeeds.Feeds.Count > 0
? river.UpdatedFeeds.Feeds.Max(f => f.WhenLastUpdate)
: DateTimeOffset.MinValue;
var parser = new RiverLoader();
River[] rivers = await Task.WhenAll(from url in feedUrls select parser.FetchRiver(url));
Log.AggregateRefreshPulledRivers(id, rivers.Length);
var newFeeds = new List <FeedSegment>();
foreach (River feedRiver in rivers)
{
FeedSegment[] newUpdates =
feedRiver.UpdatedFeeds.Feeds.Where(rf => rf.WhenLastUpdate > lastUpdated).ToArray();
Log.AggregateNewUpdates(id, feedRiver.Metadata.OriginUrl, newUpdates.Length);
if (newUpdates.Length > 0)
{
Item[] newItems = newUpdates.SelectMany(rf => rf.Items).ToArray();
DateTimeOffset biggestUpdate = newUpdates.Max(rf => rf.WhenLastUpdate);
Log.AggregateFeedState(id, feedRiver.Metadata.OriginUrl, newUpdates.Length, biggestUpdate);
newFeeds.Add(newUpdates[0].With(whenLastUpdate: biggestUpdate, items: newItems));
}
}
// Sort all the new feeds by time they were updated (latest time first).
Log.AggregateHasNewFeeds(id, newFeeds.Count);
newFeeds = newFeeds.OrderByDescending(f => f.WhenLastUpdate).ToList();
River newRiver = river.With(
updatedFeeds: river.UpdatedFeeds.With(feeds: newFeeds.Concat(river.UpdatedFeeds.Feeds)));
newRiver = await MaybeArchiveRiver(id, newRiver);
Log.UpdatingAggregate(id);
await this.aggregateStore.WriteAggregate(id, newRiver);
Log.AggregateRefreshed(id, aggregateTimer);
return(newRiver);
}
async Task <River> MaybeArchiveRiver(string id, River river)
{
if (river.UpdatedFeeds.Feeds.Count > UpdateLimit)
{
Log.SplittingFeed(id, river);
ImmutableList <FeedSegment> feeds = river.UpdatedFeeds.Feeds;
IEnumerable <FeedSegment> oldFeeds = feeds.Skip(UpdateSize);
IEnumerable <FeedSegment> newFeeds = feeds.Take(UpdateSize);
River oldRiver = river.With(updatedFeeds: river.UpdatedFeeds.With(feeds: oldFeeds));
string archiveKey = await this.archiveStore.WriteRiverArchive(oldRiver);
Log.WroteArchive(id, river, archiveKey);
river = river.With(
updatedFeeds: river.UpdatedFeeds.With(feeds: newFeeds),
metadata: river.Metadata.With(next: archiveKey)
);
}
return(river);
}
public async Task <River> FetchAndUpdateRiver(Uri uri)
{
River river = await feedStore.LoadRiverForFeed(uri);
if ((river.Metadata.LastStatus != HttpStatusCode.MovedPermanently) &&
(river.Metadata.LastStatus != HttpStatusCode.Gone))
{
river = await UpdateAsync(river);
if (river.Metadata.LastStatus == HttpStatusCode.MovedPermanently)
{
// The first time this happens we need to move all the items across.
await feedItemStore.UpdateOriginUrl(uri, river.Metadata.OriginUrl);
}
await WriteRiver(uri, river);
}
if (river.Metadata.LastStatus == HttpStatusCode.MovedPermanently)
{
return(await FetchAndUpdateRiver(river.Metadata.OriginUrl));
}
return(river);
}