public void does_not_update_feed_that_was_not_modified()
{
var cached = new LocalFeedInfo()
{
Url = "http://cached.feed/rss",
Content = "cached-content",
Etag = "cached-etag",
LastFetch = new DateTime(2012, 2, 3)
};
_feedsProvider.Db().LocalFeedInfo.Insert(cached);
A.CallTo(
() => _remoteData.ReadRemoteSource(
A <LocalFeedInfo> .That.Matches(x => x.Url == cached.Url && x.Etag == cached.Etag && x.LastFetch == cached.LastFetch)
)
)
.Returns(RemoteContentResponse.NotModified(HttpStatusCode.OK));
execute(cached.Url);
List <LocalFeedInfo> fromDb = _feedsProvider.Db().LocalFeedInfo.All().ToList <LocalFeedInfo>();
Assert.Equal(1, fromDb.Count);
Assert.Equal(cached.Url, fromDb[0].Url);
Assert.Equal(cached.Content, fromDb[0].Content);
Assert.Equal(cached.Etag, fromDb[0].Etag);
Assert.Equal(cached.LastFetch, fromDb[0].LastFetch);
}
public void issues_HEAD_request_before_GET_to_see_if_new_content_available_when_configured_to_do_so___workaround_for_servers_that_ignore_http_cache_request_headers_like_etag_and_ifmodifiedsince()
{
_configurationProvider.ProvideConfig().PrefetchHeadRequest = true;
_validUri = "http://www.blogojciec.pl/feed/";
configure_webrequest();
var localFeedInfo = new LocalFeedInfo
{
Url = _validUri,
};
var first_time_response = execute(localFeedInfo);
Assert.True(first_time_response.HasNewContent);
localFeedInfo.LastFetch = ApplicationTime.Current.AddHours(1);
var subsequent_response = execute(localFeedInfo);
Assert.False(subsequent_response.HasNewContent);
localFeedInfo.LastFetch = ApplicationTime.Current.AddDays(-10);
var response_with_changed_content = execute(localFeedInfo);
Assert.True(response_with_changed_content.HasNewContent);
}
public void updates_cache_with_new_feed_content()
{
var now = new DateTime(2012, 2, 20);
ApplicationTime._replaceCurrentTimeLogic(() => now);
var cached = new LocalFeedInfo()
{
Url = "http://cached.feed/rss",
Content = "cached-content",
Etag = "cached-etag",
LastFetch = new DateTime(2012, 2, 3)
};
_feedsProvider.Db().LocalFeedInfo.Insert(cached);
var newFeed = prepare_fake_feed();
A.CallTo(
() => _remoteData.ReadRemoteSource(
A <LocalFeedInfo> .That.Matches(x => x.Url == cached.Url && x.Etag == cached.Etag && x.LastFetch == cached.LastFetch)
)
)
.Returns(new RemoteContentResponse(true, "new-etag", newFeed, HttpStatusCode.OK));
execute(cached.Url);
List <LocalFeedInfo> fromDb = _feedsProvider.Db().LocalFeedInfo.All().ToList <LocalFeedInfo>();
Assert.Equal(1, fromDb.Count);
Assert.Equal(cached.Url, fromDb[0].Url);
Assert.Equal(newFeed.GetRssString(), fromDb[0].Content);
Assert.Equal("new-etag", fromDb[0].Etag);
Assert.Equal(now, fromDb[0].LastFetch);
}
private RemoteContentResponse execute(LocalFeedInfo feedInfo)
{
return(_remoteData.ReadRemoteSource(feedInfo));
}
public RemoteContentResponse ReadRemoteSource(LocalFeedInfo feedInfo)
{
_log.Debug("Trying to read remote source '{0}' (etag: {1}, last fetch: {2})", feedInfo.Url, feedInfo.Etag, feedInfo.LastFetch);
try
{
if (feedInfo.LastFetch.HasValue && _configProvider.ProvideConfig().PrefetchHeadRequest)
{
var head_request = _httpRequestFactory.CreateRequest(feedInfo.Url);
head_request.Method = "HEAD";
using (var head_response = (HttpWebResponse)head_request.GetResponseEx())
{
DateTime last_modified = head_response.LastModified.ToUniversalTime();
DateTime last_fetch = feedInfo.LastFetch.Value.ToUniversalTime();
if (last_modified < last_fetch)
{
_log.Debug("HEAD request indicates that there is no new content available (last_modified={0}, last_fetch={1}), returning NotModified.", last_modified, last_fetch);
return(RemoteContentResponse.NotModified(head_response.StatusCode));
}
}
}
var get_request = _httpRequestFactory.CreateRequest(feedInfo.Url);
if (feedInfo.LastFetch != null)
{
get_request.IfModifiedSince = feedInfo.LastFetch.Value;
}
if (string.IsNullOrWhiteSpace(feedInfo.Etag) == false)
{
get_request.Headers[HttpRequestHeader.IfNoneMatch] = feedInfo.Etag;
}
using (var response = (HttpWebResponse)get_request.GetResponseEx())
{
if (response.StatusCode == HttpStatusCode.OK)
{
string etag = response.Headers[HttpResponseHeader.ETag];
Stream responseStream = response.GetResponseStream();
using (var streamReader = new StreamReader(responseStream))
{
string responseText = streamReader.ReadToEnd();
responseText = _contentPreProcessor.PreProcess(responseText);
using (var stringReader = new StringReader(responseText))
{
using (var reader = XmlReader.Create(stringReader))
{
var feed = SyndicationFeed.Load(reader);
_log.Info("Returning new content for remote source '{0}' with {1} items", feedInfo.Url, feed.Items.Count());
return(new RemoteContentResponse(true, etag, feed, response.StatusCode));
}
}
}
}
else
{
string message = string.Format("No new content for remote source '{0}', response status is {1}", feedInfo.Url, response.StatusCode);
if (response.StatusCode == HttpStatusCode.NotModified)
{
_log.Debug(message);
}
else
{
_log.Warn(message);
}
return(RemoteContentResponse.NotModified(response.StatusCode));
}
}
}
catch (Exception exc)
{
_log.ErrorException(string.Format("Failed to read remote source '{0}'", feedInfo.Url), exc);
throw new CannotReachRemoteSourceException(feedInfo.Url, exc);
}
}
protected virtual void OnNoNewContentInRemoteSource(dynamic db, LocalFeedInfo feed, RemoteContentResponse remoteResponse)
{
}
protected virtual void OnFeedUpdated(dynamic db, LocalFeedInfo feed, RemoteContentResponse remoteResponse)
{
}
protected virtual void OnBeforeReadingRemoteSource(dynamic db, LocalFeedInfo feed)
{
}
public void UpdateFeed(string url)
{
var db = _localFeedsProvider.Db();
dynamic read_view = GetLocalFeedInfoView(db);
LocalFeedInfo feed = read_view.FindByUrl(url)
?? new LocalFeedInfo
{
Url = url,
};
_log.Debug("Updating {0} feed '{1}'", feed.Id == 0 ? "new" : "existing", url);
RemoteContentResponse remoteResponse = null;
try
{
OnBeforeReadingRemoteSource(db, feed);
remoteResponse = _remoteData.ReadRemoteSource(feed);
}
catch (Exception exc)
{
OnReadingRemoteSourceError(db, feed, remoteResponse, exc);
throw;
}
if (remoteResponse.HasNewContent == false)
{
_log.Debug("Feed '{0}' does not have new content", url);
OnNoNewContentInRemoteSource(db, feed, remoteResponse);
return;
}
bool newFeed = feed.LastFetch == null;
string previousContent = feed.Content ?? string.Empty;
feed.Content = GetFeedStringContent(remoteResponse);
feed.LastFetch = ApplicationTime.Current;
feed.Etag = remoteResponse.Etag;
if (newFeed)
{
_log.Info("Inserting new feed '{0}' to local cache", url);
InsertNewFeed(db, feed, url);
}
else
{
_log.Info("Updating feed '{0}' in local cache", url);
UpdateExistingFeed(db, feed, url);
}
OnFeedUpdated(db, feed, remoteResponse);
if (previousContent.GetHashCode() == feed.Content.GetHashCode())
{
_log.Warn("Wasting resources: updating feed '{0}' with the same content it already had!", url);
}
}
protected virtual void UpdateExistingFeed(dynamic db, LocalFeedInfo feed, string url)
{
db.LocalFeedInfo.Update(feed);
}
/// <summary>
/// Inserts new feed into DB if not fetched before
/// </summary>
protected virtual void InsertNewFeed(dynamic db, LocalFeedInfo feed, string url)
{
db.LocalFeedInfo.Insert(feed);
}
protected virtual void OnReadingRemoteSourceError(dynamic db, LocalFeedInfo feed, RemoteContentResponse remoteResponse, Exception exc)
{
}
