public void does_not_update_feed_that_was_not_modified() { var cached = new LocalFeedInfo() { Url = "http://cached.feed/rss", Content = "cached-content", Etag = "cached-etag", LastFetch = new DateTime(2012, 2, 3) }; _feedsProvider.Db().LocalFeedInfo.Insert(cached); A.CallTo( () => _remoteData.ReadRemoteSource( A <LocalFeedInfo> .That.Matches(x => x.Url == cached.Url && x.Etag == cached.Etag && x.LastFetch == cached.LastFetch) ) ) .Returns(RemoteContentResponse.NotModified(HttpStatusCode.OK)); execute(cached.Url); List <LocalFeedInfo> fromDb = _feedsProvider.Db().LocalFeedInfo.All().ToList <LocalFeedInfo>(); Assert.Equal(1, fromDb.Count); Assert.Equal(cached.Url, fromDb[0].Url); Assert.Equal(cached.Content, fromDb[0].Content); Assert.Equal(cached.Etag, fromDb[0].Etag); Assert.Equal(cached.LastFetch, fromDb[0].LastFetch); }
public void issues_HEAD_request_before_GET_to_see_if_new_content_available_when_configured_to_do_so___workaround_for_servers_that_ignore_http_cache_request_headers_like_etag_and_ifmodifiedsince() { _configurationProvider.ProvideConfig().PrefetchHeadRequest = true; _validUri = "http://www.blogojciec.pl/feed/"; configure_webrequest(); var localFeedInfo = new LocalFeedInfo { Url = _validUri, }; var first_time_response = execute(localFeedInfo); Assert.True(first_time_response.HasNewContent); localFeedInfo.LastFetch = ApplicationTime.Current.AddHours(1); var subsequent_response = execute(localFeedInfo); Assert.False(subsequent_response.HasNewContent); localFeedInfo.LastFetch = ApplicationTime.Current.AddDays(-10); var response_with_changed_content = execute(localFeedInfo); Assert.True(response_with_changed_content.HasNewContent); }
public void updates_cache_with_new_feed_content() { var now = new DateTime(2012, 2, 20); ApplicationTime._replaceCurrentTimeLogic(() => now); var cached = new LocalFeedInfo() { Url = "http://cached.feed/rss", Content = "cached-content", Etag = "cached-etag", LastFetch = new DateTime(2012, 2, 3) }; _feedsProvider.Db().LocalFeedInfo.Insert(cached); var newFeed = prepare_fake_feed(); A.CallTo( () => _remoteData.ReadRemoteSource( A <LocalFeedInfo> .That.Matches(x => x.Url == cached.Url && x.Etag == cached.Etag && x.LastFetch == cached.LastFetch) ) ) .Returns(new RemoteContentResponse(true, "new-etag", newFeed, HttpStatusCode.OK)); execute(cached.Url); List <LocalFeedInfo> fromDb = _feedsProvider.Db().LocalFeedInfo.All().ToList <LocalFeedInfo>(); Assert.Equal(1, fromDb.Count); Assert.Equal(cached.Url, fromDb[0].Url); Assert.Equal(newFeed.GetRssString(), fromDb[0].Content); Assert.Equal("new-etag", fromDb[0].Etag); Assert.Equal(now, fromDb[0].LastFetch); }
private RemoteContentResponse execute(LocalFeedInfo feedInfo) { return(_remoteData.ReadRemoteSource(feedInfo)); }
public RemoteContentResponse ReadRemoteSource(LocalFeedInfo feedInfo) { _log.Debug("Trying to read remote source '{0}' (etag: {1}, last fetch: {2})", feedInfo.Url, feedInfo.Etag, feedInfo.LastFetch); try { if (feedInfo.LastFetch.HasValue && _configProvider.ProvideConfig().PrefetchHeadRequest) { var head_request = _httpRequestFactory.CreateRequest(feedInfo.Url); head_request.Method = "HEAD"; using (var head_response = (HttpWebResponse)head_request.GetResponseEx()) { DateTime last_modified = head_response.LastModified.ToUniversalTime(); DateTime last_fetch = feedInfo.LastFetch.Value.ToUniversalTime(); if (last_modified < last_fetch) { _log.Debug("HEAD request indicates that there is no new content available (last_modified={0}, last_fetch={1}), returning NotModified.", last_modified, last_fetch); return(RemoteContentResponse.NotModified(head_response.StatusCode)); } } } var get_request = _httpRequestFactory.CreateRequest(feedInfo.Url); if (feedInfo.LastFetch != null) { get_request.IfModifiedSince = feedInfo.LastFetch.Value; } if (string.IsNullOrWhiteSpace(feedInfo.Etag) == false) { get_request.Headers[HttpRequestHeader.IfNoneMatch] = feedInfo.Etag; } using (var response = (HttpWebResponse)get_request.GetResponseEx()) { if (response.StatusCode == HttpStatusCode.OK) { string etag = response.Headers[HttpResponseHeader.ETag]; Stream responseStream = response.GetResponseStream(); using (var streamReader = new StreamReader(responseStream)) { string responseText = streamReader.ReadToEnd(); responseText = _contentPreProcessor.PreProcess(responseText); using (var stringReader = new StringReader(responseText)) { using (var reader = XmlReader.Create(stringReader)) { var feed = SyndicationFeed.Load(reader); _log.Info("Returning new content for remote source '{0}' with {1} items", feedInfo.Url, feed.Items.Count()); return(new RemoteContentResponse(true, etag, feed, response.StatusCode)); } } } } else { string message = string.Format("No new content for remote source '{0}', response status is {1}", feedInfo.Url, response.StatusCode); if (response.StatusCode == HttpStatusCode.NotModified) { _log.Debug(message); } else { _log.Warn(message); } return(RemoteContentResponse.NotModified(response.StatusCode)); } } } catch (Exception exc) { _log.ErrorException(string.Format("Failed to read remote source '{0}'", feedInfo.Url), exc); throw new CannotReachRemoteSourceException(feedInfo.Url, exc); } }
protected virtual void OnNoNewContentInRemoteSource(dynamic db, LocalFeedInfo feed, RemoteContentResponse remoteResponse) { }
protected virtual void OnFeedUpdated(dynamic db, LocalFeedInfo feed, RemoteContentResponse remoteResponse) { }
protected virtual void OnBeforeReadingRemoteSource(dynamic db, LocalFeedInfo feed) { }
public void UpdateFeed(string url) { var db = _localFeedsProvider.Db(); dynamic read_view = GetLocalFeedInfoView(db); LocalFeedInfo feed = read_view.FindByUrl(url) ?? new LocalFeedInfo { Url = url, }; _log.Debug("Updating {0} feed '{1}'", feed.Id == 0 ? "new" : "existing", url); RemoteContentResponse remoteResponse = null; try { OnBeforeReadingRemoteSource(db, feed); remoteResponse = _remoteData.ReadRemoteSource(feed); } catch (Exception exc) { OnReadingRemoteSourceError(db, feed, remoteResponse, exc); throw; } if (remoteResponse.HasNewContent == false) { _log.Debug("Feed '{0}' does not have new content", url); OnNoNewContentInRemoteSource(db, feed, remoteResponse); return; } bool newFeed = feed.LastFetch == null; string previousContent = feed.Content ?? string.Empty; feed.Content = GetFeedStringContent(remoteResponse); feed.LastFetch = ApplicationTime.Current; feed.Etag = remoteResponse.Etag; if (newFeed) { _log.Info("Inserting new feed '{0}' to local cache", url); InsertNewFeed(db, feed, url); } else { _log.Info("Updating feed '{0}' in local cache", url); UpdateExistingFeed(db, feed, url); } OnFeedUpdated(db, feed, remoteResponse); if (previousContent.GetHashCode() == feed.Content.GetHashCode()) { _log.Warn("Wasting resources: updating feed '{0}' with the same content it already had!", url); } }
protected virtual void UpdateExistingFeed(dynamic db, LocalFeedInfo feed, string url) { db.LocalFeedInfo.Update(feed); }
/// <summary> /// Inserts new feed into DB if not fetched before /// </summary> protected virtual void InsertNewFeed(dynamic db, LocalFeedInfo feed, string url) { db.LocalFeedInfo.Insert(feed); }
protected virtual void OnReadingRemoteSourceError(dynamic db, LocalFeedInfo feed, RemoteContentResponse remoteResponse, Exception exc) { }