static FeedSegment LoadFeed(Uri feedUrl, XElement item) { var rf = new FeedSegment(feedUrl: feedUrl); foreach (XElement xe in item.Elements()) { Func <FeedSegment, XElement, FeedSegment> action; if (FeedElements.TryGetValue(xe.Name, out action)) { rf = action(rf, xe); } } if (String.IsNullOrWhiteSpace(rf.FeedTitle)) { string title = null; if (!String.IsNullOrWhiteSpace(rf.FeedDescription)) { title = rf.FeedDescription; } else if (!String.IsNullOrWhiteSpace(rf.WebsiteUrl)) { title = rf.WebsiteUrl; } else if (rf.FeedUrl != null) { title = rf.FeedUrl.AbsoluteUri; } rf = rf.With(feedTitle: title); } return(rf); }
public FetchResult( FeedSegment feed, HttpStatusCode status, Uri feedUrl, string etag, DateTimeOffset?lastModified) { Feed = feed; Status = status; FeedUrl = feedUrl; Etag = etag; LastModified = lastModified; }
static FeedSegment HandleAtomLink(FeedSegment feed, XElement link) { string rel = link.Attribute(XNames.Atom.Rel)?.Value ?? "alternate"; string type = link.Attribute(XNames.Atom.Type)?.Value ?? "text/html"; string href = link.Attribute(XNames.Atom.Href)?.Value; if (String.Equals(rel, "alternate", StringComparison.OrdinalIgnoreCase) && type.StartsWith("text/html", StringComparison.OrdinalIgnoreCase)) { feed = feed.With(websiteUrl: link.Attribute(XNames.Atom.Href)?.Value); } return(feed); }
static async Task <FetchResult> FetchAsync(Uri uri, string etag, DateTimeOffset?lastModified) { Stopwatch loadTimer = Stopwatch.StartNew(); try { Log.BeginGetFeed(uri); HttpResponseMessage response = null; Uri requestUri = uri; for (int i = 0; i < 30; i++) { response = await Policies.HttpPolicy.ExecuteAsync(_ => { var request = new HttpRequestMessage(HttpMethod.Get, requestUri); if (etag != null) { request.Headers.IfNoneMatch.Add(new EntityTagHeaderValue(etag)); } request.Headers.IfModifiedSince = lastModified; return(client.SendAsync(request)); }, new Dictionary <string, object> { { "uri", requestUri } }); if ((response.StatusCode != HttpStatusCode.TemporaryRedirect) && (response.StatusCode != HttpStatusCode.Found) && (response.StatusCode != HttpStatusCode.SeeOther)) { break; } Uri newUri = response.Headers.Location; if (!newUri.IsAbsoluteUri) { newUri = new Uri(requestUri, newUri); } requestUri = newUri; } if (response.StatusCode == HttpStatusCode.NotModified) { Log.EndGetFeedNotModified(uri, response, loadTimer); return(new FetchResult( feed: null, status: HttpStatusCode.NotModified, feedUrl: uri, etag: etag, lastModified: lastModified)); } if (response.StatusCode == HttpStatusCode.MovedPermanently) { Log.EndGetFeedMovedPermanently(uri, response, loadTimer); Uri newUri = response.Headers.Location; if (!newUri.IsAbsoluteUri) { newUri = new Uri(requestUri, newUri); } return(new FetchResult( feed: null, status: HttpStatusCode.MovedPermanently, feedUrl: newUri, etag: etag, lastModified: lastModified)); } if (!response.IsSuccessStatusCode) { //string body = await response.Content.ReadAsStringAsync(); string body = string.Empty; Log.EndGetFeedFailure(uri, response, body, loadTimer); return(new FetchResult( feed: null, status: response.StatusCode, feedUrl: uri, etag: etag, lastModified: lastModified)); } Uri responseUri = response.RequestMessage.RequestUri; await response.Content.LoadIntoBufferAsync(); using (Stream responseStream = await response.Content.ReadAsStreamAsync()) using (var textReader = new StreamReader(responseStream)) using (var reader = XmlReader.Create(textReader)) // TODO: BASE URI? { FeedSegment result = null; XElement element = XElement.Load(reader, LoadOptions.SetBaseUri); result = FeedParser.ParseFeed(responseUri, element, out FeedFormat format); if (result != null) { Log.EndGetFeed(uri, "ok", format.ToString(), response, result, loadTimer); } else { Log.UnrecognizableFeed(uri, response, element.ToString(), loadTimer); } string newEtag = response.Headers.ETag?.Tag; DateTimeOffset?newLastModified = response.Content.Headers.LastModified; return(new FetchResult( feed: result, status: HttpStatusCode.OK, feedUrl: responseUri, etag: newEtag, lastModified: newLastModified)); } } catch (TaskCanceledException requestException) { Log.FeedTimeout(uri, loadTimer, requestException); return(new FetchResult( feed: null, status: 0, feedUrl: uri, etag: etag, lastModified: lastModified)); } catch (HttpRequestException requestException) { Log.NetworkError(uri, requestException, loadTimer); return(new FetchResult( feed: null, status: 0, feedUrl: uri, etag: etag, lastModified: lastModified)); } catch (WebException requestException) { Log.NetworkError(uri, requestException, loadTimer); return(new FetchResult( feed: null, status: 0, feedUrl: uri, etag: etag, lastModified: lastModified)); } catch (XmlException xmlException) { Log.XmlError(uri, xmlException, loadTimer); return(new FetchResult( feed: null, status: 0, feedUrl: uri, etag: etag, lastModified: lastModified)); } }