static FeedSegment LoadFeed(Uri feedUrl, XElement item)
{
var rf = new FeedSegment(feedUrl: feedUrl);
foreach (XElement xe in item.Elements())
{
Func <FeedSegment, XElement, FeedSegment> action;
if (FeedElements.TryGetValue(xe.Name, out action))
{
rf = action(rf, xe);
}
}
if (String.IsNullOrWhiteSpace(rf.FeedTitle))
{
string title = null;
if (!String.IsNullOrWhiteSpace(rf.FeedDescription))
{
title = rf.FeedDescription;
}
else if (!String.IsNullOrWhiteSpace(rf.WebsiteUrl))
{
title = rf.WebsiteUrl;
}
else if (rf.FeedUrl != null)
{
title = rf.FeedUrl.AbsoluteUri;
}
rf = rf.With(feedTitle: title);
}
return(rf);
}
public FetchResult(
FeedSegment feed,
HttpStatusCode status,
Uri feedUrl,
string etag,
DateTimeOffset?lastModified)
{
Feed = feed;
Status = status;
FeedUrl = feedUrl;
Etag = etag;
LastModified = lastModified;
}
static FeedSegment HandleAtomLink(FeedSegment feed, XElement link)
{
string rel = link.Attribute(XNames.Atom.Rel)?.Value ?? "alternate";
string type = link.Attribute(XNames.Atom.Type)?.Value ?? "text/html";
string href = link.Attribute(XNames.Atom.Href)?.Value;
if (String.Equals(rel, "alternate", StringComparison.OrdinalIgnoreCase) &&
type.StartsWith("text/html", StringComparison.OrdinalIgnoreCase))
{
feed = feed.With(websiteUrl: link.Attribute(XNames.Atom.Href)?.Value);
}
return(feed);
}
static async Task <FetchResult> FetchAsync(Uri uri, string etag, DateTimeOffset?lastModified)
{
Stopwatch loadTimer = Stopwatch.StartNew();
try
{
Log.BeginGetFeed(uri);
HttpResponseMessage response = null;
Uri requestUri = uri;
for (int i = 0; i < 30; i++)
{
response = await Policies.HttpPolicy.ExecuteAsync(_ =>
{
var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
if (etag != null)
{
request.Headers.IfNoneMatch.Add(new EntityTagHeaderValue(etag));
}
request.Headers.IfModifiedSince = lastModified;
return(client.SendAsync(request));
},
new Dictionary <string, object> {
{ "uri", requestUri }
});
if ((response.StatusCode != HttpStatusCode.TemporaryRedirect) &&
(response.StatusCode != HttpStatusCode.Found) &&
(response.StatusCode != HttpStatusCode.SeeOther))
{
break;
}
Uri newUri = response.Headers.Location;
if (!newUri.IsAbsoluteUri)
{
newUri = new Uri(requestUri, newUri);
}
requestUri = newUri;
}
if (response.StatusCode == HttpStatusCode.NotModified)
{
Log.EndGetFeedNotModified(uri, response, loadTimer);
return(new FetchResult(
feed: null,
status: HttpStatusCode.NotModified,
feedUrl: uri,
etag: etag,
lastModified: lastModified));
}
if (response.StatusCode == HttpStatusCode.MovedPermanently)
{
Log.EndGetFeedMovedPermanently(uri, response, loadTimer);
Uri newUri = response.Headers.Location;
if (!newUri.IsAbsoluteUri)
{
newUri = new Uri(requestUri, newUri);
}
return(new FetchResult(
feed: null,
status: HttpStatusCode.MovedPermanently,
feedUrl: newUri,
etag: etag,
lastModified: lastModified));
}
if (!response.IsSuccessStatusCode)
{
//string body = await response.Content.ReadAsStringAsync();
string body = string.Empty;
Log.EndGetFeedFailure(uri, response, body, loadTimer);
return(new FetchResult(
feed: null,
status: response.StatusCode,
feedUrl: uri,
etag: etag,
lastModified: lastModified));
}
Uri responseUri = response.RequestMessage.RequestUri;
await response.Content.LoadIntoBufferAsync();
using (Stream responseStream = await response.Content.ReadAsStreamAsync())
using (var textReader = new StreamReader(responseStream))
using (var reader = XmlReader.Create(textReader)) // TODO: BASE URI?
{
FeedSegment result = null;
XElement element = XElement.Load(reader, LoadOptions.SetBaseUri);
result = FeedParser.ParseFeed(responseUri, element, out FeedFormat format);
if (result != null)
{
Log.EndGetFeed(uri, "ok", format.ToString(), response, result, loadTimer);
}
else
{
Log.UnrecognizableFeed(uri, response, element.ToString(), loadTimer);
}
string newEtag = response.Headers.ETag?.Tag;
DateTimeOffset?newLastModified = response.Content.Headers.LastModified;
return(new FetchResult(
feed: result,
status: HttpStatusCode.OK,
feedUrl: responseUri,
etag: newEtag,
lastModified: newLastModified));
}
}
catch (TaskCanceledException requestException)
{
Log.FeedTimeout(uri, loadTimer, requestException);
return(new FetchResult(
feed: null,
status: 0,
feedUrl: uri,
etag: etag,
lastModified: lastModified));
}
catch (HttpRequestException requestException)
{
Log.NetworkError(uri, requestException, loadTimer);
return(new FetchResult(
feed: null,
status: 0,
feedUrl: uri,
etag: etag,
lastModified: lastModified));
}
catch (WebException requestException)
{
Log.NetworkError(uri, requestException, loadTimer);
return(new FetchResult(
feed: null,
status: 0,
feedUrl: uri,
etag: etag,
lastModified: lastModified));
}
catch (XmlException xmlException)
{
Log.XmlError(uri, xmlException, loadTimer);
return(new FetchResult(
feed: null,
status: 0,
feedUrl: uri,
etag: etag,
lastModified: lastModified));
}
}
