private async Task <FeedItemExt> DownloadFeedContentAsync(FeedCfg feedCfg,
FeedItem feedItem)
{
WebsiteCfg webCfg = null;
try
{
if (feedItem.Link != null)
{
webCfg = WebsitesConfig.FindConfig(feedItem.Link);
feedItem.Link = feedItem.MakeLink(webCfg);
}
//
// Check & update publishing dates
if (feedCfg.UsePubDate)
{
if (feedItem.PublishingDate == null)
{
LogTo.Warning("Date missing, or unknown format for feed {Name}, item title '{Title}', raw date '{PublishingDateString}'",
feedCfg.Name, feedItem.Title, feedItem.PublishingDateString);
return(null);
}
if (feedItem.PublishingDate <= feedCfg.LastPubDate)
{
return(null);
}
}
//
// Check guid
if (feedCfg.UseGuid && feedCfg.EntriesGuid.Contains(feedItem.Id))
{
return(null);
}
//
// Check categories
if (feedCfg.ShouldExcludeCategory(feedItem))
{
return(null);
}
//
// Download content or use inline content
if (feedItem.Link != null)
{
var httpReq = webCfg?.CreateRequest(
feedItem.Link,
string.IsNullOrWhiteSpace(webCfg.Cookie) ? null : new FlurlClient() /*.Configure(s => s.CookiesEnabled = false)*/)
?? feedItem.Link.CreateRequest();
var httpResp = await httpReq.GetStringAsync().ConfigureAwait(false);
if (httpResp != null)
{
feedItem.Content = httpResp;
}
else
{
feedItem.Content = null;
LogTo.Warning("Failed to download content for feed {Name}, item title '{Title}', link '{Link}'.", feedCfg.Name,
feedItem.Title, feedItem.Link);
}
}
else
{
feedItem.Content ??= feedItem.Description;
}
if (string.IsNullOrWhiteSpace(feedItem.Content))
{
return(null);
}
//
// Process content if necessary & push back
feedItem.Content = await webCfg.ProcessContent(feedItem.Content, feedItem.Link).ConfigureAwait(false);
// Add feed item
return(new FeedItemExt(feedItem, webCfg));
}
catch (UriFormatException ex)
{
LogTo.Warning(ex, "Invalid content URI in feed {Name}, item title '{Title}', link '{Link}'.", feedCfg.Name,
feedItem.Title, feedItem.Link);
}
catch (FlurlHttpException ex)
{
LogTo.Warning(ex, "Failed to download content for feed {Name}, item title '{Title}', link '{Link}'.", feedCfg.Name,
feedItem.Title, feedItem.Link);
}
catch (Exception ex)
{
LogTo.Error(ex, "Exception while downloading content for feed {Name}, item title '{Title}', link '{Link}'", feedCfg.Name,
feedItem.Title, feedItem.Link);
}
return(null);
}
private static async Task <FeedItemExt> DownloadFeedContent(FeedCfg feedCfg,
FeedItem feedItem,
SemaphoreSlim throttler,
HttpClient client)
{
try
{
await throttler.WaitAsync();
//
// Check & update publishing dates
if (feedCfg.UsePubDate)
{
if (feedItem.PublishingDate == null)
{
LogTo.Warning(
$"Date missing, or unknown format for feed {feedCfg.Name}, item title '{feedItem.Title}', raw date '{feedItem.PublishingDateString}'");
return(null);
}
if (feedItem.PublishingDate <= feedCfg.LastPubDate)
{
return(null);
}
}
//
// Check guid
if (feedCfg.UseGuid)
{
if (feedCfg.EntriesGuid.Contains(feedItem.Id))
{
return(null);
}
}
//
// Check categories
if (feedCfg.ShouldExcludeCategory(feedItem))
{
return(null);
}
//
// Download content or use inline content
if (feedItem.Link != null)
{
var httpReq = new HttpRequestMessage(HttpMethod.Get,
feedItem.MakeLink(feedCfg));
var httpResp = await client.SendAsync(httpReq);
if (httpResp != null && httpResp.IsSuccessStatusCode)
{
feedItem.Content = await httpResp.Content.ReadAsStringAsync();
}
else
{
feedItem.Content = null;
LogTo.Warning(
$"Failed to download content for feed {feedCfg.Name}, item title '{feedItem.Title}', link '{feedItem.MakeLink(feedCfg)}'. HTTP Status code : {httpResp?.StatusCode}");
}
}
else
{
feedItem.Content = feedItem.Content ?? feedItem.Description;
}
if (string.IsNullOrWhiteSpace(feedItem.Content))
{
return(null);
}
//
// Process content if necessary & push back
if (feedCfg.Filters.Any())
{
feedItem.Content = string.Join(
"\r\n",
feedCfg.Filters
.Select(f => f.Filter(feedItem.Content))
.Where(s => string.IsNullOrWhiteSpace(s) == false)
);
}
if (feedItem.Link != null)
{
feedItem.Content = HtmlUtils.EnsureAbsoluteLinks(feedItem.Content, new Uri(feedItem.MakeLink(feedCfg)));
}
// Add feed item
return(new FeedItemExt(feedItem));
}
catch (Exception ex)
{
LogTo.Error(ex, $"Exception while downloading content for feed {feedCfg.Name}, item title '{feedItem.Title}', link '{feedItem.MakeLink(feedCfg)}'");
}
finally
{
throttler.Release();
}
return(null);
}