Beispiel #1
0
        public static bool ShouldExcludeCategory(this FeedCfg feedCfg, FeedItem feedItem)
        {
            foreach (var catFilter in feedCfg.CategoryFilters)
            {
                switch (catFilter.Mode)
                {
                case Models.FilterMode.Exclude:
                    if (feedItem.Categories?.Contains(catFilter.Category) ?? false)
                    {
                        return(true);
                    }

                    break;

                case Models.FilterMode.Include:
                    if (feedItem.Categories?.Contains(catFilter.Category) ?? false)
                    {
                        return(false);
                    }

                    break;

                default:
                    throw new NotImplementedException($"No such FilterMode {catFilter.Mode}");
                }
            }

            return(false);
        }
Beispiel #2
0
        private async Task <FeedData> DownloadFeedAsync(FeedCfg feedCfg)
        {
            try
            {
                var lastRefresh = DateTime.Now;
                var feed        = await FeedReader.ReadAsync(feedCfg.InterpolateUrl()).ConfigureAwait(false);

                if (feed?.Items == null || feed.Items.Count <= 0)
                {
                    return(null);
                }

                feedCfg.PendingRefreshDate = lastRefresh;
                var feedData = await DownloadFeedContentsAsync(new FeedData(feedCfg, feed)).ConfigureAwait(false);

                if (feedData == null)
                {
                    return(null);
                }

                feedData.NewItems = feedData.NewItems
                                    .OrderByDescending(fi => fi.PublishingDate ?? DateTime.MinValue)
                                    .ToList();

                return(feedData);
            }
            catch (Exception ex)
            {
                LogTo.Warning(ex, "Exception while reading feed {Name}", feedCfg.Name);
            }

            return(null);
        }
Beispiel #3
0
 public static string InterpolateUrl(this FeedCfg feedCfg)
 {
     return(feedCfg.SourceUrl.Interpolate(
                ("now", DateTime.Now),
                ("lastPubDate", feedCfg.LastPubDate),
                ("lastRefreshDate", feedCfg.LastRefreshDate)
                ));
 }
Beispiel #4
0
        private async Task <FeedItemExt> DownloadFeedContentAsync(FeedCfg feedCfg,
                                                                  FeedItem feedItem)
        {
            WebsiteCfg webCfg = null;

            try
            {
                if (feedItem.Link != null)
                {
                    webCfg        = WebsitesConfig.FindConfig(feedItem.Link);
                    feedItem.Link = feedItem.MakeLink(webCfg);
                }

                //
                // Check & update publishing dates

                if (feedCfg.UsePubDate)
                {
                    if (feedItem.PublishingDate == null)
                    {
                        LogTo.Warning("Date missing, or unknown format for feed {Name}, item title '{Title}', raw date '{PublishingDateString}'",
                                      feedCfg.Name, feedItem.Title, feedItem.PublishingDateString);
                        return(null);
                    }

                    if (feedItem.PublishingDate <= feedCfg.LastPubDate)
                    {
                        return(null);
                    }
                }

                //
                // Check guid

                if (feedCfg.UseGuid && feedCfg.EntriesGuid.Contains(feedItem.Id))
                {
                    return(null);
                }

                //
                // Check categories

                if (feedCfg.ShouldExcludeCategory(feedItem))
                {
                    return(null);
                }

                //
                // Download content or use inline content

                if (feedItem.Link != null)
                {
                    var httpReq = webCfg?.CreateRequest(
                        feedItem.Link,
                        string.IsNullOrWhiteSpace(webCfg.Cookie) ? null : new FlurlClient() /*.Configure(s => s.CookiesEnabled = false)*/)
                                  ?? feedItem.Link.CreateRequest();

                    var httpResp = await httpReq.GetStringAsync().ConfigureAwait(false);

                    if (httpResp != null)
                    {
                        feedItem.Content = httpResp;
                    }
                    else
                    {
                        feedItem.Content = null;
                        LogTo.Warning("Failed to download content for feed {Name}, item title '{Title}', link '{Link}'.", feedCfg.Name,
                                      feedItem.Title, feedItem.Link);
                    }
                }

                else
                {
                    feedItem.Content ??= feedItem.Description;
                }

                if (string.IsNullOrWhiteSpace(feedItem.Content))
                {
                    return(null);
                }

                //
                // Process content if necessary & push back

                feedItem.Content = await webCfg.ProcessContent(feedItem.Content, feedItem.Link).ConfigureAwait(false);

                // Add feed item
                return(new FeedItemExt(feedItem, webCfg));
            }
            catch (UriFormatException ex)
            {
                LogTo.Warning(ex, "Invalid content URI in feed {Name}, item title '{Title}', link '{Link}'.", feedCfg.Name,
                              feedItem.Title, feedItem.Link);
            }
            catch (FlurlHttpException ex)
            {
                LogTo.Warning(ex, "Failed to download content for feed {Name}, item title '{Title}', link '{Link}'.", feedCfg.Name,
                              feedItem.Title, feedItem.Link);
            }
            catch (Exception ex)
            {
                LogTo.Error(ex, "Exception while downloading content for feed {Name}, item title '{Title}', link '{Link}'", feedCfg.Name,
                            feedItem.Title, feedItem.Link);
            }

            return(null);
        }
Beispiel #5
0
 public static string MakeLink(this FeedItem feedItem, FeedCfg feedCfg)
 {
     return(feedItem.Link + (feedCfg.LinkParameter ?? string.Empty));
 }
Beispiel #6
0
        private static async Task <FeedItemExt> DownloadFeedContent(FeedCfg feedCfg,
                                                                    FeedItem feedItem,
                                                                    SemaphoreSlim throttler,
                                                                    HttpClient client)
        {
            try
            {
                await throttler.WaitAsync();

                //
                // Check & update publishing dates

                if (feedCfg.UsePubDate)
                {
                    if (feedItem.PublishingDate == null)
                    {
                        LogTo.Warning(
                            $"Date missing, or unknown format for feed {feedCfg.Name}, item title '{feedItem.Title}', raw date '{feedItem.PublishingDateString}'");
                        return(null);
                    }

                    if (feedItem.PublishingDate <= feedCfg.LastPubDate)
                    {
                        return(null);
                    }
                }

                //
                // Check guid

                if (feedCfg.UseGuid)
                {
                    if (feedCfg.EntriesGuid.Contains(feedItem.Id))
                    {
                        return(null);
                    }
                }

                //
                // Check categories

                if (feedCfg.ShouldExcludeCategory(feedItem))
                {
                    return(null);
                }

                //
                // Download content or use inline content

                if (feedItem.Link != null)
                {
                    var httpReq = new HttpRequestMessage(HttpMethod.Get,
                                                         feedItem.MakeLink(feedCfg));
                    var httpResp = await client.SendAsync(httpReq);

                    if (httpResp != null && httpResp.IsSuccessStatusCode)
                    {
                        feedItem.Content = await httpResp.Content.ReadAsStringAsync();
                    }

                    else
                    {
                        feedItem.Content = null;
                        LogTo.Warning(
                            $"Failed to download content for feed {feedCfg.Name}, item title '{feedItem.Title}', link '{feedItem.MakeLink(feedCfg)}'. HTTP Status code : {httpResp?.StatusCode}");
                    }
                }

                else
                {
                    feedItem.Content = feedItem.Content ?? feedItem.Description;
                }

                if (string.IsNullOrWhiteSpace(feedItem.Content))
                {
                    return(null);
                }

                //
                // Process content if necessary & push back

                if (feedCfg.Filters.Any())
                {
                    feedItem.Content = string.Join(
                        "\r\n",
                        feedCfg.Filters
                        .Select(f => f.Filter(feedItem.Content))
                        .Where(s => string.IsNullOrWhiteSpace(s) == false)
                        );
                }

                if (feedItem.Link != null)
                {
                    feedItem.Content = HtmlUtils.EnsureAbsoluteLinks(feedItem.Content, new Uri(feedItem.MakeLink(feedCfg)));
                }

                // Add feed item
                return(new FeedItemExt(feedItem));
            }
            catch (Exception ex)
            {
                LogTo.Error(ex, $"Exception while downloading content for feed {feedCfg.Name}, item title '{feedItem.Title}', link '{feedItem.MakeLink(feedCfg)}'");
            }
            finally
            {
                throttler.Release();
            }

            return(null);
        }
Beispiel #7
0
 /// <inheritdoc />
 public FeedData(FeedCfg feedCfg, Feed feed)
 {
     FeedCfg = feedCfg;
     Feed    = feed;
 }