public async Task TestAllFilesForException_Async()
{
var files = System.IO.Directory.EnumerateFiles("Feeds");
foreach (var file in files)
{
var feed = await FeedReader.ReadFromFileAsync(file).ConfigureAwait(false);
if (feed != null)
{
Assert.IsTrue(!string.IsNullOrEmpty(feed.Link));
TestItunesParsingForException(feed);
}
}
}
public string getFeedURL(string url)
{
var urlsTask = FeedReader.GetFeedUrlsFromUrlAsync(url);
var urls = urlsTask.Result;
string feedUrl = url;
if (urls == null || urls.Count() < 1)
{
return(url);
}
else
{
return(urls.First().Url); //if 1 or 2 urls, then its usually a feed and a comments feed, so take the first per default
}
}
/// <summary>
/// Returns a list of feeds downloaded by the FeedReader.
/// Shows an error if the internet connection isn't working.
/// </summary>
/// <param name="urls">URLs to be downloaded</param>
/// <returns>The list of downloaded feeds</returns>
public Task <List <Feed> > GetFeeds(params string[] urls)
{
var results = new List <Feed>();
foreach (var url in urls)
{
var feedTask = FeedReader.ReadAsync(url);
Feed f = feedTask.Result;
f.Link = url;
results.Add(f);
}
return(Task.Run(() => results));
}
public async Task <Channel> GetChannelWithGamesAsync(string source)
{
try
{
var rawChannel = await FeedReader.ReadAsync(source).ConfigureAwait(false);
var channel = Mapper.Map <Channel>(rawChannel);
channel.LinkRSS = source;
return(channel);
}
catch (Exception e)
{
throw new UriFormatException($"Incorrect data source. {source}", e);
}
}
IEnumerator GetRequest(string uri)
{
var feed = FeedReader.ReadAsync(uri).GetAwaiter().GetResult();
Debug.Log("Feed Title: " + feed.Title);
Debug.Log("Feed Description: " + feed.Description);
Debug.Log("Feed Image: " + feed.ImageUrl);
foreach (var item in feed.Items)
{
string desc = item.Description.Replace(" ", "");
Debug.Log(item.Title + " - " + item.Link + "\n" + desc);
}
yield return(null);
}
public async Task AddFeed(string uri)
{
var result = await FeedReader.ReadAsync(uri);
var newFeed = new Models.Feed
{
Uri = uri,
Title = result.Title,
Added = DateTime.Now,
LastChecked = DateTime.Now
};
await AddFeed(newFeed);
await GetNewItems(newFeed, result.Items);
}
public async Task SkipPreheader_Html_Throws()
{
var input = "<html>";
FeedReader feedReader;
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(input)))
using (var reader = new StreamReader(stream))
{
feedReader = new FeedReader(reader, _logger);
Func <Task> act = async() => await feedReader.SkipPreheader();
await act.Should().ThrowAsync <InvalidPodcastFeedException>();
}
}
private void Consumer()
{
while (true)
{
if (this.queue.Count <= 0)
{
Thread.Sleep(1000);
}
if (!this.queue.TryDequeue(out string feed))
{
continue;
}
try
{
var sf = SyndicationFeed.Load(XmlReader.Create(feed));
using (var connection = this.helper.GetDbConnection())
{
this.helper.ParseArticles(sf, feed.Md5(), connection);
}
NiologManager.CreateLogger().Info()
.Message($"refreshed {feed}")
.Write();
}
catch
{
try
{
var feedEntity = FeedReader.ReadAsync(feed).Result;
using (var connection = this.helper.GetDbConnection())
{
this.helper.ParseArticles(feedEntity, feed.Md5(), connection);
}
NiologManager.CreateLogger().Info()
.Message($"refreshed {feed}")
.Write();
}
catch (Exception e)
{
NiologManager.CreateLogger().Error()
.Message($"error occured when refreshing {feed}")
.Exception(e)
.Write();
}
}
}
}
//Rss
public async void _Rss(string Url)
{
try
{
if (TicketRss.Controls.Count == 0)
{
var feed = await FeedReader.ReadAsync(Url);
foreach (var item in feed.Items)
{
_Add(item.Link, item.Title, false);
}
;
}
else
{
var update = await FeedReader.ReadAsync(Url);
foreach (var item in update.Items)
{
if (item.Link == TicketRss.Controls[0].Name)
{
break;
}
else
{
_Add(item.Link, item.Title, true);
break;
}
}
;
}
pathrss.Item2 = Url;
}
catch
{
pathrss.Item2 = null;
TicketRss.Controls.Clear();
}
finally
{
if (string.IsNullOrEmpty(pathrss.Item2))
{
pathrss.Item2 = null;
TicketRss.Controls.Clear();
}
}
}
public static List <NewfeedRss> GetTinMoiNhat()
{
List <NewfeedRss> listResult = new List <NewfeedRss>();
var feed = FeedReader.Read("https://vnexpress.net/rss/tin-moi-nhat.rss");
foreach (var item in feed.Items)
{
listResult.Add(new NewfeedRss
{
Title = item.Title,
Link = item.Link,
PublishingDate = item.PublishingDate,
});
}
return(listResult.Take(10).ToList());
}
public async Task <FeedName[]> Read()
{
var result = await FeedReader.ReadAsync("https://www.delfi.lt/rss/feeds/index.xml").ConfigureAwait(false);
var feeds = new List <FeedName>();
foreach (var item in result.Items)
{
var feed = new FeedName();
feed.Category = Path.GetFileNameWithoutExtension(item.Link);
feed.Url = item.Link;
feeds.Add(feed);
}
return(feeds.ToArray());
}
private string GetInfo(string url)
{
try
{
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls | SecurityProtocolType.Ssl3;
mWebClient wc = new mWebClient();
var feedxml = wc.DownloadString(url);
var reader = FeedReader.ReadFromString(feedxml);
return(reader.Title);
}
catch (Exception ex)
{
MessageBox.Show(ex.Message, "Error");
return("");
}
}
public async Task ReadDocumentMetadata_InvalidChannel_Throws()
{
var input = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss version=\"2.0\"\r\n\txmlns:content=\"http://purl.org/rss/1.0/modules/content/\"\r\n\txmlns:wfw=\"http://wellformedweb.org/CommentAPI/\"\r\n\txmlns:dc=\"http://purl.org/dc/elements/1.1/\"\r\n\txmlns:atom=\"http://www.w3.org/2005/Atom\"\r\n\txmlns:sy=\"http://purl.org/rss/1.0/modules/syndication/\"\r\n\txmlns:slash=\"http://purl.org/rss/1.0/modules/slash/\"\r\n\txmlns:georss=\"http://www.georss.org/georss\" xmlns:geo=\"http://www.w3.org/2003/01/geo/wgs84_pos#\" \r\n\txmlns:itunes=\"http://www.itunes.com/dtds/podcast-1.0.dtd\"\r\nxmlns:media=\"http://search.yahoo.com/mrss/\"\r\n\t>\r\n\r\n<flannel>";
FeedReader feedReader;
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(input)))
using (var reader = new StreamReader(stream))
{
feedReader = new FeedReader(reader, _logger);
await feedReader.SkipPreheader();
Action act = () => feedReader.ReadDocumentHeader();
act.Should().Throw <InvalidPodcastFeedException>();
}
}
public async Task GetShowAsync_AmpersandAfterCDataContent_DoesNotThrow()
{
var input = File.ReadAllText($@"{TestDataRoot}Invalid\samplefeed_ampersandaftercdatacontent.xml");
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(input)))
using (var reader = new StreamReader(stream))
{
var feedReader = new FeedReader(reader, _logger);
await feedReader.SkipPreheader();
feedReader.ReadDocumentHeader();
Func <Task> act = async() => await feedReader.GetShowXmlAsync();
await act.Should().NotThrowAsync <InvalidPodcastFeedException>();
}
}
public MainWindow()
{
CategoryReader categoryReader = new CategoryReader();
categoryList = categoryReader.Read(Constants.CategoriesFileName);
FeedReader feedReader = new FeedReader();
feedList = feedReader.Read(Constants.FeedsFileName);
PageFeed = new wpfFeed(feedList);
PageAdd = new wpfAddFeed(feedList, categoryList);
PageSettings = new wpfSettings();
PageRemove = new wpfRemoveFeed();
InitializeComponent();
MainFrame.Navigate(PageFeed);
}
public void SetUp()
{
_bus = new InMemoryBus("test");
_consumer = new TestHandler <Message>();
_bus.Subscribe(_consumer);
_subscriptionDispatcher =
new PublishSubscribeDispatcher
<ReaderSubscriptionManagement.Subscribe,
ReaderSubscriptionManagement.ReaderSubscriptionManagementMessage,
EventReaderSubscriptionMessage>(_bus, v => v.SubscriptionId, v => v.SubscriptionId);
_testQueryDefinition = GivenQuerySource();
_feedReader = new FeedReader(
_subscriptionDispatcher, SystemAccount.Principal, _testQueryDefinition, GivenFromPosition(), 10,
Guid.NewGuid(), new PublishEnvelope(_bus), new RealTimeProvider());
Given();
When();
}
public async Task <List <Feed> > GetAllFeeds()
{
var list = new List <Feed>();
string[] feedUrls =
{
"https://news.ycombinator.com/rss",
"https://www.reddit.com/r/ProgrammerHumor/.rss"
};
foreach (var feedUrl in feedUrls)
{
list.Add(await FeedReader.ReadAsync(feedUrl));
}
return(list);
}
static int Main(string[] args)
{
return(Task.Run(async() =>
{
try
{
var file = new FileInfo(args[0]);
if (file.Exists)
{
var json = await File.ReadAllTextAsync(file.FullName);
var config = JsonConvert.DeserializeObject <Config>(json);
if (config != null)
{
var keyStore = KeyStore.Restore <PublicSignedKeyStore>(config.account);
if (await keyStore.DecryptKeyAsync(config.password, true))
{
var client = new HeleusClient(new Uri(config.nodeurl));
if (await client.SetServiceAccount(keyStore, config.password, true))
{
// test
await FeedReader.ReadAsync(config.feedurl);
var processor = new FeedProcessor(config.feedurl, client);
while (true)
{
await processor.CheckFeed();
await Task.Delay(TimeSpan.FromMinutes(1));
}
}
}
}
}
else
{
Console.WriteLine($"File does not exist {file.FullName}.");
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
return 0;
}).Result);
}
public async Task TestRss20ParseSwedishFeedWithIso8859_1_Async()
{
var feed = (Rss20Feed)(await FeedReader.ReadFromFileAsync("Feeds/Rss20ISO88591Intranet30.xml").ConfigureAwait(false)).SpecificFeed;
Eq("intranet30", feed.Title);
Eq("http://www.retriever-info.com", feed.Link);
Eq("RSS 2.0 News feed from Retriever Norge AS", feed.Description);
var item = (Rss20FeedItem)feed.Items.First();
Eq("SVART MÅNAD - DÖDSOLYCKA i Vetlanda", item.Title);
Eq("https://www.retriever-info.com/go/?a=30338&d=00201120180819281555686&p=200108&s=2011&sa=2016177&u=http%3A%2F%2Fwww.hoglandsnytt.se%2Fsvart-manad-dodsolycka-i-vetlanda%2F&x=33d88e677ce6481d9882de22c76e4234", item.Link);
Eq("Under juli 2018 omkom 39 personer och 1 521 skadades i vägtrafiken. Det visar de preliminära uppgifter som inkommit till Transportstyrelsen fram till den 15 augusti 2018. Det är åtta fler omkomna jämfört med juli månad 2017.", item.Description);
Eq("Sun, 19 Aug 2018 07:14:00 GMT", item.PublishingDateString);
Eq("00201120180819281555686", item.Guid);
Eq("Höglandsnytt", item.Author);
}
public async Task <(FeedItemModel, string)> FetchLatestPost(string url)
{
try
{
var feed = await FeedReader.ReadAsync(url);
var feedItem = feed.Items.FirstOrDefault();
var mapper = ModelMapper.Init();
var feedItemModel = mapper.Map <FeedItem, FeedItemModel>(feedItem);
return(feedItemModel, string.Empty);
}
catch (Exception exp)
{
return(null, exp.Message);
}
}
public async Task <IDictionary <PostModel, CategoryModel> > FeedItems(ChannelModel channel)
{
var items = new Dictionary <PostModel, CategoryModel>();
var readerTask = await FeedReader.ReadAsync(channel.Url);
//readerTask.ConfigureAwait(false);
foreach (var item in readerTask.Items)
{
var image = item.SpecificItem.Element.Descendants().ToList();
var channelItem = new PostModel
{
Channel = channel,
ChannelId = channel.Id,
Title = item.Title,
CreatedAt = item.PublishingDate ?? DateTime.Now,
IsSeen = false,
IsNew = true,
PostUrl = item.Link,
Body = item.Description,
ImageUrl = image.FirstOrDefault(x => x.Name.LocalName.Contains("enclosure")) != null
? item.SpecificItem.Element.Descendants()
.First(x => x.Name.LocalName == "enclosure").Attribute("url")
?.Value
: channel.Image,
CategoryName = item.Categories.FirstOrDefault()
};
var category = _uow.GetRepository <Category>().All()
.FirstOrDefault(x => x.Name == item.Categories.FirstOrDefault() && x.ChannelId == channel.Id);
var categoryModel = new CategoryModel();
if (category == null)
{
categoryModel.Name = item.Categories.FirstOrDefault();
categoryModel.ChannelId = channel.Id;
categoryModel.Channel = channel;
}
items.Add(channelItem, categoryModel);
}
return(items);
}
public async Task CheckFeed()
{
var feed = await FeedReader.ReadAsync(_url);
if (_history.Count == 0)
{
foreach (var item in feed.Items)
{
_history.Add(item.Link);
}
if (_history.Count > 0)
{
_history.RemoveAt(0);
}
}
var newItems = new List <FeedItem>();
foreach (var item in feed.Items)
{
var link = item.Link;
if (!_history.Contains(link))
{
_history.Insert(newItems.Count, link);
newItems.Insert(0, item);
}
}
while (_history.Count > 250)
{
_history.RemoveAt(_history.Count - 1);
}
var tasks = new List <Task>();
foreach (var newItem in newItems)
{
tasks.Add(SendMessage(newItem));
}
await Task.WhenAll(tasks);
await _client.CloseConnection(DisconnectReasons.Graceful);
}
private async Task <BlogFeed> readRss(Blog blog)
{
if (blog.RSS == null || blog.IsActive == false)
{
return(null);
}
var feed = await FeedReader.ReadAsync(blog.RSS);
if (feed == null)
{
return(null);
}
// string result = "";
// result += "Feed Title: " + feed.Title + "\n";
// result += "Feed Description: " + feed.Description + "\n";
// result += "Feed Image: " + feed.ImageUrl + "\n";
if (feed.Items.Count > 0)
{
var latestFeed = await _repo.GetLatestBlogFeed(blog);
var firstItem = feed.Items.First();
if (latestFeed == null || latestFeed.PublishingDate < firstItem.PublishingDate)
{
string firstImageUrl = "";
if (firstItem.Content != null)
{
firstImageUrl = Regex.Match(firstItem.Content, "<img.+?src=[\"'](.+?)[\"'].*?>", RegexOptions.IgnoreCase).Groups[1].Value;
}
var blogFeed = new BlogFeed()
{
BlogId = blog.Id,
Blog = blog,
Title = firstItem.Title,
ImageUrl = string.IsNullOrEmpty(firstImageUrl) ? null : firstImageUrl,
PublishingDate = firstItem.PublishingDate == null ? DateTime.Now : firstItem.PublishingDate,
Url = firstItem.Link
};
return(blogFeed);
}
}
return(null);
}
protected async override void OnAppearing()
{
base.OnAppearing();
var current = Connectivity.NetworkAccess;
if (current == NetworkAccess.Internet)
{
var rssFeeds = new Feed();
try
{
rssFeeds = await FeedReader.ReadAsync($"https://www.youtube.com/feeds/videos.xml?channel_id={Xamarin.Forms.Application.Current.Resources["ChannelId"].ToString()}");
}
catch (Exception ex)
{
Console.WriteLine(ex);
_feeds.Add(new RSSFeedObject()
{
Title = "Test", Date = "January 2099", Link = "www.example.com"
});
PopulateList();
return;
}
this.LoadFeeds(rssFeeds);
this.PopulateList();
}
else
{
var optionSelected = await Xamarin.Forms.Application.Current.MainPage.DisplayActionSheet("Error de red", "Recargar!", "Cancelar!", "Señor usuario si desea continuar habillite la conexion a internet y seleccione la acción Recargar!");
if (optionSelected == "Recargar!")
{
Xamarin.Forms.Application.Current.MainPage = new Xamarin.Forms.NavigationPage(new StreamPage())
{
BarTextColor = Color.FromRgb(255, 255, 255),
BackgroundColor = Color.FromRgb(1, 124, 194),
};
}
else if (optionSelected == "Cancelar!" || string.IsNullOrEmpty(optionSelected))
{
return;
}
}
}
/// <summary>
/// Feed Reader
/// https://github.com/codehollow/FeedReader/blob/master/FeedReader.ConsoleSample/Program.cs
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public static async Task <Feed> ReadAtomFeed(string url)
{
try
{
var reader = await FeedReader.ReadAsync(url);
foreach (var item in reader.Items)
{
Console.WriteLine(item.Title + " - " + item.Link);
}
return(reader);
}
catch (Exception ex)
{
Console.WriteLine($"An error occurred: {ex.InnerException.Message}{Environment.NewLine}{ex.InnerException.ToString()}");
return(null);
}
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("blog post info retrieved.");
var blogPostFeed = await FeedReader.ReadAsync(Constants.BlogFeedURL);
var randomIndex = new Random().Next(blogPostFeed.Items.Count) - 1;
var blgFeedItem = new AutoFeedItem()
{
Title = blogPostFeed.Items.ToArray()[randomIndex].Title,
Link = blogPostFeed.Items.ToArray()[randomIndex].Link,
};
log.LogInformation("blog post info info returned.");
return(new OkObjectResult(blgFeedItem));
}
private static IEnumerable <Task <Feed> > ProcessHtmlFeedLinks(IEnumerable <HtmlFeedLink> htmlFeedLinks)
{
var feedTasks = new List <Task <Feed> >();
foreach (var htmlFeedLink in htmlFeedLinks)
{
try
{
feedTasks.Add(FeedReader.ReadAsync(htmlFeedLink.Url));
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
return(feedTasks);
}
// public static RSSList GetRssFeed()
public static List <RSS> GetRssFeed(String _blogURL)
{
List <RSS> RSSitems = new List <RSS>();
RSSList myRSS = new RSSList();
var reader = new FeedReader();
var items = reader.RetrieveFeed(_blogURL); //("http://www.nytimes.com/services/xml/rss/nyt/International.xml");
foreach (var i in items)
{
RSSitems.Add(new Models.RSS(i.Uri.ToString(), i.Date.ToString(), i.Title, i.Summary));
}
return(RSSitems);
}
public void TestAtomYouTubeInvestmentPunk()
{
var feed = (AtomFeed)FeedReader.ReadFromFile("Feeds/AtomYoutubeInvestmentPunk.xml").SpecificFeed;
Eq("http://www.youtube.com/feeds/videos.xml?channel_id=UCmEN5ZnsHUXIxgpLitRTmWw", feed.Links.First().Href);
Eq("yt:channel:UCmEN5ZnsHUXIxgpLitRTmWw", feed.Id);
Eq("Investment Punk Academy by Gerald Hörhan", feed.Title);
Eq("http://www.youtube.com/channel/UCmEN5ZnsHUXIxgpLitRTmWw", feed.Links.ElementAt(1).Href);
Eq("Investment Punk Academy by Gerald Hörhan", feed.Author.Name);
Eq("http://www.youtube.com/channel/UCmEN5ZnsHUXIxgpLitRTmWw", feed.Author.Uri);
var item = (AtomFeedItem)feed.Items.First();
Eq("yt:video:AFA8ZtMwrvc", item.Id);
Eq("Zukunft von Vertretern I Kernfusion I Musikgeschäft #ASKTHEPUNK 71", item.Title);
Eq("alternate", item.Links.First().Relation);
Eq("2017-01-23T18:14:49+00:00", item.UpdatedDateString);
Eq("2017-01-20T16:00:00+00:00", item.PublishedDateString);
}
public async Task GetNextEpisodeAsync_OneEpisode_Returns()
{
var input = File.ReadAllText($@"{TestDataRoot}Invalid\samplefeed_oneepisodecontent.xml");
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(input)))
using (var reader = new StreamReader(stream))
{
var feedReader = new FeedReader(reader, _logger);
await feedReader.SkipPreheader();
feedReader.ReadDocumentHeader();
await feedReader.GetShowXmlAsync();
var xml = await feedReader.GetNextEpisodeXmlAsync();
xml.Should().NotBeNull();
}
}
public async Task GetShowAsync_TooLongContent_Throws()
{
var input = File.ReadAllText($@"{TestDataRoot}Invalid\samplefeed_toolongcontent.xml");
FeedReader feedReader;
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(input)))
using (var reader = new StreamReader(stream))
{
feedReader = new FeedReader(reader, _logger);
await feedReader.SkipPreheader();
feedReader.ReadDocumentHeader();
Func <Task> act = async() => await feedReader.GetShowXmlAsync();
await act.Should().ThrowAsync <InvalidPodcastFeedException>();
}
}
public void SetUp()
{
_bus = new InMemoryBus("test");
_consumer = new TestHandler<Message>();
_bus.Subscribe(_consumer);
_subscriptionDispatcher =
new PublishSubscribeDispatcher
<ReaderSubscriptionManagement.Subscribe,
ReaderSubscriptionManagement.ReaderSubscriptionManagementMessage,
EventReaderSubscriptionMessage>(_bus, v => v.SubscriptionId, v => v.SubscriptionId);
_testQueryDefinition = GivenQuerySource();
_feedReader = new FeedReader(
_subscriptionDispatcher, SystemAccount.Principal, _testQueryDefinition, GivenFromPosition(), 10,
Guid.NewGuid(), new PublishEnvelope(_bus), new RealTimeProvider());
Given();
When();
}
