public void GetFeed(RssFile file) { var rssFormatter = RssParser.GetFeed(file.AtomLink); file.LastUpdateDate = rssFormatter.LastUpdatedTime; file.Description = rssFormatter.Description.Text; file.Language = rssFormatter.Language; file.Title = rssFormatter.Title.Text; foreach (SyndicationItem item in rssFormatter.Items) { RssItem rssItem = new RssItem { Title = item.Title.Text, Description = item.Summary.Text, PublishDate = item.PublishDate.DateTime, RssFileId = file.Id, Link = item.Id }; _rssItemRepository.AddRssItem(rssItem); } ListenServiceClient service = new ListenServiceClient(); service.FireDatabaseEvents(); }
public void Parse_WithChannelData_CorrectObjectReturns() { var fx = new Fixture(); var title = fx.Create <string>(); var link = fx.Create <Uri>(); var description = fx.Create <string>(); var lastBuildDateString = "Fri, 16 Mar 2018 20:20:20 +0000"; var lastBuildDate = new DateTimeOffset(2018, 3, 16, 20, 20, 20, TimeSpan.FromHours(0)); var rssXml = new RssXmlBuilder() .WithTitle(title) .WithLink(link.ToString()) .WithDescription(description) .WithLastBuildDate(lastBuildDateString) .Build(); var parser = new RssParser(); var rssFeed = parser.Parse(rssXml); Assert.Equal(title, rssFeed.Title); Assert.Equal(link, rssFeed.Link); Assert.Equal(description, rssFeed.Description); Assert.Equal(lastBuildDate, rssFeed.LastBuildDate); }
protected override void Run() { discoveredFeed = new NewsFeed(); try { //feedInfo = feedHandler.GetFeedDetails(this.feedUrl, this.credentials); using (Stream mem = SyncWebRequest.GetResponseStream(this.feedUrl, this.credentials, RssBanditApplication.UserAgent, this.Proxy)) { NewsFeed f = new NewsFeed(); f.link = feedUrl; if (RssParser.CanProcessUrl(feedUrl)) { feedInfo = RssParser.GetItemsForFeed(f, mem, false); if (feedInfo.ItemsList != null && feedInfo.ItemsList.Count > 0) f.containsNewMessages = true; } } } catch (ThreadAbortException) { // eat up } catch (Exception e) { // fatal errors p_operationException = e; } finally { WorkDone.Set(); } }
public static IEnumerable <RssSchema> ParseRSS(string feedContent) { RssParser parser = new RssParser(); IEnumerable <RssSchema> rss = parser.Parse(feedContent); return(rss); }
/// <summary> /// Initializes a new instance of the <see cref="FeedSource"/> class. /// </summary> /// <param name="configuration">The configuration.</param> /// <param name="location">The feedlist URL</param> internal BanditFeedSource(INewsComponentsConfiguration configuration, SubscriptionLocation location) { p_configuration = configuration; if (p_configuration == null) { p_configuration = DefaultConfiguration; } this.location = location; // check for programmers error in configuration: ValidateAndThrow(Configuration); LoadFeedlistSchema(); rssParser = new RssParser(this); if (!String.IsNullOrEmpty(EnclosureFolder)) { enclosureDownloader = new BackgroundDownloadManager(this); enclosureDownloader.DownloadCompleted += OnEnclosureDownloadComplete; } AsyncWebRequest = new AsyncWebRequest(); AsyncWebRequest.OnAllRequestsComplete += OnAllRequestsComplete; }
private async Task <List <Article> > ScrapeArticles() { var articles = new List <Article>(); var feed = await httpHelper.Get(FeedUrl); if (string.IsNullOrEmpty(feed)) { throw new NullReferenceException($"Could not get feed from url {FeedUrl}"); } var parser = new RssParser(); var rss = parser.Parse(feed); var newestArticlesSchemas = rss.OrderByDescending(x => x.PublishDate).Take(10); foreach (var schema in newestArticlesSchemas) { if (schema.Categories.Contains("Vijesti")) { articles.Add(new Article { Source = NewsSource.IndexHr, Guid = GetGuidFromUrl(schema.InternalID), Title = schema.Title, Image = schema.ImageUrl, Text = await GetArticleText(schema.InternalID), Summary = schema.Summary, Keywords = keywordHelper.GetKeywordsFromTitle(schema.Title), SourceUrl = schema.InternalID }); } } return(articles); }
public async Task <List <Article> > GetArticles(string endpoint) { string feed = null; using (var client = new HttpClient()) { var feedResponse = await client.GetAsync(endpoint); if (!feedResponse.IsSuccessStatusCode) { throw new Exception("Feed unavailable"); } feed = await feedResponse.Content.ReadAsStringAsync(); } var parser = new RssParser(); var rss = parser.Parse(feed); var articles = new List <Article>(); foreach (var element in rss) { var article = new Article(element.Title, element.Summary, element.ImageUrl, element.PublishDate); articles.Add(article); } return(articles); }
public void GetItemsForFeedWithRssRdfv091() { using (var stream = Resources.Resource.GetResourceAsStream("TestFeeds.rss_rdf_0_91.xml")) { NewsFeed f = new NewsFeed(); f.link = TEST_BASE_URL + "rss_rdf_0_91.xml"; var feedInfo = RssParser.GetItemsForFeed(f, stream, false, true); Assert.IsNotNull(feedInfo); Assert.IsNotNull(feedInfo.ItemsList); Assert.AreEqual(40, feedInfo.ItemsList.Count); Assert.AreEqual(new DateTime(2013, 9, 12, 12, 0, 0, DateTimeKind.Utc), feedInfo.ItemsList[39].Date); string requestUrl = null; var xElem = RssHelper.GetOptionalElement(feedInfo.OptionalElements, "image"); if (xElem != null) { var urlNode = xElem.SelectSingleNode("url"); if (urlNode != null) { requestUrl = urlNode.InnerText; } } Assert.AreEqual("http://www.golem.de/staticrl/images/golem-rss.png", requestUrl); } }
public async Task <List <RssParser.FeedItem> > GetItemsAsync(string feed) { RssParser rssParser = new RssParser(); switch (feed.ToLower()) { case "shortage": case "shortages": return(await rssParser.GetListAsync(shortageURL)); case "dc": case "discontinued": return(await rssParser.GetListAsync(discontinuedURL)); case "resolved": return(await rssParser.GetListAsync(resolvedURL)); case "unavailable": case "notavailable": return(await rssParser.GetListAsync(notavailableURL)); default: return(await rssParser.GetListAsync(shortageURL)); } }
public static async void ParseRSS() { string feed = null; using (var client = new HttpClient()) { try { feed = await client.GetStringAsync("https://news.yandex.ru/internet.rss"); } catch { } } if (feed != null) { var parser = new RssParser(); var rss = parser.Parse(feed); foreach (var element in rss) { Console.WriteLine($"Заголовок: {element.Title}"); Console.WriteLine($"Текст: {element.Summary}"); Console.WriteLine($"Ссылка: {element.FeedUrl}"); } } }
public void TestImportFeedlist() { RssParser handler = new RssParser(APP_NAME); handler.LoadFeedlist(BASE_URL + "FeedList03Feeds.xml", null); Assertion.Assert("Feeds should be valid!", handler.FeedsListOK); Assertion.AssertEquals("FeedList03Feeds.xml should contain 3 feeds. Hence the name.", 3, handler.FeedsTable.Count); //Now import a feed list. using (FileStream stream = File.OpenRead(Path.Combine(WEBROOT_PATH, @"RssParserTestFiles\FeedList04Feeds.xml"))) { handler.ImportFeedlist(stream); stream.Close(); } Assertion.AssertEquals("3 + 4 = 7. 7 Feeds expected.", 7, handler.FeedsTable.Count); Assertion.AssertEquals("MSDN: Visual C#", handler.FeedsTable["http://msdn.microsoft.com/vcsharp/rss.xml"].title); Assertion.AssertEquals("Development", handler.FeedsTable["http://msdn.microsoft.com/vcsharp/rss.xml"].category); Assertion.AssertEquals("ASP.NET Forums: Architecture", handler.FeedsTable["http://www.asp.net/Forums/rss.aspx?forumid=16"].title); Assertion.AssertEquals("Forums", handler.FeedsTable["http://www.asp.net/Forums/rss.aspx?forumid=16"].category); Assertion.AssertEquals("Slashdot", handler.FeedsTable["http://slashdot.org/slashdot.rss"].title); Assertion.AssertEquals("News Technology", handler.FeedsTable["http://slashdot.org/slashdot.rss"].category); Assertion.AssertEquals("Torsten's .NET Blog", handler.FeedsTable["http://www.rendelmann.info/blog/SyndicationService.asmx/GetRss"].title); Assertion.AssertEquals("Blogs", handler.FeedsTable["http://www.rendelmann.info/blog/SyndicationService.asmx/GetRss"].category); Assertion.AssertEquals("you've been HAACKED", handler.FeedsTable["http://haacked.com/Rss.aspx"].title); Assertion.AssertEquals("Blogs", handler.FeedsTable["http://haacked.com/Rss.aspx"].category); Assertion.AssertEquals("kuro5hin.org", handler.FeedsTable["http://www.kuro5hin.org/backend.rdf"].title); Assertion.AssertEquals("News Technology", handler.FeedsTable["http://www.kuro5hin.org/backend.rdf"].category); Assertion.AssertEquals("Dare Obasanjo aka Carnage4Life", handler.FeedsTable["http://www.25hoursaday.com/weblog/SyndicationService.asmx/GetRss"].title); Assertion.AssertEquals("Blogs Microsoft", handler.FeedsTable["http://www.25hoursaday.com/weblog/SyndicationService.asmx/GetRss"].category); }
public void TestImportFeedListWithDuplicate() { //Start by loading 3 feeds. RssParser handler = new RssParser(APP_NAME); handler.LoadFeedlist(BASE_URL + "FeedList03Feeds.xml", null); Assertion.Assert("Feeds should be valid!", handler.FeedsListOK); Assertion.AssertEquals("FeedList03Feeds.xml should contain 3 feeds. Hence the name.", 3, handler.FeedsTable.Count); // Now import a feed list that contains 1 new feed // and one that already exists. using (FileStream stream = File.OpenRead(Path.Combine(WEBROOT_PATH, @"RssParserTestFiles\FeedListWithDuplicateFrom03.xml"))) { handler.ImportFeedlist(stream); stream.Close(); } Assertion.AssertEquals("3 + 1 = 4. 4 Feeds expected because one is a duplicate!", 4, handler.FeedsTable.Count); Assertion.AssertEquals("MSDN: Visual C#", handler.FeedsTable["http://msdn.microsoft.com/vcsharp/rss.xml"].title); Assertion.AssertEquals("Development", handler.FeedsTable["http://msdn.microsoft.com/vcsharp/rss.xml"].category); Assertion.AssertEquals("ASP.NET Forums: Architecture", handler.FeedsTable["http://www.asp.net/Forums/rss.aspx?forumid=16"].title); Assertion.AssertEquals("Forums", handler.FeedsTable["http://www.asp.net/Forums/rss.aspx?forumid=16"].category); Assertion.AssertEquals("Slashdot", handler.FeedsTable["http://slashdot.org/slashdot.rss"].title); Assertion.AssertEquals("News Technology", handler.FeedsTable["http://slashdot.org/slashdot.rss"].category); Assertion.AssertEquals("Channel 9", handler.FeedsTable["http://channel9.msdn.com/rss.aspx"].title); Assertion.AssertEquals("Microsoft", handler.FeedsTable["http://channel9.msdn.com/rss.aspx"].category); }
private async Task <List <Article> > ScrapeArticles() { var articles = new List <Article>(); var feed = await httpHelper.Get(FeedUrl); if (string.IsNullOrEmpty(feed)) { throw new Exception($"Could not get feed from url {FeedUrl}"); } var parser = new RssParser(); var rss = parser.Parse(feed); var newestArticlesSchemas = rss.OrderByDescending(x => x.PublishDate).Take(10); foreach (var schema in newestArticlesSchemas) { var document = await httpHelper.GetDocumentFromUrl(schema.InternalID); articles.Add(new Article { Source = NewsSource.PriznajemHr, Guid = GetGuidFromUrl(schema.InternalID), Title = schema.Title, Image = GetArticleImage(document), Text = await httpHelper.GetArticleText(schema.Content), Summary = await httpHelper.GetFirstParagraph(schema.Content), Keywords = keywordHelper.GetKeywordsFromTitle(schema.Title), SourceUrl = schema.InternalID }); } return(articles); }
public void Parse_WithItem_RssFeedItemWithCorrectDataReturns() { //Given var fx = new Fixture(); var title = fx.Create <string>(); var link = fx.Create <Uri>(); var description = fx.Create <string>(); var pubDateString = "Sun, 18 Mar 2018 16:48:17 +0000"; var pubDate = new DateTimeOffset(2018, 3, 18, 16, 48, 17, TimeSpan.FromHours(0)); var item = new RssItemXmlBuilder() .WithTitle(title) .WithDescription(description) .WithLink(link.ToString()) .WithPubDate(pubDateString) .Build(); var rssXml = new RssXmlBuilder() .WithItems(new [] { item }) .Build(); //When var parser = new RssParser(); var rssFeed = parser.Parse(rssXml); //Then Assert.Equal(1, rssFeed.Items?.Length); Assert.Equal(title, rssFeed.Items?[0].Title); Assert.Equal(description, rssFeed.Items?[0].Description); Assert.Equal(pubDate, rssFeed.Items?[0].PubDate); Assert.Equal(link, rssFeed.Items?[0].Link); }
static async Task Main(string[] args) { Console.WriteLine("Haetaan uutisotsikkoja YLE:ltä..."); HttpClient client = new HttpClient(); const string YleRSSUrl = "https://feeds.yle.fi/uutiset/v1/majorHeadlines/YLE_UUTISET.rss"; string rss = await client.GetStringAsync(YleRSSUrl); RssParser parser = new RssParser(); IEnumerable <RssSchema> feed = parser.Parse(rss); int lkm = 0; foreach (RssSchema element in feed) { Console.WriteLine($"Title: {element.Title}"); // Console.WriteLine($"Summary: {element.Summary}"); lkm++; if (lkm > 10) { break; } } }
public async Task <Publication> Run( [ActivityTrigger] PublicationConfiguration publicationConfiguration, ILogger logger) { logger.LogInformation($"Started {nameof(GetLatestPublicationFromWebActivity)} for { publicationConfiguration.PublicationSourceOwner } { publicationConfiguration.PublicationSourceName}."); Publication publication = new NullPublication(publicationConfiguration.PublicationSourceName); using (var xmlReader = XmlReader.Create(publicationConfiguration.PublicationSourceUrl, new XmlReaderSettings() { Async = true })) { var parser = new RssParser(); var reader = new RssFeedReader(xmlReader, parser); while (await reader.Read()) { if (reader.ElementType == SyndicationElementType.Item) { ISyndicationItem item = await reader.ReadItem(); publication = MapToPublication(publicationConfiguration, item); break; } } } logger.LogInformation($"Found publication on web for {publicationConfiguration.PublicationSourceName}: " + $"ID: {publication.Id}," + $"Date: {publication.PublicationDate:F}."); return(publication); }
public RssClient( HttpClient client, RssParser parser) { _client = client; _parser = parser; }
public static async Task <IEnumerable <RssSchema> > FetchAsync(string feedUrl) { string feed = null; using (var client = new HttpClient()) { try { feed = await client.GetStringAsync(feedUrl); } catch { // 这里暂不处理异常。 } } if (feed != null) { var parser = new RssParser(); var rss = parser.Parse(feed); return(rss); } return(Enumerable.Empty <RssSchema>()); }
public async Task <IEnumerable <RssSchema> > Parse(string url) { string feed = null; using (var client = new HttpClient()) { try { feed = await client.GetStringAsync(url); } catch { // ignored } } if (feed == null) { return(new List <RssSchema>()); } var parser = new RssParser(); var rss = parser.Parse(feed); return(rss); }
public virtual async Task <IEnumerable <RssSchema> > Read() { return(await Task.Run(() => { IEnumerable <RssSchema> rss = null; for (var i = 0; i < Source.Length; i++) { try { var data = Utils.HttpGET(Source[i]); if (!string.IsNullOrEmpty(data)) { var parser = new RssParser(); rss = parser.Parse(data); break; } } catch (Exception ex) { ConsoleLog.Error("Feed", ex.GetFormatString()); ConsoleLog.Error("Feed", "Target Url: ".CreateStringBuilder() .AppendLine(Source[i]).Append(ConsoleLog.ErrorLogBuilder(ex, true)).ToString()); } } return rss; })); }
/// <summary> /// Initiate a remote (web) search using the engine incl. search expression specified /// by searchFeedUrl. We assume, the specified Url will return a RSS feed. /// This can be used e.g. to get a RSS search result from feedster. /// </summary> /// <param name="searchFeedUrl">Complete Url of the search engine incl. search expression</param> /// <param name="tag">optional, can be used by the caller</param> public void SearchRemoteFeed(string searchFeedUrl, object tag) { var unreturnedMatchItems = new List <INewsItem>(1); try { unreturnedMatchItems = RssParser.DownloadItemsFromFeed(searchFeedUrl); } catch (Exception remoteSearchException) { unreturnedMatchItems.Add(FeedSource.CreateHelpNewsItemFromException(remoteSearchException)); } int feedmatches = 1; int itemmatches = unreturnedMatchItems.Count; var fi = new FeedInfo(String.Empty, String.Empty, unreturnedMatchItems, String.Empty, String.Empty, String.Empty, new Dictionary <XmlQualifiedName, string>(), String.Empty); var fil = new FeedInfoList(String.Empty) { fi }; RaiseSearchFinishedEvent(tag, fil, feedmatches, itemmatches); }
public async static Task <List <NewsInformation> > GetFilteredNewsAsync() { List <NewsInformation> results = new List <NewsInformation>(); var client = new HttpClient(); var feed = await client.GetStringAsync("https://www.dr.dk/nyheder/service/feeds/allenyheder/"); var parser = new RssParser(); var newsResult = parser.Parse(feed); results = (from item in newsResult select new NewsInformation() { Title = item.Title, Description = item.Summary, CreatedDate = item.PublishDate, //ImageUrl = item. ImageUrl = @"https://is1-ssl.mzstatic.com/image/thumb/Purple124/v4/29/55/2e/29552ea2-5952-af7a-f398-89d177968258/AppIcon-0-0-1x_U007emarketing-0-0-0-7-0-0-85-220.png/600x600wa.png" }).ToList(); // var filteredResult = results.Where(w => w.Description.Contains("corona") || w.Title.Contains("Covid-19")) var filteredResult = results.Where(w => (w.Title.IndexOf("Corona", StringComparison.OrdinalIgnoreCase) != -1 || (w.Title.IndexOf("covid-19", StringComparison.OrdinalIgnoreCase) != -1))) .OrderBy(w => w.CreatedDate) .Take(12).ToList(); return(filteredResult); //return results.Where(w => !string.IsNullOrEmpty(w.ImageUrl)).Take(10).ToList(); }
public async Task <List <RssItem> > GetAsync(string uri) { //return new List<RssItem>() //{ // new RssItem // { // Id = 1, // Title = "aaaa", // Description = "bbbbb", // PublishDate = DateTime.Now // } //}; using (var httpClient = new HttpClient()) { using (var response = await httpClient.GetAsync(uri)) { var content = await response.Content.ReadAsStringAsync(); var items = new RssParser().Parse(content) //.Descendants("item") .Select(x => new RssItem { Title = x.Title, Description = x.Content, PublishDate = x.PublishDate, ImageUrl = x.ImageUrl, IsRead = false }).ToList(); return(items); } } }
public void ParseFeed_NotRss() { using (var stream = File.OpenRead(GetExamplesWith("Sample.xml"))) { Assert.That(RssParser.Parse(stream), Is.Null); } }
/// <summary> /// 从URL获取解析后的Item的信息 /// </summary> /// <param name="url">地址</param> /// <returns></returns> public static async Task <List <RssSchema> > GetSchemaFromUrl(string url, bool isLimit = false) { string feed = null; try { feed = await GetTextFromUrl(url, isLimit); } catch (Exception) { } var list = new List <RssSchema>(); if (!string.IsNullOrEmpty(feed)) { try { var parser = new RssParser(); var rss = parser.Parse(feed); foreach (var item in rss) { list.Add(item); } } catch (Exception ex) { Debug.WriteLine(ex.Message); } } return(list); }
public int ParseFeed_Content(string filename) { using (var stream = File.OpenRead(GetExamplesWith(filename))) { return(RssParser.Parse(stream).Items.First().Content.Length); } }
public async void ParseRSS() { string feed = null; RSSFeed.Clear(); using (var client = new HttpClient()) { try { feed = await client.GetStringAsync(Url); } catch { } } if (feed != null) { var parser = new RssParser(); var rss = parser.Parse(feed); foreach (var element in rss) { RSSFeed.Add(element); } } }
public void CreateFeedList(PodcastFeed podcastFeed) { var feedTab = new TabPage(); tabCtrlFeeds.TabPages.Add(feedTab); var feedList = new ListBox(); feedList.Dock = DockStyle.Fill; feedTab.Controls.Add(feedList); var rss = WebFetcher.FetchRss(podcastFeed.Url); var feed = RssParser.GetPodcastFeed(rss); podcastFeed.PodcastEpisodes = feed.PodcastEpisodes; feedTab.Text = podcastFeed.Name; foreach (PodcastEpisode podcast in podcastFeed.PodcastEpisodes) { feedList.Items.Add(podcast.Name); } ActiveRss.Add(podcastFeed.Id, rss); TabPages.Add(podcastFeed.Id, feedTab); UpdateClocks.Add(podcastFeed.Id, new UpdateClock(podcastFeed)); SetUpdateClocks(); UpdateClocks[podcastFeed.Id].Start(); }
/// <summary> /// Gets the feed /// </summary> /// <param name="url"></param> /// <returns></returns> private static async Task <IEnumerable <Feed> > ParseRss(string url) { string feed = null; var feedList = new List <Feed>(); try { using (var client = new HttpClient(new HttpClientHandler { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate })) { ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12; // Get the XML var res = await client.GetAsync(url); if (res.IsSuccessStatusCode) { feed = await res.Content.ReadAsStringAsync(); feed = feed.Replace("\r\n", string.Empty).Trim(); // Check XML is Valid if (IsValidXml(feed)) { if (feed != null) { var parser = new RssParser(); var rss = parser.Parse(feed); foreach (var element in rss) { feedList.Add(new Feed() { Author = element.Author, Categories = element.Categories, Content = element.Content, ExtraImageUrl = element.ExtraImageUrl, FeedUrl = element.FeedUrl, ImageUrl = element.ImageUrl, InternalID = element.InternalID, MediaUrl = element.MediaUrl, PublishDate = element.PublishDate, Summary = element.Summary, Title = element.Title }); } } } } } return(feedList); } catch { return(null); } }
public void Strip_Summary() { using (var stream = File.OpenRead(GetExamplesWith("SampleRss20-03.xml"))) { var dest = RssParser.Parse(stream); Assert.That(dest.Items[0].Summary, Is.EqualTo("この画像はテスト画像です。")); } }
public void AddToHierarchy(VirtualHierarchy hierarchy) { var rssParser = new RssParser(@"http://channel9.msdn.com/Feeds/RSS/", _downloader); hierarchy.AddFolder(hierarchy.Root, "RSS", mediaSource: rssParser); }
internal static IFeedParser Sniff(Request request) { IFeedParser parser = null; if ( request.Xml.Contains("<rdf:RDF") ) { try { parser = new RdfParser(request.Uri, request.Xml); } catch ( Exception e ) { Globals.Exception(e); } } else if ( request.Xml.Contains("<rss") ) { try { parser = new RssParser(request.Uri, request.Xml); } catch ( Exception e ) { Globals.Exception(e); } } else if ( request.Xml.Contains("<feed") ) { try { parser = new AtomParser(request.Uri, request.Xml); } catch ( Exception e ) { Globals.Exception(e); } } if ( parser != null ) { if ( parser.Name == null ) { return null; } else { return parser; } } else { return null; } }