private static IObservable <object> CreateObservable(FeedReader feedReader, Config config)
{
if (config.Url is null)
{
throw new ArgumentNullException(nameof(config.Url));
}
return(Observable.Create <FeedItem>(async(observer, cancellationToken) =>
{
var lastItem = feedReader.RetrieveFeed(config.Url.ToString())
.OrderBy(o => o.PublishDate)
.LastOrDefault();
while (!cancellationToken.IsCancellationRequested)
{
await Task.Delay(config.Interval, cancellationToken);
var items = feedReader.RetrieveFeed(config.Url.ToString());
var checkDate = lastItem?.PublishDate ?? DateTimeOffset.MinValue;
foreach (var item in items
.OrderBy(o => o.PublishDate)
.Where(o => o.PublishDate > checkDate))
{
observer.OnNext(item);
lastItem = item;
}
}
observer.OnCompleted();
})
.Publish()
.RefCount());
}
public async Task <IActionResult> Get(string url)
{
var user = await _authenticator.VerifyClaim(HttpContext.Request.Cookies["sessionKey"]);
if (user == null)
{
return(new ForbidResult());
}
Console.WriteLine(url);
var reader = new FeedReader();
var items = reader.RetrieveFeed(url);
var output = new List <Post>();
foreach (var item in items)
{
output.Add(new Post
{
Title = item.Title,
Date = item.PublishDate,
Image = item.Images.FirstOrDefault(),
Summary = item.Summary,
Body = item.Content,
Anchor = item.Uri
});
}
return(new ObjectResult(output));
}
public void BasicRSSWithImageFeedTest()
{
var target = new FeedReader(true);
var items = target.RetrieveFeed(@"TestFeeds\basic_image.rss").ToArray();
Assert.AreEqual(2, items.Length);
Assert.AreEqual("http://example.org/foo/bar/1", items[0].Uri.ToString());
Assert.AreEqual("Title 1", items[0].Title);
Assert.IsTrue(items[0].Summary.StartsWith("Lorem ipsum dolor sit"));
Assert.IsNull(items[0].Content);
Assert.AreEqual("tag:example.org,1999:blog-123456789123456789123456789.post-987564321987654231", items[0].Id);
Assert.AreEqual(DateTimeOffset.Parse("2014-04-16T13:57:35.0000000+02:00", CultureInfo.InvariantCulture, DateTimeStyles.None), items[0].PublishDate);
Assert.AreEqual(DateTimeOffset.Parse("2014-04-16T13:57:35.0000000+02:00", CultureInfo.InvariantCulture, DateTimeStyles.None), items[0].LastUpdatedDate);
Assert.IsNull(items[1].Title);
Assert.IsNull(items[1].Summary);
Assert.IsNull(items[1].Content);
Assert.IsNull(items[1].Uri);
Assert.IsNull(items[1].Id);
Assert.AreEqual(DateTimeOffset.MinValue, items[1].PublishDate);
Assert.AreEqual(DateTimeOffset.MinValue, items[1].LastUpdatedDate);
Assert.IsTrue(items[0].GetContent().StartsWith("Lorem ipsum dolor sit"));
Assert.IsTrue(items[0].GetSummary().StartsWith("Lorem ipsum dolor sit"));
#pragma warning disable 0618
Assert.AreEqual(DateTimeOffset.Parse("2014-04-16T13:57:35.0000000+02:00", CultureInfo.InvariantCulture, DateTimeStyles.None), items[0].Date);
Assert.AreEqual(DateTimeOffset.MinValue, items[1].Date);
#pragma warning restore 0618
Assert.IsTrue(items[0].Images.Count() == 2);
Assert.AreEqual(items[0].Images.ElementAt(0).ToString(), "http://example.org/foo/bar/123abc.png");
Assert.AreEqual(items[0].Images.ElementAt(1).ToString(), "http://example.org/foo/bar/123abc_2.png");
}
public void ReadFeed(string url)
{
//Notification notification;
List <string> titels = FeedFileReader(url);
var reader = new FeedReader();
while (true)
{
var items = reader.RetrieveFeed(url);
int posOffset = 0;
foreach (var i in items)
{
if (titels.Contains(i.Title))
{
continue;
}
//Debug.WriteLine(string.Format("{0}\t{1}\t{2}", i.Date.ToString("g"), i.Title, i.Summary));
//notification = new Notification(i.Title, i.Summary, i.Uri.Host, posOffset);
//notification.Show();
Notification.Create(i.Title, i.Summary, i.Uri.Host, posOffset);
Thread.Sleep(20);
posOffset++;
titels.Add(i.Title);
FeedFileWriter(url, i.Title);
}
Thread.Sleep(60000);
}
}
public void BasicAtomFeedTest()
{
var target = new FeedReader(true);
var items = target.RetrieveFeed(@"TestFeeds\basic.atom").ToArray();
Assert.AreEqual(2, items.Length);
Assert.AreEqual("http://example.org/foo/bar/1", items[0].Uri.ToString());
Assert.AreEqual("Summary1", items[0].Summary);
Assert.AreEqual("Test1", items[0].Title);
Assert.AreEqual("HTML content", items[0].Content);
Assert.AreEqual("urn:uuid:0ea6c57b-4546-4264-8b96-13434c349d87", items[0].Id);
Assert.AreEqual(DateTimeOffset.Parse("2013-03-13T13:37:31.0000000+00:00", CultureInfo.InvariantCulture, DateTimeStyles.None), items[0].PublishDate);
Assert.AreEqual(DateTimeOffset.Parse("2014-04-16T13:57:35.0000000+00:00", CultureInfo.InvariantCulture, DateTimeStyles.None), items[0].LastUpdatedDate);
Assert.AreEqual("http://example.org/foo/bar/2", items[1].Uri.ToString());
Assert.AreEqual("Summary2", items[1].Summary);
Assert.AreEqual("Test2", items[1].Title);
Assert.AreEqual("Text content", items[1].Content);
Assert.AreEqual("urn:uuid:d58672c4-f62e-483e-ab00-ff0940113e29", items[1].Id);
Assert.AreEqual(DateTimeOffset.MinValue, items[1].PublishDate);
Assert.AreEqual(DateTimeOffset.MinValue, items[1].LastUpdatedDate);
#pragma warning disable 0618
Assert.AreEqual(DateTimeOffset.Parse("2014-04-16T13:57:35.0000000+00:00", CultureInfo.InvariantCulture, DateTimeStyles.None), items[0].Date);
Assert.AreEqual(DateTimeOffset.MinValue, items[1].Date);
#pragma warning restore 0618
}
public static List <News> ReadRssToNews(string url, int retCount = 100)
{
List <News> newsList = new List <News>();
FeedReader reader = new FeedReader();
List <FeedItem> items = (List <FeedItem>)reader.RetrieveFeed(url);
List <FeedItem> retList = new List <FeedItem>();
if (retCount >= items.Count)
{
retCount = items.Count;
}
for (int i = 0; i < retCount; i++)
{
retList.Add(items[i]);
}
foreach (FeedItem item in retList)
{
News news = new News();
news.Title = item.Title;
news.BaseUrl = item.Uri.ToString();
news.Content = item.Content;
news.CreateTime = DateTime.Now;
news.EditTime = DateTime.Now;
newsList.Add(news);
}
return(newsList);
}
private static string GetReleaseNotesFor(string newVersion)
{
var reader = new FeedReader(new RssFeedNormalizer());
var releases = reader.RetrieveFeed("https://github.com/YetAnotherPomodoroApp/YAPA-2/releases.atom");
var release = releases.First(x => x.Title.Contains(newVersion));
return(release?.Content);
}
public void BasicRSSCategoriesTest()
{
var target = new FeedReader();
var items = target.RetrieveFeed(@"TestFeeds\categories.rss").ToArray();
Assert.AreEqual(items[0].Categories.ElementAt(0), "NEWS");
Assert.AreEqual(items[0].Categories.ElementAt(1), "TEST");
}
public void ExtendedNormalizerRSSFeedTest()
{
var target = new FeedReader(new ExtendedFeedItemNormalizer(), true);
var items = target.RetrieveFeed(@"TestFeeds\basic.rss").ToArray();
Assert.IsInstanceOfType(items[0], typeof(ExtendedFeedItem));
Assert.AreEqual(1, ((ExtendedFeedItem)items[0]).Authors.Length);
Assert.AreEqual("[email protected] (John Doe 1)", ((ExtendedFeedItem)items[0]).Authors[0]);
}
public void BasicAtomCategoriesTest()
{
var target = new FeedReader();
var items = target.RetrieveFeed(@"TestFeeds\categories.atom").ToArray();
Assert.AreEqual(items[0].Categories.ElementAt(0), "a");
Assert.AreEqual(items[1].Categories.ElementAt(0), "b");
Assert.AreEqual(items[1].Categories.ElementAt(1), "c");
}
// GET: Search
public ActionResult Index(string searchString, string searchAlgo)
{
var feed = new FeedReader();
if (!String.IsNullOrEmpty(searchString))
{
var items = feed.RetrieveFeed("http://www.antaranews.com/rss/terkini");
ArrayList result = new ArrayList();
Finder f = new Finder();
string searchResult = "";
//using htmlagilitypack here
HtmlDocument htmlDoc = new HtmlDocument();
using (var actualArticle = new HttpClient())
{
foreach (var item in items)
{
var response = actualArticle.GetAsync((item as FeedItem).Uri).Result;
if (response.IsSuccessStatusCode)
{
var responseContent = response.Content;
htmlDoc.LoadHtml(responseContent.ReadAsStringAsync().Result);
HtmlNode node = htmlDoc.DocumentNode.SelectNodes("//div[@id='content_news']").First();
if (searchAlgo.Equals("Booyer-Moore Algorithm"))
{
if (f.booyerMoore(node.InnerText, searchString, ref searchResult))
{
(item as FeedItem).Content = searchResult;
result.Add(item);
}
}
else
if (searchAlgo.Equals("Knuth–Morris–Pratt Algorithm"))
{
if (f.kmpMatch(node.InnerText, searchString, ref searchResult))
{
(item as FeedItem).Content = searchResult;
result.Add(item);
}
}
else
if (searchAlgo.Equals("Regex"))
{
if (f.regex(node.InnerText, searchString, ref searchResult))
{
(item as FeedItem).Content = searchResult;
result.Add(item);
}
}
}
}
}
ViewData["Keywords"] = searchString;
ViewData["Algo"] = searchAlgo;
ViewBag.items = result;
}
return(View());
}
public static List <FeedItem> ParseRssFile2(string url)
{
using (XmlReader reader = XmlReader.Create(url))
{
var reader1 = new FeedReader();
var items = reader1.RetrieveFeed(reader);
return(items.ToList());
//SyndicationFeed feed = SyndicationFeed.Load(reader);
//return feed.Items;
}
}
protected override async Task Execute()
{
var token = CancellationTokenSource.Token;
var reader = new FeedReader();
using HttpClient vClient = new HttpClient();
var http = await vClient.GetAsync(_rssSource.Url, token);
if (http.IsSuccessStatusCode)
{
await Task.Factory.StartNew(() =>
{
try
{
var items = reader.RetrieveFeed(_rssSource.Url);
foreach (var i in items)
{
if ((keys != null && (i.Content != null && keys.Any(t => i.Content.Contains(t))) || keys?.Count == 0))
{
var keywords = _catalog.GetKeywords(_rssSource.LanguageSource.Description, i.Summary);
IntelItem intelItem = new IntelItem
{
Url = _rssSource.Url,
Content = i.GetSummary(),
Description = i.Title,
DateTimeCollected = i.LastUpdatedDate.DateTime,
Remarks = i?.Categories?.FirstOrDefault()
};
intelItem.Keywords.AddRange(i.Categories);
intelItem.Keywords.AddRange(keywords);
intelItem.SourceCountry = _rssSource.SourceCountry;
intelItem.LevelTrustable = _rssSource.Trustable;
intelItem.LanguageIntel = _rssSource.LanguageSource;
intelItem.CovertArea = _rssSource.CovertArea;
intelItem.Type = _rssSource.Type;
ProcessItemAdded(intelItem);
}
}
ProcessCompleted();
}
catch (Exception e)
{
_logger.Error(e.Message);
}
}, token);
}
else
{
_logger.Warn("bad url " + _rssSource.Url);
}
}
public void BasicActualAtomFeedTest()
{
var target = new FeedReader(new GoogleFeedItemNormalizer(), true);
var items = target.RetrieveFeed(@"TestFeeds\google_snapshot.atom").ToArray();
Assert.AreEqual(10, items.Length);
Assert.IsTrue(items[0].GetContent().StartsWith("(CNN) -- Rescue boats"));
Assert.IsTrue(items[1].GetContent().StartsWith("Pro-Russian troops guard"));
Assert.IsTrue(items[2].GetContent().StartsWith("(CNN) -- Former New York"));
Assert.IsTrue(items[3].GetContent().StartsWith("Two blasts near the"));
Assert.IsTrue(items[4].GetContent().StartsWith("A three-year-old boy"));
}
// public static RSSList GetRssFeed()
public static List <RSS> GetRssFeed(String _blogURL)
{
List <RSS> RSSitems = new List <RSS>();
RSSList myRSS = new RSSList();
var reader = new FeedReader();
var items = reader.RetrieveFeed(_blogURL); //("http://www.nytimes.com/services/xml/rss/nyt/International.xml");
foreach (var i in items)
{
RSSitems.Add(new Models.RSS(i.Uri.ToString(), i.Date.ToString(), i.Title, i.Summary));
}
return(RSSitems);
}
public static List <FeedItem> ReadRss(string url, int retCount = 100)
{
FeedReader reader = new FeedReader();
//"http://www.ruanyifeng.com/blog/atom.xml"
List <FeedItem> items = (List <FeedItem>)reader.RetrieveFeed(url);
List <FeedItem> retList = new List <FeedItem>();
if (retCount >= items.Count)
{
retCount = items.Count;
}
for (int i = 0; i < retCount; i++)
{
retList.Add(items[i]);
}
return(retList);
}
public void DefaultNormalizationTest()
{
var target = new FeedReader(true);
var items = target.RetrieveFeed(@"TestFeeds\decoding.rss").ToArray();
var i0 = items[0];
Assert.IsNull(i0.Summary);
Assert.IsNull(i0.Title);
var i1 = items[1];
Assert.AreEqual("foo&bar", i1.Summary);
Assert.AreEqual("foo&bar", i1.Title);
var i2 = items[2];
Assert.AreEqual(i2.Title, i2.Content);
}
List <ArticleViewModel> IFeedService.GetFeedFor(Uri feedAddress)
{
var results = new List <ArticleViewModel>();
var reader = new FeedReader(throwOnError: true);
var articles = reader.RetrieveFeed(feedAddress.ToString());
foreach (var article in articles)
{
results.Add(new ArticleViewModel()
{
Title = article.Title,
Content = article.Title,
PublishDate = article.PublishDate
});
}
return(results);
}
public void PruebaFeed()
{
string url = "http://rss.cnn.com/rss/cnn_topstories.rss";
//string url = "https://www.nasa.gov/rss/dyn/breaking_news.rss";
//string url = "http://www.codeguru.com/rss/csharp/";
for (int i = 0; i < 5; i++)
{
var reader = new FeedReader(true);
var items = reader.RetrieveFeed(url);
}
//foreach (SyndicationItem item in feed.Items)
////foreach (SyndicationItem item in rssFormater.Feed.Items)
//{
// String subject = item.Title.Text;
// String summary = item.Summary.Text;
//}
}
public IEnumerable <ReleaseForm> Check(string url)
{
//ServicePointManager.Expect100Continue = false;
//ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, sslPolicyErrors) => true;
//ServicePointManager.CheckCertificateRevocationList = false;
var reader = new FeedReader();
var items = reader.RetrieveFeed(url);
return(items.Any()
? items.Select(i => new ReleaseForm
{
Date = i.PublishDate.LocalDateTime,
Title = i.Title,
Summary = Html2Text(i.Summary),
Url = i.Uri.ToString(),
UrlHash = i.Uri.ToString().GetIntHash()
})
: CheckUsingWebClient(url));
}
public IEnumerable <ReleaseForm> CheckUsingWebClient(string url)
{
using (var webClient = WebClientFactory.ChromeClient())
{
try
{
var xmlString = webClient.DownloadString(url);
var reader = new FeedReader();
var items = reader.RetrieveFeed(XmlReader.Create(new StringReader(xmlString)));
return(items.Select(i => new ReleaseForm
{
Date = i.PublishDate.LocalDateTime,
Title = i.Title,
Summary = Html2Text(i.Summary),
Url = i.Uri.ToString(),
UrlHash = i.Uri.ToString().GetIntHash()
}));
}
catch (Exception ex) {}
}
return(new List <ReleaseForm>());
}
private void CarregaFeeds()
{
var reader = new FeedReader();
var items = reader.RetrieveFeed("https://jovemnerd.com.br/feed/");
List <Item> oItens = new List <Item>();
Item oItem = new Item();
foreach (var i in items)
{
oItem = new Item
{
item_titulo = i.Title,
item_url = i.Uri.ToString(),
item_dthr = i.Date.LocalDateTime,
item_conteudo = i.Content
};
oItens.Add(oItem);
}
itemBindingSource.DataSource = oItens;
}
public IEnumerable <FeedItem> ReadFeed(string url)
{
return(feedReader.RetrieveFeed(url));
}
public ActionResult Search(string Algorithm, string Keyword)
{
if (!String.IsNullOrEmpty(Keyword) && !String.IsNullOrEmpty(Algorithm))
{
var feed = new FeedReader();
var news = feed.RetrieveFeed("http://www.antaranews.com/rss/terkini");
ArrayList result = new ArrayList();
HtmlDocument htmlDoc = new HtmlDocument();
using (var article = new HttpClient())
{
foreach (var i in news)
{
var response = article.GetAsync((i as FeedItem).Uri).Result;
if (response.IsSuccessStatusCode)
{
var content = response.Content;
htmlDoc.LoadHtml(content.ReadAsStringAsync().Result);
HtmlNodeCollection nodes = htmlDoc.DocumentNode.SelectNodes("//div[@id='content_news']");
if (nodes != null)
{
var node = nodes.First();
StringMatch testSearch = new StringMatch(node.InnerText, Keyword);
if (Algorithm.Equals("KMP"))
{
//search with KMP
//if found, replace i.Content to string that contain keyword, then add to result
String searchResult = testSearch.KMPsearch();
if (!String.IsNullOrEmpty(searchResult))
{
i.Content = searchResult;
result.Add(i as FeedItem);
}
}
else if (Algorithm.Equals("Booyer-Moore"))
{
//search with Booyer-Moore
//if found, replace i.summary to string that contain keyword, then add to result
String searchResult = testSearch.BoyerMooreSearch();
if (!String.IsNullOrEmpty(searchResult))
{
i.Content = searchResult;
result.Add(i as FeedItem);
}
}
else if (Algorithm.Equals("Regex"))
{
//search with Regex
//if found, replace i.summary to string that contain keyword, then add to result
String searchResult = testSearch.RegexSearch();
if (!String.IsNullOrEmpty(searchResult))
{
i.Content = searchResult;
result.Add(i as FeedItem);
}
}
}
}
}
}
ViewBag.items = result;
}
return(View());
}
public void ThrowsWhenRequiredFeedTest1()
{
var target = new FeedReader(true);
var items = target.RetrieveFeed(@"TestFeeds\non_existing.rss");
}
public static List <RSS> GetAllRssFeeds(String CollectionName)
{
List <RSS> RSSitems = new List <RSS>();
RSSList myRSS = new RSSList();
List <String> Urls = new List <string>();
var reader = new FeedReader();
Urls = GetCollectionUrls(CollectionName);
List <List <RSS> > NotSortedFeedList = new List <List <RSS> >();
foreach (var feedurl in Urls)
{
List <RSS> Temp = new List <RSS>();
var items = reader.RetrieveFeed(feedurl);//("http://www.nytimes.com/services/xml/rss/nyt/International.xml");
foreach (var i in items)
{
Temp.Add(new Models.RSS(i.Uri.ToString(), i.Date.ToString(), i.Title, i.Summary));
}
NotSortedFeedList.Add(Temp);
}
int max = 0;//максимальное количество постов в фиде
foreach (var feed in NotSortedFeedList)
{
int i = 0;
foreach (var post in feed)
{
i++;
}
if (i > max)
{
max = i;
}
}
//-----------------
//начальное значение Pstop (2) - это количество постов от одного фида в группке:
int Pstop = 2; int Pstart = 0;//индексы первого и последнего постов в группке
for (Pstop = 2; Pstop < max; Pstop += 2)
{
foreach (var feed in NotSortedFeedList)
{
for (int post_i = Pstart; post_i < Pstop; post_i++)
{
try
{
if (feed[post_i] != null)
{
RSSitems.Add(feed[post_i]);
}
} catch (Exception) { };
}
}
Pstart += 2;
}
return(RSSitems);
}
public void SuppressesExceptionsWhenRequiredFeedTest()
{
var target = new FeedReader(false);
var items = target.RetrieveFeed(@"TestFeeds\non_existing.rss");
}
public void DTDInjectionTest2()
{
var target = new FeedReader(true); //We want to check the exception so don't suppress it
var items = target.RetrieveFeed(@"TestFeeds\xml_injection2.rss");
}
public void ThrowsWhenRequiredFeedTest2()
{
var target = new FeedReader(true);
var items = target.RetrieveFeed(@"http://example.org/non_existing.rss");
}
private void monitor()
{
const int inGameOnlyStartDelayMilliSecs = 5 * 60 * 1000; // 5 mins
const int alwaysOnIntervalMilliSecs = 2 * 60 * 1000; // 2 mins
const int inGameOnlyIntervalMilliSecs = 30 * 1000; // 30 secs
if (!configuration.galnetAlwaysOn)
{
// Wait at least 5 minutes after starting before polling for new articles, but only if galnetAlwaysOn is false
Thread.Sleep(inGameOnlyStartDelayMilliSecs);
}
while (running)
{
if (configuration.galnetAlwaysOn)
{
monitorGalnet();
Thread.Sleep(alwaysOnIntervalMilliSecs);
}
else
{
// We'll update the Galnet Monitor only if a journal event has taken place within the specified number of minutes
if ((DateTime.UtcNow - EDDI.Instance.JournalTimeStamp).TotalMinutes < 10)
{
monitorGalnet();
}
else
{
Logging.Debug("No in-game activity detected, skipping galnet feed update");
}
Thread.Sleep(inGameOnlyIntervalMilliSecs);
}
void monitorGalnet()
{
List <News> newsItems = new List <News>();
string firstUid = null;
locales.TryGetValue(configuration.language, out locale);
string url = GetGalnetResource("sourceURL");
altURL = false;
try
{
WebRequest request = WebRequest.Create(url);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
}
catch (WebException wex)
{
Logging.Warn("Exception contacting primary galnet feed, trying alternate: ", wex.Message);
url = GetGalnetResource("alternateURL");
altURL = true;
}
Logging.Debug("Fetching Galnet articles from " + url);
IEnumerable <FeedItem> items = null;
try
{
FeedReader feedReader = new FeedReader(new GalnetFeedItemNormalizer(), true);
items = feedReader.RetrieveFeed(url);
}
catch (WebException wex)
{
Logging.Warn("Exception attempting to obtain galnet feed: ", wex);
}
catch (System.Xml.XmlException xex)
{
Logging.Error("Exception attempting to obtain galnet feed: ", xex);
}
if (items != null)
{
try
{
foreach (GalnetFeedItemNormalizer.ExtendedFeedItem item in items)
{
if (firstUid == null)
{
// Obtain the ID of the first item that we read as a marker
firstUid = item.Id;
}
if (item.Id == configuration.lastuuid)
{
// Reached the first item we have already seen - go no further
break;
}
News newsItem = new News(item.Id, assignCategory(item.Title, item.GetContent()), item.Title, item.GetContent(), item.PublishDate.DateTime, false);
newsItems.Add(newsItem);
GalnetSqLiteRepository.Instance.SaveNews(newsItem);
}
if (firstUid != null && firstUid != configuration.lastuuid)
{
Logging.Debug("Updated latest UID to " + firstUid);
configuration.lastuuid = firstUid;
configuration.ToFile();
}
if (newsItems.Count > 0)
{
// Spin out event in to a different thread to stop blocking
Thread thread = new Thread(() =>
{
try
{
EDDI.Instance.enqueueEvent(new GalnetNewsPublishedEvent(DateTime.UtcNow, newsItems));
}
catch (ThreadAbortException)
{
Logging.Debug("Thread aborted");
}
})
{
IsBackground = true
};
thread.Start();
}
}
catch (Exception ex)
{
Logging.Error("Exception attempting to handle galnet feed: ", ex);
}
}
}
}
}