static void Do(string url)
{
var links = FeedReader.GetFeedUrlsFromUrlAsync(url).Result;
foreach (var link in links)
{
try
{
string title = link.Title;
if (string.IsNullOrEmpty(title))
{
title = url.Replace("https", "").Replace("http", "").Replace("www.", "");
}
title = Regex.Replace(title.ToLower(), "[^a-z]*", "");
var curl = FeedReader.GetAbsoluteFeedUrl(url, link);
string content = Helpers.DownloadAsync(curl.Url).Result;
System.IO.File.WriteAllText("c:\\data\\feeds\\" + title + "_" + Guid.NewGuid().ToString() + ".xml", content);
Console.Write("+");
}
catch (Exception ex)
{
Console.WriteLine(link.Title + " - " + link.Url + ": " + ex.ToString());
}
}
}
public static async Task <string> FindUrlFeed(this string url)
{
Log.Information($"Scanning {url} ..");
var urls = await FeedReader.GetFeedUrlsFromUrlAsync(url);
Log.Information($"UrlFeeds: {urls.ToJson()}");
string feedUrl = "";
if (urls.Count() == 1) // no url - probably the url is already the right feed url
{
feedUrl = url;
}
else if (urls.Count() == 1)
{
feedUrl = urls.First().Url;
}
else if (urls.Count() == 2
) // if 2 urls, then its usually a feed and a comments feed, so take the first per default
{
feedUrl = urls.First().Url;
}
return(feedUrl);
}
static void Main(string[] args)
{
try
{
Console.WriteLine("Please enter feed url:");
string url = Console.ReadLine();
var urlsTask = FeedReader.GetFeedUrlsFromUrlAsync(url);
urlsTask.ConfigureAwait(false);
var urls = urlsTask.Result;
string feedUrl;
if (urls == null || urls.Count() < 1)
{
feedUrl = url;
}
else if (urls.Count() == 1)
{
feedUrl = urls.First().Url;
}
else if (urls.Count() == 2) // if 2 urls, then its usually a feed and a comments feed, so take the first per default
{
feedUrl = urls.First().Url;
}
else
{
int i = 1;
Console.WriteLine("Found multiple feed, please choose:");
foreach (var feedurl in urls)
{
Console.WriteLine($"{i++.ToString()} - {feedurl.Title} - {feedurl.Url}");
}
var input = Console.ReadLine();
if (!int.TryParse(input, out int index) || index < 1 || index > urls.Count())
{
Console.WriteLine("Wrong input. Press key to exit");
Console.ReadKey();
return;
}
feedUrl = urls.ElementAt(index).Url;
}
var readerTask = FeedReader.ReadAsync(feedUrl);
readerTask.ConfigureAwait(false);
foreach (var item in readerTask.Result.Items)
{
Console.WriteLine(item.Title + " - " + item.Link);
}
}
catch (Exception ex)
{
Console.WriteLine($"An error occurred: {ex.InnerException.Message}{Environment.NewLine}{ex.InnerException.ToString()}");
}
Console.ReadKey();
}
public static async Task <string> FindUrlFeed(this string url)
{
Log.Information("Scanning {Url} ..", url);
var urls = await FeedReader.GetFeedUrlsFromUrlAsync(url);
Log.Debug("UrlFeeds: {@URLs}", urls);
var feedUrl = urls.FirstOrDefault()?.Url;
return(feedUrl);
}
public void TestParseAndAbsoluteUrlDerStandard1()
{
string url = "derstandard.at";
var links = FeedReader.GetFeedUrlsFromUrlAsync(url).Result;
foreach (var link in links)
{
var absoluteUrl = FeedReader.GetAbsoluteFeedUrl(url, link);
Assert.IsTrue(absoluteUrl.Url.StartsWith("http://"));
}
}
private void carregaLista()
{
//List<Feed> oFeed = FeedDados.RetornaNaoLidos();
try
{
var urls = FeedReader.GetFeedUrlsFromUrlAsync("https://iplusall.com/feed/").Result;
string feedUrl = "https://iplusall.com/feed/";
if (urls == null || urls.Count() < 1)
{
feedUrl = "https://iplusall.com/feed/";
}
else if (urls.Count() == 1)
{
feedUrl = urls.First().Url;
}
else if (urls.Count() == 2) // if 2 urls, then its usually a feed and a comments feed, so take the first per default
{
feedUrl = urls.First().Url;
}
var reader = FeedReader.ReadAsync(feedUrl).Result;
foreach (var item in reader.Items)
{
Console.WriteLine(item.Title + " - " + item.Link);
}
List <Item> oItens = new List <Item>();
foreach (var oUrl in reader.Items)
{
Item oItem = new Item();
oItem.item_titulo = oUrl.Title;
oItem.item_dthr = oUrl.PublishingDate;
oItem.item_conteudo = oUrl.Content;
oItem.item_url = oUrl.Link;
oItem.feed_codigo = 1;
oItens.Add(oItem);
Console.WriteLine(oUrl.Title + " - " + oUrl.Link);
}
itemBindingSource.DataSource = oItens;
//itemBindingSource.DataSource = ItemDados.RetornaNaoLidos();
}
catch (Exception ex)
{
Console.WriteLine($"An error occurred: {ex.InnerException.Message}{Environment.NewLine}{ex.InnerException.ToString()}");
}
Console.ReadKey();
}
public async Task TestParseAndAbsoluteUrlDerStandard1()
{
string url = "derstandard.at";
var links = await FeedReader.GetFeedUrlsFromUrlAsync(url).ConfigureAwait(false);
foreach (var link in links)
{
var absoluteUrl = FeedReader.GetAbsoluteFeedUrl(url, link);
Assert.IsTrue(absoluteUrl.Url.StartsWith("http://"));
}
}
public async Task AddFeedUrlAsync([Summary("The name/title of the feed")] string name, [Summary("The url to the feed (Atom/RSS)")] string url, [Summary("Optional: The name of the channel where the Feed updates should be posted. Defaults to current channel")] string channelName = "")
{
if (url.IsEmpty())
{
await ReplyAsync("Please enter a name for the feed!").ConfigureAwait(false);
return;
}
if (url.IsEmpty())
{
await ReplyAsync("Please enter a feed url!").ConfigureAwait(false);
return;
}
ITextChannel channel = await GetTextChannelInGuildAsync(channelName, true).ConfigureAwait(false);
if (channel == null)
{
await ReplyAsync("The specified channel was not found").ConfigureAwait(false);
return;
}
var urls = await FeedReader.GetFeedUrlsFromUrlAsync(url).ConfigureAwait(false);
string feedUrl;
if (!urls.Any()) // no url - probably the url is already the right feed url
{
feedUrl = url;
}
else if (urls.Count() == 1)
{
feedUrl = urls.First().Url;
}
else
{
await ReplyAsync($"Multiple feeds were found at this url. Please be more specific:{Environment.NewLine}{string.Join(Environment.NewLine, urls)}").ConfigureAwait(false);
return;
}
var currentFeeds = await feedService.GetFeedsForGuildAsync(Context.Guild.Id, channel.Id).ConfigureAwait(false);
if (currentFeeds.Any(x => x.feedUrl == feedUrl || x.name == name))
{
await ReplyAsync("The specified feed is already in the list!").ConfigureAwait(false);
return;
}
await feedService.AddFeedAsync(name, feedUrl, Context.Guild.Id, channel.Id).ConfigureAwait(false);
await ReplyAsync("Feed added").ConfigureAwait(false);
}
static void Main(string[] args)
{
Console.WriteLine("Start job");
const string tokenFile = ".token";
var token = File.ReadAllText(tokenFile);
const string url = "https://livesino.net/feed";
const string titlesName = "titles-livesino.txt";
var titlesPath = Environment.CurrentDirectory;
var botClient = new TelegramBotClient(token);
for (; ;)
{
var allTitles = File.ReadAllText(Path.Combine(titlesPath, titlesName));
var urlsTask = FeedReader.GetFeedUrlsFromUrlAsync(url);
urlsTask.ConfigureAwait(false);
var readerTask = FeedReader.ReadAsync(url);
readerTask.ConfigureAwait(false);
foreach (var item in readerTask.Result.Items)
{
if (allTitles.Contains(item.Title))
{
continue;
}
using (var outputFile = new StreamWriter(Path.Combine(titlesPath, titlesName), true))
{
outputFile.WriteLine(item.Title);
}
var ivLink = $"https://t.me/iv?url={item.Link}&rhash=e0127ad0c39cac";
botClient.SendTextMessageAsync("@livesino",
$"<b>{item.Title}</b>\n{ivLink}",
ParseMode.Html
);
Console.WriteLine($"{item.Title}\n{ivLink}");
}
Console.WriteLine($"yet another try at {DateTime.Now}");
Thread.Sleep(600000);
}
}
public string getFeedURL(string url)
{
var urlsTask = FeedReader.GetFeedUrlsFromUrlAsync(url);
var urls = urlsTask.Result;
string feedUrl = url;
if (urls == null || urls.Count() < 1)
{
return(url);
}
else
{
return(urls.First().Url); //if 1 or 2 urls, then its usually a feed and a comments feed, so take the first per default
}
}
public (string, Feed) Download(string name, string url)
{
try
{
var urlsTask = FeedReader.GetFeedUrlsFromUrlAsync(url);
var urls = urlsTask.Result;
string feedUrl;
if (urls == null || urls.Count() < 1)
{
feedUrl = url;
}
else if (urls.Count() == 1)
{
feedUrl = urls.First().Url;
}
else if (urls.Count() == 2)
{
feedUrl = urls.First().Url;
}
else
{
int i = 1;
Console.WriteLine("Found multiple feed, please choose:");
foreach (var feedurl in urls)
{
Console.WriteLine($"{i++.ToString()} - {feedurl.Title} - {feedurl.Url}");
}
var input = Console.ReadLine();
if (!int.TryParse(input, out int index) || index < 1 || index > urls.Count())
{
Console.WriteLine("Wrong input. Press key to exit");
Console.ReadKey();
return(name, Task.FromResult <Feed>(null).Result);
}
feedUrl = urls.ElementAt(index).Url;
}
return(name, FeedReader.ReadAsync(feedUrl).Result);
}
catch (Exception ex)
{
return(name, Task.FromResult <Feed>(null).Result);
}
}
private static async Task <List <Feed> > ProcessSearchResultAsync(VprWebSearchResult vprWebSearchResult)
{
var results = new List <Feed>();
if (vprWebSearchResult.Url.Contains("feedspot.com"))
{
return(results);
}
try
{
var htmlFeedLinks = await FeedReader.GetFeedUrlsFromUrlAsync(vprWebSearchResult.Url);
var feedLinks = htmlFeedLinks.ToList();
if (feedLinks.Any())
{
var feedTasks = ProcessHtmlFeedLinks(feedLinks);
var feeds = await Task.WhenAll(feedTasks);
foreach (var feed in feeds)
{
if (results.Any(x => x.Link == feed.Link) || feed.Items.Count <= 0)
{
continue;
}
if (string.IsNullOrEmpty(feed.Title) || string.IsNullOrEmpty(feed.Description))
{
continue;
}
feed.Title = feed.Title.Trim();
results.Add(feed);
}
}
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
return(results);
}
public static async Task <string> GetUrlRssFeed(string url)
{
var urls = await FeedReader.GetFeedUrlsFromUrlAsync(url);
string feedUrl = url;
if (urls.Count() < 1) // no url - probably the url is already the right feed url
{
feedUrl = url;
}
else if (urls.Count() == 1)
{
feedUrl = urls.First().Url;
}
else if (urls.Count() == 2) // if 2 urls, then its usually a feed and a comments feed, so take the first per default
{
feedUrl = urls.First().Url;
}
return(feedUrl);
}
private async Task <Feed> getFeed(string url)
{
IEnumerable <HtmlFeedLink> urls = await FeedReader.GetFeedUrlsFromUrlAsync(url);
string feedUrl;
if (urls == null || urls.Count() < 1)
{
feedUrl = url;
}
else if (urls.Count() == 1)
{
feedUrl = urls.First().Url;
}
else if (urls.Count() == 2) // if 2 urls, then its usually a feed and a comments feed, so take the first per default
{
feedUrl = urls.First().Url;
}
else
{
int i = 1;
Console.WriteLine("Found multiple feed, please choose:");
foreach (var feedurl in urls)
{
Console.WriteLine($"{i++.ToString()} - {feedurl.Title} - {feedurl.Url}");
}
var input = Console.ReadLine();
if (!int.TryParse(input, out int index) || index < 1 || index > urls.Count())
{
Console.WriteLine("Wrong input. Press key to exit");
Console.ReadKey();
return(null);
}
feedUrl = urls.ElementAt(index).Url;
}
return(await FeedReader.ReadAsync(feedUrl));
}
public async Task AddFeedAsync(string feed_url)
{
var found_feed_urls = from url in await FeedReader.GetFeedUrlsFromUrlAsync(feed_url) select url.Url;
if (found_feed_urls.Count() > 0)
{
var blog_url = found_feed_urls.First();
feed_url = blog_url.StartsWith('/') ? $"{feed_url.TrimEnd('/')}{blog_url}" : blog_url;
}
if (Feeds.Any(f => f.FeedLink == feed_url))
{
throw new Exception("There's already a feed with that url.");
}
else
{
var parsed_feed = await FeedReader.ReadAsync(feed_url);
var feed_uri = new Uri(parsed_feed.Link);
var feed_title = string.IsNullOrWhiteSpace(parsed_feed.Title) ? feed_uri.Host : parsed_feed.Title;
var feed_id = AvoidFeedIdCollision(MakeSafeId(feed_title));
string feed_image_path = await DownloadFeedImageAsync(feed_id, parsed_feed.ImageUrl, feed_uri.Host);
var feed = new Feed(feed_id, feed_url)
{
Title = feed_title,
Link = $"{feed_uri.Scheme}://{feed_uri.Host}",
ImagePath = feed_image_path,
DateLastUpdated = parsed_feed.LastUpdatedDate is null ? DateTime.Now : (DateTime)parsed_feed.LastUpdatedDate,
Description = ExtractTextFromHtml(parsed_feed.Description)
};
await WriteToFileAsync(Path.Combine(FeedDirectory, $"{feed_id}.feed"), feed.Serialize());
Directory.CreateDirectory(Path.Combine(FeedDirectory, feed_id));
await AddPostsToFeedAsync(feed, parsed_feed.Items);
Feeds.Add(feed);
}
string AvoidFeedIdCollision(string id)
{
var pre_path = Path.Combine(FeedDirectory, id);
if (Directory.Exists($"{pre_path}.feed"))
{
int suffix = 2;
while (Directory.Exists($"{pre_path}_{suffix}.feed"))
{
suffix++;
}
id = $"{id}_{suffix}";
}
return(id);
}
async Task <string> DownloadFeedImageAsync(string feed_id, string feed_image_url, string feed_host)
{
string feed_image_path = null;
if (string.IsNullOrEmpty(feed_image_url))
{
try
{
var web_client = new WebClient();
web_client.Headers.Add(HttpRequestHeader.UserAgent, "Fidd RSS Reader v1.0");
var favicon_api_response = await web_client.DownloadStringTaskAsync($"http://favicongrabber.com/api/grab/{feed_host}");
var deserialized_response = JsonConvert.DeserializeObject <FaviconGrabberResponse>(favicon_api_response);
if (deserialized_response.Icons.Count > 0)
{
feed_image_url = deserialized_response.Icons.First().Source;
}
} catch (Exception e) {; }
}
if (!string.IsNullOrEmpty(feed_image_url))
{
if (feed_image_url.StartsWith("data:image/"))
{
feed_image_url = feed_image_url.Substring(11);
var image_extension = feed_image_url.Substring(0, feed_image_url.IndexOf(';'));
feed_image_path = $"{feed_id}.{image_extension}";
var image_data_base64 = feed_image_url.Substring(feed_image_url.IndexOf(',') + 1);
await File.WriteAllBytesAsync(Path.Combine(FeedDirectory, feed_image_path), Convert.FromBase64String(image_data_base64));
}
else
{
var downloader = new WebClient();
string image_extension = ExtractExtensionFromUrl(feed_image_url);
feed_image_path = $"{feed_id}{image_extension}";
await downloader.DownloadFileTaskAsync(feed_image_url, Path.Combine(FeedDirectory, feed_image_path));
}
}
return(feed_image_path);
}
}
public async Task <IEnumerable <string> > GetFeedUrls(string baseUrl) => (await FeedReader.GetFeedUrlsFromUrlAsync(baseUrl)).Select(x => x.Url);
static void Main()
{
string hlp = "Please enter feed url or exit to stop the program:";
Console.WriteLine(hlp);
while (true)
{
try
{
string url = Console.ReadLine();
if (url.Equals("exit", StringComparison.InvariantCultureIgnoreCase))
{
break;
}
var urlsTask = FeedReader.GetFeedUrlsFromUrlAsync(url);
var urls = urlsTask.Result;
string feedUrl;
if (urls == null || urls.Count() < 1)
{
feedUrl = url;
}
else if (urls.Count() == 1)
{
feedUrl = urls.First().Url;
}
else if (urls.Count() == 2) // if 2 urls, then its usually a feed and a comments feed, so take the first per default
{
feedUrl = urls.First().Url;
}
else
{
int i = 1;
Console.WriteLine("Found multiple feed, please choose:");
foreach (var feedurl in urls)
{
Console.WriteLine($"{i++} - {feedurl.Title} - {feedurl.Url}");
}
var input = Console.ReadLine();
if (!int.TryParse(input, out int index) || index < 1 || index > urls.Count())
{
Console.WriteLine("Wrong input. Press key to exit");
Console.ReadKey();
return;
}
feedUrl = urls.ElementAt(index).Url;
}
var readerTask = FeedReader.ReadAsync(feedUrl);
readerTask.ConfigureAwait(false);
foreach (var item in readerTask.Result.Items)
{
Console.WriteLine(item.Title + " - " + item.Link);
}
}
catch (Exception ex)
{
Console.WriteLine($"An error occurred: {ex.InnerException.Message}{Environment.NewLine}{ex.InnerException}");
}
finally
{
Console.WriteLine("================================================");
Console.WriteLine(hlp);
}
}
}