public static IEnumerable <NewsItem> GetTodayNews() { var lastTime = DateTime.Now; var today = DateTime.Today.Day; var parser = new NewsParser(); var newsHtml = new HashSet <NewsItem>(); do { var href = GenerateLink(lastTime); var data = GetPageByUrl(href); var page = parser.ParseNewsList(data); foreach (var singlePreview in page) { var link = parser.ParseHrefOfNews(singlePreview); lastTime = parser.ParseTimeOfNews(singlePreview); var htmlPage = GetPageByUrl(link); var item = new NewsItem(link, lastTime); item.DownloadContent(parser, htmlPage); if (!newsHtml.Contains(item)) { newsHtml.Add(item); } } }while (lastTime.Day == today); return(newsHtml.Where(x => x.dateTime.Day == today)); }
static void Main(string[] args) { Console.OutputEncoding = Encoding.UTF8; var frequencies = new Dictionary <string, int>(); var parser = new NewsParser(); var filter = new WordFilter(); Console.WriteLine("Getting today's news."); var news = HtmlTools.GetTodayNews(); foreach (var item in news) { NewsParser.CalculateStats(item, filter, frequencies); } var result = NewsParser.GetSortedDictionary(frequencies); foreach (var keyValuePair in result) { Console.Write(keyValuePair.Key); Console.Write(" "); Console.WriteLine(keyValuePair.Value); } Console.ReadKey(); }
public void DownloadContent(NewsParser parser, string htmlPage) { Text = parser.ParseTextOfNews(htmlPage); }