public static IEnumerable <NewsItem> GetTodayNews()
{
var lastTime = DateTime.Now;
var today = DateTime.Today.Day;
var parser = new NewsParser();
var newsHtml = new HashSet <NewsItem>();
do
{
var href = GenerateLink(lastTime);
var data = GetPageByUrl(href);
var page = parser.ParseNewsList(data);
foreach (var singlePreview in page)
{
var link = parser.ParseHrefOfNews(singlePreview);
lastTime = parser.ParseTimeOfNews(singlePreview);
var htmlPage = GetPageByUrl(link);
var item = new NewsItem(link, lastTime);
item.DownloadContent(parser, htmlPage);
if (!newsHtml.Contains(item))
{
newsHtml.Add(item);
}
}
}while (lastTime.Day == today);
return(newsHtml.Where(x => x.dateTime.Day == today));
}
static void Main(string[] args)
{
Console.OutputEncoding = Encoding.UTF8;
var frequencies = new Dictionary <string, int>();
var parser = new NewsParser();
var filter = new WordFilter();
Console.WriteLine("Getting today's news.");
var news = HtmlTools.GetTodayNews();
foreach (var item in news)
{
NewsParser.CalculateStats(item, filter, frequencies);
}
var result = NewsParser.GetSortedDictionary(frequencies);
foreach (var keyValuePair in result)
{
Console.Write(keyValuePair.Key);
Console.Write(" ");
Console.WriteLine(keyValuePair.Value);
}
Console.ReadKey();
}
public void DownloadContent(NewsParser parser, string htmlPage)
{
Text = parser.ParseTextOfNews(htmlPage);
}
