private static void TestNewsImporter()
{
var providers = new List <NewsProvider>
{
// Fashion sites.
new NewsProvider(@"riamoda.ru", 2, 24, @"http://riamoda.ru/rss.xml", NewsProviderType.Fashion),
new NewsProvider(@"Modanews.ru", 2, 24, @"http://modanews.ru/rss/news/", NewsProviderType.Fashion),
new NewsProvider(@"glianec.com.u", 2, 24, @"http://glianec.com.ua/?format=feed&type=rss", NewsProviderType.Fashion),
new NewsProvider(@"fashiontime.ru - публикации", 2, 24, @"http://www.fashiontime.ru/rss/content.xml", NewsProviderType.Fashion),
new NewsProvider(@"fashiontime.ru - новости", 2, 24, @"http://www.fashiontime.ru/rss/content_news.xml", NewsProviderType.Fashion),
new NewsProvider(@"fashiontime.ru - статьи", 2, 24, @"http://www.fashiontime.ru/rss/content_article.xml", NewsProviderType.Fashion),
new NewsProvider(@"intermoda.ru", 2, 24, @"http://www.intermoda.ru/rss/news", NewsProviderType.Fashion),
new NewsProvider(@"profashion.ru", 2, 24, @"http://profashion.ru/news/rss/", NewsProviderType.Fashion),
new NewsProvider(@"fashionunited.ru", 2, 24, @"https://fashionunited.ru/rss-novosti", NewsProviderType.Fashion),
new NewsProvider(@"Style.rbc.ru", 2, 24, @"http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/style.rbc.ru/popural.rss", NewsProviderType.Fashion),
new NewsProvider(@"be-in.ru", 2, 24, @"http://www.be-in.ru/rss/moda", NewsProviderType.Fashion),
new NewsProvider(@"dress-code.com.ua", 2, 24, @"http://www.dress-code.com.ua/rss/common.rss", NewsProviderType.Fashion),
new NewsProvider(@"trendymen.ru", 2, 24, @"http://trendymen.ru/_services/rss.php", NewsProviderType.Fashion),
new NewsProvider(@"fashionpeople.ru", 2, 24, @"http://www.fashionpeople.ru/xml/rss.xml", NewsProviderType.Fashion),
// Tech sites
new NewsProvider(@"3dnews.ru", 2, 24, @"http://www.3dnews.ru/news/rss"),
//new NewsProvider(@"popmech.ru", 2, 24, @"http://www.popmech.ru/rss"),
new NewsProvider(@"mobbit", 2, 24, @"http://mobbit.info/rss.php"),
new NewsProvider(@"ixbt.co", 2, 24, @"http://www.ixbt.com/export/articles.rdf"),
new NewsProvider(@"mobile-review", 2, 24, @"http://www.mobile-review.com/newslistouter/rssnewsfull.xml"),
new NewsProvider(@"mobile-review-reviews", 2, 24, @"http://www.mobile-review.com/rss-review.xml"),
new NewsProvider(@"tech", 2, 24, @"https://www.google.com/news?output=rss&topic=t&ned=ru_ru&cf=all&num=20"),
new NewsProvider(@"astronet.ru", 2, 24, @"http://www.astronet.ru/db/rss.xml"),
new NewsProvider(@"nkj.ru", 2, 24, @"http://www.nkj.ru/rss/iblock_rss_31.xml"),
new NewsProvider(@"elementy.r", 2, 24, @"http://elementy.ru/rss/news"),
new NewsProvider(@"postnauka.ru", 2, 24, @"http://postnauka.ru/feed"),
new NewsProvider(@"habrahabr-startups", 2, 24, @"http://habrahabr.ru/rss/tag/%D1%81%D1%82%D0%B0%D1%80%D1%82%D0%B0%D0%BF%D1%8B"), // стартапы
new NewsProvider(@"habrahabr", 2, 24, @"http://www.habrahabr.ru/rss/main/"), // feed/http://habrahabr.ru/rss/best/
new NewsProvider(@"geektimes.ru", 2, 24, @"http://geektimes.ru/rss/all/"), // http://geektimes.ru/rss/best/
//new NewsProvider(@"", 2, 24, @""),
new NewsProvider(@"The Verge", 1, 24, @"http://www.theverge.com/rss/full.xml"),
new NewsProvider(@"Engadget", 1, 24, @"http://www.engadget.com/rss-full.xml"),
new NewsProvider(@"Lifehacker", 1, 24, @"http://feeds.gawker.com/lifehacker/vip"), // Lifehacker
//new NewsProvider(@"readwriteweb", 1, 24, @"http://www.readwriteweb.com/rss.xml"),
new NewsProvider(@"Techcrunch", 1, 24, @"http://feeds.feedburner.com/Techcrunch"), //Techcrunch
new NewsProvider(@"Wired", 1, 24, @"http://feeds.wired.com/wired/index"),
new NewsProvider(@"Gizmodo", 1, 24, @"http://feeds.gawker.com/gizmodo/full"), // Gizmodo
new NewsProvider(@"Mashable", 1, 24, @"http://feeds.mashable.com/Mashable"),
new NewsProvider(@"arstechnica", 1, 24, @"http://feeds.arstechnica.com/arstechnica/index/"),
new NewsProvider(@"CNET", 1, 24, @"http://news.com.com/2547-1_3-0-5.xml"),
new NewsProvider(@"Slashdot", 1, 24, @"http://rss.slashdot.org/Slashdot/slashdot"),
new NewsProvider(@"Google Blog", 1, 24, @"http://googleblog.blogspot.com/atom.xml"),
new NewsProvider(@"The Next Web", 1, 24, @"http://feeds2.feedburner.com/thenextweb"),
new NewsProvider(@"boingboing", 1, 24, @"http://boingboing.net/feed"),
new NewsProvider(@"thinkgeek", 1, 24, @"http://www.thinkgeek.com/thinkgeek.rss"),
new NewsProvider(@"TED Talks", 1, 24, @"http://feeds.feedburner.com/tedtalks_video"),
new NewsProvider(@"YCombinator", 1, 24, @"https://news.ycombinator.com/rss"),
//new NewsProvider(@"", 1, 24, @""),
};
Console.WriteLine("Getting news...");
var newsAggregator = new NewsAggregator();
Dictionary <string, List <News> > news = newsAggregator.GetNews(providers);
Console.WriteLine("Saving news...");
using (var sw = new StreamWriter(@"C:\rss_notaccepted.txt"))
{
using (var swAccept = new StreamWriter(@"C:\rss_accepted.txt"))
{
foreach (string newsprovider in news.Keys)
{
NewsProvider providerInfo = providers.FirstOrDefault(x => x.ProviderName == newsprovider);
foreach (var newsItem in news[newsprovider])
{
if (IsNewsAcceptable(newsItem, providerInfo))
{
swAccept.WriteLine(string.Format("{0} --- {1} --- {2}", providerInfo.ProviderName, newsItem.Header, DetectTheme(newsItem)));
}
else
{
// Debug.Assert(newsItem.Description.Length < 4000);
sw.WriteLine(string.Format("{0} --- {1}", providerInfo.ProviderName, newsItem.Header));
}
}
}
}
}
}
public void UpdateNews()
{
try
{
var providers = new List <NewsProvider>
{
// Fashion sites.
new NewsProvider(@"riamoda.ru", 2, NewsConstants.UpdateHours, @"http://riamoda.ru/rss.xml", NewsProviderType.Fashion),
new NewsProvider(@"Modanews.ru", 2, NewsConstants.UpdateHours, @"http://modanews.ru/rss/news/", NewsProviderType.Fashion),
new NewsProvider(@"glianec.com.u", 2, NewsConstants.UpdateHours, @"http://glianec.com.ua/?format=feed&type=rss", NewsProviderType.Fashion),
new NewsProvider(@"fashiontime.ru/main", 2, NewsConstants.UpdateHours, @"http://www.fashiontime.ru/rss/content.xml", NewsProviderType.Fashion),
new NewsProvider(@"fashiontime.ru/news", 2, NewsConstants.UpdateHours, @"http://www.fashiontime.ru/rss/content_news.xml", NewsProviderType.Fashion),
new NewsProvider(@"fashiontime.ru/articles", 2, NewsConstants.UpdateHours, @"http://www.fashiontime.ru/rss/content_article.xml", NewsProviderType.Fashion),
new NewsProvider(@"intermoda.ru", 2, NewsConstants.UpdateHours, @"http://www.intermoda.ru/rss/news", NewsProviderType.Fashion),
new NewsProvider(@"profashion.ru", 2, NewsConstants.UpdateHours, @"http://profashion.ru/news/rss/", NewsProviderType.Fashion),
new NewsProvider(@"fashionunited.ru", 2, NewsConstants.UpdateHours, @"https://fashionunited.ru/rss-novosti", NewsProviderType.Fashion),
new NewsProvider(@"Style.rbc.ru", 2, NewsConstants.UpdateHours, @"http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/style.rbc.ru/popural.rss", NewsProviderType.Fashion),
new NewsProvider(@"be-in.ru", 2, NewsConstants.UpdateHours, @"http://www.be-in.ru/rss/moda", NewsProviderType.Fashion),
new NewsProvider(@"dress-code.com.ua", 2, NewsConstants.UpdateHours, @"http://www.dress-code.com.ua/rss/common.rss", NewsProviderType.Fashion),
new NewsProvider(@"trendymen.ru", 2, NewsConstants.UpdateHours, @"http://trendymen.ru/_services/rss.php", NewsProviderType.Fashion),
new NewsProvider(@"fashionpeople.ru", 2, NewsConstants.UpdateHours, @"http://www.fashionpeople.ru/xml/rss.xml", NewsProviderType.Fashion),
// Tech sites
new NewsProvider(@"3dnews.ru", 2, NewsConstants.UpdateHours, @"http://www.3dnews.ru/news/rss"),
//new NewsProvider(@"popmech.ru", 2, NewsConstants.UpdateHours, @"http://www.popmech.ru/rss"),
new NewsProvider(@"mobbit", 2, NewsConstants.UpdateHours, @"http://mobbit.info/rss.php"),
new NewsProvider(@"ixbt.co", 2, NewsConstants.UpdateHours, @"http://www.ixbt.com/export/articles.rdf"),
new NewsProvider(@"mobile-review", 2, NewsConstants.UpdateHours, @"http://www.mobile-review.com/newslistouter/rssnewsfull.xml"),
new NewsProvider(@"mobile-review-reviews", 2, NewsConstants.UpdateHours, @"http://www.mobile-review.com/rss-review.xml"),
new NewsProvider(@"tech", 2, NewsConstants.UpdateHours, @"https://www.google.com/news?output=rss&topic=t&ned=ru_ru&cf=all&num=20"),
new NewsProvider(@"astronet.ru", 2, NewsConstants.UpdateHours, @"http://www.astronet.ru/db/rss.xml"),
new NewsProvider(@"nkj.ru", 2, NewsConstants.UpdateHours, @"http://www.nkj.ru/rss/iblock_rss_31.xml"),
new NewsProvider(@"elementy.r", 2, NewsConstants.UpdateHours, @"http://elementy.ru/rss/news"),
new NewsProvider(@"postnauka.ru", 2, NewsConstants.UpdateHours, @"http://postnauka.ru/feed"),
new NewsProvider(@"habrahabr-startups", 2, NewsConstants.UpdateHours, @"http://habrahabr.ru/rss/tag/%D1%81%D1%82%D0%B0%D1%80%D1%82%D0%B0%D0%BF%D1%8B"), // стартапы
new NewsProvider(@"habrahabr", 2, NewsConstants.UpdateHours, @"http://www.habrahabr.ru/rss/main/"), // feed/http://habrahabr.ru/rss/best/
new NewsProvider(@"geektimes.ru", 2, NewsConstants.UpdateHours, @"http://geektimes.ru/rss/all/"), // http://geektimes.ru/rss/best/
//new NewsProvider(@"", 2, NewsConstants.UpdateHours, @""),
new NewsProvider(@"The Verge", 1, NewsConstants.UpdateHours, @"http://www.theverge.com/rss/full.xml"),
new NewsProvider(@"Engadget", 1, NewsConstants.UpdateHours, @"http://www.engadget.com/rss-full.xml"),
new NewsProvider(@"Lifehacker", 1, NewsConstants.UpdateHours, @"http://feeds.gawker.com/lifehacker/vip"), // Lifehacker
//new NewsProvider(@"readwriteweb", 1, NewsConstants.UpdateHours, @"http://www.readwriteweb.com/rss.xml"),
new NewsProvider(@"Techcrunch", 1, NewsConstants.UpdateHours, @"http://feeds.feedburner.com/Techcrunch"), //Techcrunch
new NewsProvider(@"Wired", 1, NewsConstants.UpdateHours, @"http://feeds.wired.com/wired/index"),
new NewsProvider(@"Gizmodo", 1, NewsConstants.UpdateHours, @"http://feeds.gawker.com/gizmodo/full"), // Gizmodo
new NewsProvider(@"Mashable", 1, NewsConstants.UpdateHours, @"http://feeds.mashable.com/Mashable"),
new NewsProvider(@"arstechnica", 1, NewsConstants.UpdateHours, @"http://feeds.arstechnica.com/arstechnica/index/"),
new NewsProvider(@"CNET", 1, NewsConstants.UpdateHours, @"http://news.com.com/2547-1_3-0-5.xml"),
new NewsProvider(@"Slashdot", 1, NewsConstants.UpdateHours, @"http://rss.slashdot.org/Slashdot/slashdot"),
new NewsProvider(@"Google Blog", 1, NewsConstants.UpdateHours, @"http://googleblog.blogspot.com/atom.xml"),
new NewsProvider(@"The Next Web", 1, NewsConstants.UpdateHours, @"http://feeds2.feedburner.com/thenextweb"),
new NewsProvider(@"BoingBoing", 1, NewsConstants.UpdateHours, @"http://boingboing.net/feed"),
new NewsProvider(@"ThinkGeek", 1, NewsConstants.UpdateHours, @"http://www.thinkgeek.com/thinkgeek.rss"),
new NewsProvider(@"TED Talks", 1, NewsConstants.UpdateHours, @"http://feeds.feedburner.com/tedtalks_video"),
new NewsProvider(@"YCombinator", 1, NewsConstants.UpdateHours, @"https://news.ycombinator.com/rss"),
//new NewsProvider(@"", 1, NewsConstants.UpdateHours, @""),
};
if (_dataStore.IsNeedToUpdate(NewsConstants.UpdateHours))
{
Dictionary <string, List <News> > news = _newsAggregator.GetNews(providers);
if (news != null)
{
foreach (string providerName in news.Keys)
{
NewsProvider providerInfo = providers.FirstOrDefault(x => x.ProviderName == providerName);
foreach (var item in news[providerName])
{
if (IsNewsAcceptable(item, providerInfo))
{
item.ThemeOrCategory = DetectTheme(item);
InsertNewsItem(
ModelAdapter.ConvertNewsToBlog(item, providerInfo));
}
}
}
}
}
}
catch (Exception ex)
{
_logger.Error("Exception UpdateNews", ex);
}
}
