public async Task <IActionResult> Aggregate([Bind("Id,Title,Content,Url,Rating,NewsDate,DateCollect,RssSourceId")] NewsDto news)
{
try
{
var rssSources = (await _rssSourceService.GetRssSources());
Log.Information("***Try to aggregate news");
var allSourcesNews = new List <NewsDto>();
foreach (var rssSource in rssSources)
{
Log.Information($"Aggregate from {rssSource.Name}");
var oneSourceNewsList = await _newsService.GetNewsFromRssSource(rssSource);
Log.Information($"News in rss source: {oneSourceNewsList.Count()}");
foreach (var oneSourceNews in oneSourceNewsList)
{
if (rssSource.Name == "Igromania")
{
oneSourceNews.Body = await _igromaniaParser.Parse(oneSourceNews.Url);
}
if (rssSource.Name == "Shazoo")
{
oneSourceNews.Body = await _shazooParser.Parse(oneSourceNews.Url);
}
if (rssSource.Name == "Onliner")
{
oneSourceNews.Body = await _onlinerParser.Parse(oneSourceNews.Url);
}
if (rssSource.Name == "4Pda")
{
oneSourceNews.Body = await _4PdaParser.Parse(oneSourceNews.Url);
}
if (rssSource.Name == "Wylsa")
{
oneSourceNews.Body = await _wylsaParser.Parse(oneSourceNews.Url);
}
}
allSourcesNews.AddRange(oneSourceNewsList);
Log.Information($"Finished aggregate from {rssSource.Name}");
}
Log.Information($"***Total aggregated news count {allSourcesNews.Count()}");
await _newsService.AddRange(allSourcesNews);
}
catch (Exception e)
{
Log.Error(e, $"Failed to aggregate news ");
throw;
}
return(RedirectToAction(nameof(Index), (new Guid("E3512D7D-381A-4655-8B60-584C08D9254A"))));
}
public async Task <IEnumerable <NewsInfoFromRssSourseDto> > GetNewsInfoFromRssSourse(RssSourceModel rssSourceModel)
{
var news = new List <NewsInfoFromRssSourseDto>();
var newCategories = new List <Category>();
//Use the default configuration for AngleSharp
var config = Configuration.Default;// для парсинга страницы
//Create a new context for evaluating webpages with the given config
var context = BrowsingContext.New(config);// для парсинга страницы
using (var reader = XmlReader.Create(rssSourceModel.Url))
{
var feed = SyndicationFeed.Load(reader);
reader.Close();
newCategories = await _categoryService.CheckCategoriesForDublication(feed); // чекаем дубликаты
await _categoryService.CreateManyCategories(newCategories); //после того, как мы чекнули название категорий, те, которых нет в базе туда заносятся
try
{
if (feed.Items.Any())
{
foreach (var syndicationItem in feed.Items)
{
string categoryName = null;
foreach (var category in syndicationItem.Categories)
{
categoryName = category.Name;
}
var currentNewsUrls = await _unitOfWork.News
.GetAll(false)//rssSourseId must be not nullable
.Select(n => n.Url)
.ToListAsync();
var document = await context.OpenAsync(req => req.Content(syndicationItem.Summary.Text));// для парсинга страницы
try
{
if (!currentNewsUrls.Any(url => url.Equals(syndicationItem.Id)))
{
string lastText = null;
string imageUrl = null;
if (rssSourceModel.Name.Equals("TUT.by"))
{
lastText = (await _tutByParser.Parse(syndicationItem)).NewsText;
}
else if (rssSourceModel.Name.Equals("Onliner"))
{
var fullNewsText = (await _onlinerParser.Parse(syndicationItem));
if (fullNewsText is null)
{
break;
}
lastText = fullNewsText.NewsText;
imageUrl = fullNewsText.ImageUrl;
}
else if (rssSourceModel.Name.Equals("igromania"))
{
var fullNewsText = (await _igromaniaParser.Parse(syndicationItem));
if (fullNewsText is null)
{
break;
}
lastText = fullNewsText.NewsText;
imageUrl = fullNewsText.ImageUrl;
}
else if (rssSourceModel.Name.Equals("OON"))
{
var fullNewsText = (await _OONParser.Parse(syndicationItem));
if (fullNewsText is null)
{
break;
}
lastText = fullNewsText.NewsText;
imageUrl = fullNewsText.ImageUrl;
}
var newsDto = new NewsInfoFromRssSourseDto()
{
Id = Guid.NewGuid(),
RssSourceId = rssSourceModel?.Id,
Url = syndicationItem.Id,
HeadImgUrl = imageUrl,
Title = syndicationItem.Title.Text,
Summary = document.DocumentElement.TextContent, //syndicationItem.Summary.Text, //clean from html(?)
Authors = syndicationItem.Authors.Select(x => x.Name),
Body = lastText,
CategoryId = (await _categoryService.FindCategoryByName(categoryName))?.Id
};
news.Add(newsDto);
}
}
catch (Exception ex)
{
Log.Error($"Something went wrong when trying to gave news from {rssSourceModel.Name}. Message: {ex.Message}");
}
}
}
}
catch (Exception ex)
{
Log.Error($"Something went wrong when trying to gave news from {rssSourceModel.Name}. Message: {ex.Message}");
}
}
return(news);
}
