public async Task <IActionResult> Aggregate([Bind("Id,Title,Content,Url,Rating,NewsDate,DateCollect,RssSourceId")] NewsDto news) { try { var rssSources = (await _rssSourceService.GetRssSources()); Log.Information("***Try to aggregate news"); var allSourcesNews = new List <NewsDto>(); foreach (var rssSource in rssSources) { Log.Information($"Aggregate from {rssSource.Name}"); var oneSourceNewsList = await _newsService.GetNewsFromRssSource(rssSource); Log.Information($"News in rss source: {oneSourceNewsList.Count()}"); foreach (var oneSourceNews in oneSourceNewsList) { if (rssSource.Name == "Igromania") { oneSourceNews.Body = await _igromaniaParser.Parse(oneSourceNews.Url); } if (rssSource.Name == "Shazoo") { oneSourceNews.Body = await _shazooParser.Parse(oneSourceNews.Url); } if (rssSource.Name == "Onliner") { oneSourceNews.Body = await _onlinerParser.Parse(oneSourceNews.Url); } if (rssSource.Name == "4Pda") { oneSourceNews.Body = await _4PdaParser.Parse(oneSourceNews.Url); } if (rssSource.Name == "Wylsa") { oneSourceNews.Body = await _wylsaParser.Parse(oneSourceNews.Url); } } allSourcesNews.AddRange(oneSourceNewsList); Log.Information($"Finished aggregate from {rssSource.Name}"); } Log.Information($"***Total aggregated news count {allSourcesNews.Count()}"); await _newsService.AddRange(allSourcesNews); } catch (Exception e) { Log.Error(e, $"Failed to aggregate news "); throw; } return(RedirectToAction(nameof(Index), (new Guid("E3512D7D-381A-4655-8B60-584C08D9254A")))); }
public async Task <IEnumerable <NewsInfoFromRssSourseDto> > GetNewsInfoFromRssSourse(RssSourceModel rssSourceModel) { var news = new List <NewsInfoFromRssSourseDto>(); var newCategories = new List <Category>(); //Use the default configuration for AngleSharp var config = Configuration.Default;// для парсинга страницы //Create a new context for evaluating webpages with the given config var context = BrowsingContext.New(config);// для парсинга страницы using (var reader = XmlReader.Create(rssSourceModel.Url)) { var feed = SyndicationFeed.Load(reader); reader.Close(); newCategories = await _categoryService.CheckCategoriesForDublication(feed); // чекаем дубликаты await _categoryService.CreateManyCategories(newCategories); //после того, как мы чекнули название категорий, те, которых нет в базе туда заносятся try { if (feed.Items.Any()) { foreach (var syndicationItem in feed.Items) { string categoryName = null; foreach (var category in syndicationItem.Categories) { categoryName = category.Name; } var currentNewsUrls = await _unitOfWork.News .GetAll(false)//rssSourseId must be not nullable .Select(n => n.Url) .ToListAsync(); var document = await context.OpenAsync(req => req.Content(syndicationItem.Summary.Text));// для парсинга страницы try { if (!currentNewsUrls.Any(url => url.Equals(syndicationItem.Id))) { string lastText = null; string imageUrl = null; if (rssSourceModel.Name.Equals("TUT.by")) { lastText = (await _tutByParser.Parse(syndicationItem)).NewsText; } else if (rssSourceModel.Name.Equals("Onliner")) { var fullNewsText = (await _onlinerParser.Parse(syndicationItem)); if (fullNewsText is null) { break; } lastText = fullNewsText.NewsText; imageUrl = fullNewsText.ImageUrl; } else if (rssSourceModel.Name.Equals("igromania")) { var fullNewsText = (await _igromaniaParser.Parse(syndicationItem)); if (fullNewsText is null) { break; } lastText = fullNewsText.NewsText; imageUrl = fullNewsText.ImageUrl; } else if (rssSourceModel.Name.Equals("OON")) { var fullNewsText = (await _OONParser.Parse(syndicationItem)); if (fullNewsText is null) { break; } lastText = fullNewsText.NewsText; imageUrl = fullNewsText.ImageUrl; } var newsDto = new NewsInfoFromRssSourseDto() { Id = Guid.NewGuid(), RssSourceId = rssSourceModel?.Id, Url = syndicationItem.Id, HeadImgUrl = imageUrl, Title = syndicationItem.Title.Text, Summary = document.DocumentElement.TextContent, //syndicationItem.Summary.Text, //clean from html(?) Authors = syndicationItem.Authors.Select(x => x.Name), Body = lastText, CategoryId = (await _categoryService.FindCategoryByName(categoryName))?.Id }; news.Add(newsDto); } } catch (Exception ex) { Log.Error($"Something went wrong when trying to gave news from {rssSourceModel.Name}. Message: {ex.Message}"); } } } } catch (Exception ex) { Log.Error($"Something went wrong when trying to gave news from {rssSourceModel.Name}. Message: {ex.Message}"); } } return(news); }