public ActionResult Create(FormCollection collection) { try { var model = new NewsEntity.Models.Article(); model.Content = collection.Get("Content"); model.Title = collection.Get("Title"); model.Anons = collection.Get("Anons"); model.Keywords = collection.Get("Keywords"); model.Author = collection.Get("Author"); model.Displayed_Days = Convert.ToInt16(collection.Get("Displayed_Days")); model.Published_At = null; model.Source_Url = Guid.NewGuid().ToString(); model.Category = Convert.ToInt32(collection.Get("Category")); model.Save(); string notice = "Публикация " + model.Title + " успешна создана"; return(RedirectToAction("Index", new { error = "", notice = notice })); } catch (Exception ex) { string err = ex.Message; if (ex.InnerException != null) { err += ": " + ex.InnerException.Message; } return(RedirectToAction("Index", "Article", new { error = err, notice = "" })); } }
private void GrabberNews(string urlNews, string urlImage) { try { string urlSite = "http://primpogoda.ru"; string urlAddress = urlSite + urlNews; HttpWebRequest request = (HttpWebRequest)WebRequest.Create(urlAddress); HttpWebResponse response = (HttpWebResponse)request.GetResponse(); if (response.StatusCode == HttpStatusCode.OK) { Stream receiveStream = response.GetResponseStream(); StreamReader readStream = null; if (response.CharacterSet == null) { readStream = new StreamReader(receiveStream); } else { readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet)); } string data = readStream.ReadToEnd(); response.Close(); readStream.Close(); var doc = new HtmlAgilityPack.HtmlDocument(); HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty; doc.LoadHtml(data); foreach (HtmlNode a in doc.DocumentNode.SelectNodes("//a[@href]")) { var TextTag = HtmlNode.CreateNode(a.InnerText); a.ParentNode.ReplaceChild(TextTag, a); } string xpathDivSelector = "//div[@class='news-detail']"; var tagNewsDetail = doc.DocumentNode.SelectSingleNode(xpathDivSelector); if (tagNewsDetail == null) { throw new Exception("Не обнаружен тег с классом news-detail"); } tagNewsDetail.RemoveChild(tagNewsDetail.FirstChild); Log(tagNewsDetail.InnerHtml); var tagImgNewsPhoto = tagNewsDetail.SelectSingleNode("//img[@class='news_photo']"); if (tagImgNewsPhoto != null) { tagImgNewsPhoto.Attributes["src"].Value = urlImage; } var tagList = tagNewsDetail.SelectNodes("//p"); int MaxAnons = 1; int indexAnons = 0; string newsAnons = ""; foreach (var p in tagList) { indexAnons++; if (indexAnons <= MaxAnons) { newsAnons += "<p>" + p.InnerHtml + "</p>"; } } //var query = tagNewsDetail.Descendants("a"); //foreach (var a in query.ToList()) //{ // var TextTag = HtmlNode.CreateNode(a.InnerText); // a.ParentNode.ReplaceChild(TextTag, a); //} var tagH6Date = tagNewsDetail.SelectSingleNode("//h6[@class='date']"); if (tagH6Date == null) { throw new Exception("Не обнаружен тег h6 с классом date"); } string strDate = tagH6Date.InnerText; DateTime dt = DateTime.ParseExact(strDate, "dd.MM.yyyy HH:mm", System.Globalization.CultureInfo.InvariantCulture); if (NewsEntity.Models.Article.GetBySource(urlAddress) == null) { NewsEntity.Models.Article theArticle = new NewsEntity.Models.Article(); theArticle.Source_Url = urlAddress; theArticle.Source_Site = urlSite; theArticle.Source_Published_At = dt; theArticle.Title = tagNewsDetail.FirstChild.InnerText; tagNewsDetail.RemoveChild(tagNewsDetail.FirstChild); // Удалить заголовок theArticle.Content = tagNewsDetail.InnerHtml; theArticle.Anons = newsAnons; theArticle.Category = 2; theArticle.Save(); } } } catch (Exception ex) { Log("GrabberNews: Error: " + urlNews); string err = ex.Message + "\n" + ex.StackTrace; if (ex.InnerException != null) { err += ex.InnerException.Message + "\n" + ex.InnerException.StackTrace; } Log(err); } }
void IGrabber.Run() { try { Log("GrabberKhabmeteoHydrology: Run"); HttpWebRequest request = (HttpWebRequest)WebRequest.Create("http://khabmeteo.ru/cgi-bin/gidrolog.cgi"); StreamReader reader = new StreamReader(request.GetResponse().GetResponseStream(), Encoding.GetEncoding(1251)); string data = reader.ReadToEnd(); reader.Close(); var doc = new HtmlAgilityPack.HtmlDocument(); HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty; doc.LoadHtml(data); string xpathDivSelector = "//div[@id='content']"; var tagContent = doc.DocumentNode.SelectSingleNode(xpathDivSelector); if (tagContent == null) { throw new Exception("Не обнаружен тег div id=content"); } var tagAnons = tagContent.FirstChild; if (tagAnons == null) { throw new Exception("Не обнаружен тег Anons"); } string Text = tagAnons.InnerText.Trim(); string Content = ""; foreach (var item in tagContent.ChildNodes) { if (item.Name == "#text") { Content += item.InnerText.Trim() + " "; } } var hydroList = NewsEntity.Models.Article.GetBySearch(Content); if (hydroList.Count == 0) { var hydroArticle = new NewsEntity.Models.Article(); hydroArticle.Anons = Text; hydroArticle.Content = Content; hydroArticle.Published_At = DateTime.Now; hydroArticle.Source_Published_At = DateTime.Now; hydroArticle.Source_Site = "khabmeteo.ru"; hydroArticle.Source_Url = "http://khabmeteo.ru/cgi-bin/gidrolog.cgi|?tmp=" + DateTime.Now.Ticks.ToString(); hydroArticle.Title = "Гидрологическая обстановка за " + DateTime.Now.ToShortDateString(); hydroArticle.Category = 5; hydroArticle.Save(); } } catch (Exception ex) { string err = ex.Message + "\n" + ex.StackTrace; if (ex.InnerException != null) { err += ex.InnerException.Message + "\n" + ex.InnerException.StackTrace; } Log(err); } }
private void GrabberNews(string urlNews) { try { string urlSite = "https://khabkrai.ru"; string urlAddress = urlSite + urlNews; HttpWebRequest request = (HttpWebRequest)WebRequest.Create(urlAddress); HttpWebResponse response = (HttpWebResponse)request.GetResponse(); if (response.StatusCode == HttpStatusCode.OK) { Stream receiveStream = response.GetResponseStream(); StreamReader readStream = null; if (response.CharacterSet == null) { readStream = new StreamReader(receiveStream); } else { readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet)); } string data = readStream.ReadToEnd(); response.Close(); readStream.Close(); var doc = new HtmlAgilityPack.HtmlDocument(); HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty; doc.LoadHtml(data); string xpathDivSelector = "//span[@class='wrapper']"; var tagNewsWrapper = doc.DocumentNode.SelectSingleNode(xpathDivSelector); if (tagNewsWrapper == null) { throw new Exception("Не обнаружен тег span с классом wraper"); } string Title = tagNewsWrapper.InnerText; var tagContent = doc.DocumentNode.SelectSingleNode("//div[@class='content-text']"); if (tagContent == null) { throw new Exception("Не обнаружен тег div с классом content-text"); } var tagWatchDateTime = doc.DocumentNode.SelectSingleNode("//div[@class='material-date data-item']"); string watchDateTimeString = tagWatchDateTime.InnerText.Trim().Replace(" ", " "); string[] months = { "января", "февраля", "марта", "апреля", "мая", "июня", "июля", "августа", "сентября", "октября", "ноября", "декабря" }; int i = 0; foreach (var month in months) { i++; watchDateTimeString = watchDateTimeString.Replace(month, i.ToString("00")); } DateTime watchDateTime = DateTime.ParseExact(watchDateTimeString, "dd MM yyyy", System.Globalization.CultureInfo.InvariantCulture); NewsEntity.Models.Article theNews = NewsEntity.Models.Article.GetBySource(urlAddress); if (theNews == null) { theNews = new NewsEntity.Models.Article(); theNews.Title = Title; tagContent.RemoveChild(tagContent.LastChild); theNews.Content = tagContent.InnerHtml + "<p>Пресс-служба Губернатора и Правительства Хабаровского края www.khabkrai.ru</p>"; theNews.Anons = tagContent.FirstChild.InnerText; theNews.Source_Published_At = watchDateTime; theNews.Source_Site = urlSite; theNews.Source_Url = urlAddress; theNews.Category = 6; theNews.Published_At = DateTime.Now; theNews.Save(); } } } catch (Exception ex) { Log("GrabberNews: Error: " + urlNews); string err = ex.Message + "\n" + ex.StackTrace; if (ex.InnerException != null) { err += ex.InnerException.Message + "\n" + ex.InnerException.StackTrace; } Log(err); } }
private void GrabberNews(string urlNews) { try { string urlSite = "http://dvrcpod.ru/"; string urlAddress = urlSite + urlNews; HttpWebRequest request = (HttpWebRequest)WebRequest.Create(urlAddress); StreamReader reader = new StreamReader(request.GetResponse().GetResponseStream(), Encoding.GetEncoding(1251)); string data = reader.ReadToEnd(); reader.Close(); var doc = new HtmlAgilityPack.HtmlDocument(); HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty; doc.LoadHtml(data); string xpathDivSelector = "//div[@class='article']"; var tagArticle = doc.DocumentNode.SelectSingleNode(xpathDivSelector); if (tagArticle == null) { throw new Exception("Не обнаружен тег div class=article"); } string Title = tagArticle.FirstChild.InnerText; tagArticle.RemoveChild(tagArticle.FirstChild); string Date = tagArticle.FirstChild.InnerText; DateTime dt = DateTime.ParseExact(Date, "yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture); tagArticle.RemoveChild(tagArticle.FirstChild); string Anons = tagArticle.FirstChild.InnerText; string Content = tagArticle.InnerHtml; NewsEntity.Models.Article theArticle = NewsEntity.Models.Article.GetBySource(urlAddress); if (theArticle == null) { theArticle = new NewsEntity.Models.Article(); theArticle.Title = Title; theArticle.Anons = Anons; theArticle.Content = Content; theArticle.Source_Published_At = dt; theArticle.Source_Site = urlSite; theArticle.Source_Url = urlAddress; theArticle.Category = 2; theArticle.Save(); } } catch (Exception ex) { Log("GrabberNews: Error: " + urlNews); string err = ex.Message + "\n" + ex.StackTrace; if (ex.InnerException != null) { err += ex.InnerException.Message + "\n" + ex.InnerException.StackTrace; } Log(err); } }
private void GrabberNews(string urlNews) { try { string urlSite = "http://geo-storm.ru"; string urlAddress = urlSite + urlNews; HttpWebRequest request = (HttpWebRequest)WebRequest.Create(urlAddress); HttpWebResponse response = (HttpWebResponse)request.GetResponse(); if (response.StatusCode == HttpStatusCode.OK) { Stream receiveStream = response.GetResponseStream(); StreamReader readStream = null; if (response.CharacterSet == null) { readStream = new StreamReader(receiveStream); } else { readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet)); } string data = readStream.ReadToEnd(); response.Close(); readStream.Close(); var doc = new HtmlAgilityPack.HtmlDocument(); HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty; doc.LoadHtml(data); string xpathDivSelector = "//div[@class='content']"; var tagNewsWrapper = doc.DocumentNode.SelectSingleNode(xpathDivSelector); if (tagNewsWrapper == null) { throw new Exception("Не обнаружен тег div с классом content"); } tagNewsWrapper.RemoveChild(tagNewsWrapper.FirstChild); tagNewsWrapper.RemoveChild(tagNewsWrapper.FirstChild); tagNewsWrapper.RemoveChild(tagNewsWrapper.FirstChild); string Title = tagNewsWrapper.FirstChild.InnerText; var tagContent = doc.DocumentNode.SelectSingleNode("//div[@class='banons b']"); if (tagContent == null) { throw new Exception("Не обнаружен тег div с классом banons b"); } var watchDateTimeString = tagContent.FirstChild.InnerText; DateTime watchDateTime = DateTime.ParseExact(watchDateTimeString, "dd.MM.yyyy", System.Globalization.CultureInfo.InvariantCulture); tagContent.RemoveChild(tagContent.FirstChild); tagContent.RemoveChild(tagContent.FirstChild); tagContent.FirstChild.Attributes["href"].Value = urlSite + tagContent.FirstChild.Attributes["href"].Value; var tagImg = tagContent.FirstChild.FirstChild; tagImg.Attributes["src"].Value = urlSite + tagImg.Attributes["src"].Value; tagContent.RemoveChild(tagContent.FirstChild); tagContent.InsertBefore(tagImg, tagContent.FirstChild); NewsEntity.Models.Article theNews = NewsEntity.Models.Article.GetBySource(urlAddress); if (theNews == null) { theNews = new NewsEntity.Models.Article(); theNews.Title = Title; tagContent.RemoveChild(tagContent.LastChild); tagContent.RemoveChild(tagContent.LastChild); tagContent.RemoveChild(tagContent.LastChild); tagContent.RemoveChild(tagContent.LastChild); tagContent.RemoveChild(tagContent.LastChild); tagContent.RemoveChild(tagContent.LastChild); tagContent.RemoveChild(tagContent.LastChild); theNews.Content = tagContent.InnerHtml; tagContent.RemoveChild(tagContent.FirstChild); theNews.Anons = tagContent.FirstChild.InnerText; theNews.Source_Published_At = watchDateTime; theNews.Source_Site = urlSite; theNews.Source_Url = urlAddress; theNews.Category = 999; theNews.Published_At = DateTime.Now; theNews.Save(); } } } catch (Exception ex) { Log("GrabberNews: Error: " + urlNews); string err = ex.Message + "\n" + ex.StackTrace; if (ex.InnerException != null) { err += ex.InnerException.Message + "\n" + ex.InnerException.StackTrace; } Log(err); } }