public ActionResult Create(FormCollection collection)
{
try
{
var model = new NewsEntity.Models.Article();
model.Content = collection.Get("Content");
model.Title = collection.Get("Title");
model.Anons = collection.Get("Anons");
model.Keywords = collection.Get("Keywords");
model.Author = collection.Get("Author");
model.Displayed_Days = Convert.ToInt16(collection.Get("Displayed_Days"));
model.Published_At = null;
model.Source_Url = Guid.NewGuid().ToString();
model.Category = Convert.ToInt32(collection.Get("Category"));
model.Save();
string notice = "Публикация " + model.Title + " успешна создана";
return(RedirectToAction("Index", new { error = "", notice = notice }));
}
catch (Exception ex)
{
string err = ex.Message;
if (ex.InnerException != null)
{
err += ": " + ex.InnerException.Message;
}
return(RedirectToAction("Index", "Article", new { error = err, notice = "" }));
}
}
private void GrabberNews(string urlNews, string urlImage)
{
try
{
string urlSite = "http://primpogoda.ru";
string urlAddress = urlSite + urlNews;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(urlAddress);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK)
{
Stream receiveStream = response.GetResponseStream();
StreamReader readStream = null;
if (response.CharacterSet == null)
{
readStream = new StreamReader(receiveStream);
}
else
{
readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet));
}
string data = readStream.ReadToEnd();
response.Close();
readStream.Close();
var doc = new HtmlAgilityPack.HtmlDocument();
HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty;
doc.LoadHtml(data);
foreach (HtmlNode a in doc.DocumentNode.SelectNodes("//a[@href]"))
{
var TextTag = HtmlNode.CreateNode(a.InnerText);
a.ParentNode.ReplaceChild(TextTag, a);
}
string xpathDivSelector = "//div[@class='news-detail']";
var tagNewsDetail = doc.DocumentNode.SelectSingleNode(xpathDivSelector);
if (tagNewsDetail == null)
{
throw new Exception("Не обнаружен тег с классом news-detail");
}
tagNewsDetail.RemoveChild(tagNewsDetail.FirstChild);
Log(tagNewsDetail.InnerHtml);
var tagImgNewsPhoto = tagNewsDetail.SelectSingleNode("//img[@class='news_photo']");
if (tagImgNewsPhoto != null)
{
tagImgNewsPhoto.Attributes["src"].Value = urlImage;
}
var tagList = tagNewsDetail.SelectNodes("//p");
int MaxAnons = 1;
int indexAnons = 0;
string newsAnons = "";
foreach (var p in tagList)
{
indexAnons++;
if (indexAnons <= MaxAnons)
{
newsAnons += "<p>" + p.InnerHtml + "</p>";
}
}
//var query = tagNewsDetail.Descendants("a");
//foreach (var a in query.ToList())
//{
// var TextTag = HtmlNode.CreateNode(a.InnerText);
// a.ParentNode.ReplaceChild(TextTag, a);
//}
var tagH6Date = tagNewsDetail.SelectSingleNode("//h6[@class='date']");
if (tagH6Date == null)
{
throw new Exception("Не обнаружен тег h6 с классом date");
}
string strDate = tagH6Date.InnerText;
DateTime dt = DateTime.ParseExact(strDate, "dd.MM.yyyy HH:mm", System.Globalization.CultureInfo.InvariantCulture);
if (NewsEntity.Models.Article.GetBySource(urlAddress) == null)
{
NewsEntity.Models.Article theArticle = new NewsEntity.Models.Article();
theArticle.Source_Url = urlAddress;
theArticle.Source_Site = urlSite;
theArticle.Source_Published_At = dt;
theArticle.Title = tagNewsDetail.FirstChild.InnerText;
tagNewsDetail.RemoveChild(tagNewsDetail.FirstChild); // Удалить заголовок
theArticle.Content = tagNewsDetail.InnerHtml;
theArticle.Anons = newsAnons;
theArticle.Category = 2;
theArticle.Save();
}
}
}
catch (Exception ex)
{
Log("GrabberNews: Error: " + urlNews);
string err = ex.Message + "\n" + ex.StackTrace;
if (ex.InnerException != null)
{
err += ex.InnerException.Message + "\n" + ex.InnerException.StackTrace;
}
Log(err);
}
}
void IGrabber.Run()
{
try
{
Log("GrabberKhabmeteoHydrology: Run");
HttpWebRequest request = (HttpWebRequest)WebRequest.Create("http://khabmeteo.ru/cgi-bin/gidrolog.cgi");
StreamReader reader = new StreamReader(request.GetResponse().GetResponseStream(), Encoding.GetEncoding(1251));
string data = reader.ReadToEnd();
reader.Close();
var doc = new HtmlAgilityPack.HtmlDocument();
HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty;
doc.LoadHtml(data);
string xpathDivSelector = "//div[@id='content']";
var tagContent = doc.DocumentNode.SelectSingleNode(xpathDivSelector);
if (tagContent == null)
{
throw new Exception("Не обнаружен тег div id=content");
}
var tagAnons = tagContent.FirstChild;
if (tagAnons == null)
{
throw new Exception("Не обнаружен тег Anons");
}
string Text = tagAnons.InnerText.Trim();
string Content = "";
foreach (var item in tagContent.ChildNodes)
{
if (item.Name == "#text")
{
Content += item.InnerText.Trim() + " ";
}
}
var hydroList = NewsEntity.Models.Article.GetBySearch(Content);
if (hydroList.Count == 0)
{
var hydroArticle = new NewsEntity.Models.Article();
hydroArticle.Anons = Text;
hydroArticle.Content = Content;
hydroArticle.Published_At = DateTime.Now;
hydroArticle.Source_Published_At = DateTime.Now;
hydroArticle.Source_Site = "khabmeteo.ru";
hydroArticle.Source_Url = "http://khabmeteo.ru/cgi-bin/gidrolog.cgi|?tmp=" + DateTime.Now.Ticks.ToString();
hydroArticle.Title = "Гидрологическая обстановка за " + DateTime.Now.ToShortDateString();
hydroArticle.Category = 5;
hydroArticle.Save();
}
}
catch (Exception ex)
{
string err = ex.Message + "\n" + ex.StackTrace;
if (ex.InnerException != null)
{
err += ex.InnerException.Message + "\n" + ex.InnerException.StackTrace;
}
Log(err);
}
}
private void GrabberNews(string urlNews)
{
try
{
string urlSite = "https://khabkrai.ru";
string urlAddress = urlSite + urlNews;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(urlAddress);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK)
{
Stream receiveStream = response.GetResponseStream();
StreamReader readStream = null;
if (response.CharacterSet == null)
{
readStream = new StreamReader(receiveStream);
}
else
{
readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet));
}
string data = readStream.ReadToEnd();
response.Close();
readStream.Close();
var doc = new HtmlAgilityPack.HtmlDocument();
HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty;
doc.LoadHtml(data);
string xpathDivSelector = "//span[@class='wrapper']";
var tagNewsWrapper = doc.DocumentNode.SelectSingleNode(xpathDivSelector);
if (tagNewsWrapper == null)
{
throw new Exception("Не обнаружен тег span с классом wraper");
}
string Title = tagNewsWrapper.InnerText;
var tagContent = doc.DocumentNode.SelectSingleNode("//div[@class='content-text']");
if (tagContent == null)
{
throw new Exception("Не обнаружен тег div с классом content-text");
}
var tagWatchDateTime = doc.DocumentNode.SelectSingleNode("//div[@class='material-date data-item']");
string watchDateTimeString = tagWatchDateTime.InnerText.Trim().Replace(" ", " ");
string[] months = { "января", "февраля", "марта", "апреля", "мая", "июня", "июля", "августа", "сентября", "октября", "ноября", "декабря" };
int i = 0;
foreach (var month in months)
{
i++;
watchDateTimeString = watchDateTimeString.Replace(month, i.ToString("00"));
}
DateTime watchDateTime = DateTime.ParseExact(watchDateTimeString, "dd MM yyyy", System.Globalization.CultureInfo.InvariantCulture);
NewsEntity.Models.Article theNews = NewsEntity.Models.Article.GetBySource(urlAddress);
if (theNews == null)
{
theNews = new NewsEntity.Models.Article();
theNews.Title = Title;
tagContent.RemoveChild(tagContent.LastChild);
theNews.Content = tagContent.InnerHtml + "<p>Пресс-служба Губернатора и Правительства Хабаровского края www.khabkrai.ru</p>";
theNews.Anons = tagContent.FirstChild.InnerText;
theNews.Source_Published_At = watchDateTime;
theNews.Source_Site = urlSite;
theNews.Source_Url = urlAddress;
theNews.Category = 6;
theNews.Published_At = DateTime.Now;
theNews.Save();
}
}
}
catch (Exception ex)
{
Log("GrabberNews: Error: " + urlNews);
string err = ex.Message + "\n" + ex.StackTrace;
if (ex.InnerException != null)
{
err += ex.InnerException.Message + "\n" + ex.InnerException.StackTrace;
}
Log(err);
}
}
private void GrabberNews(string urlNews)
{
try
{
string urlSite = "http://dvrcpod.ru/";
string urlAddress = urlSite + urlNews;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(urlAddress);
StreamReader reader = new StreamReader(request.GetResponse().GetResponseStream(), Encoding.GetEncoding(1251));
string data = reader.ReadToEnd();
reader.Close();
var doc = new HtmlAgilityPack.HtmlDocument();
HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty;
doc.LoadHtml(data);
string xpathDivSelector = "//div[@class='article']";
var tagArticle = doc.DocumentNode.SelectSingleNode(xpathDivSelector);
if (tagArticle == null)
{
throw new Exception("Не обнаружен тег div class=article");
}
string Title = tagArticle.FirstChild.InnerText;
tagArticle.RemoveChild(tagArticle.FirstChild);
string Date = tagArticle.FirstChild.InnerText;
DateTime dt = DateTime.ParseExact(Date, "yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture);
tagArticle.RemoveChild(tagArticle.FirstChild);
string Anons = tagArticle.FirstChild.InnerText;
string Content = tagArticle.InnerHtml;
NewsEntity.Models.Article theArticle = NewsEntity.Models.Article.GetBySource(urlAddress);
if (theArticle == null)
{
theArticle = new NewsEntity.Models.Article();
theArticle.Title = Title;
theArticle.Anons = Anons;
theArticle.Content = Content;
theArticle.Source_Published_At = dt;
theArticle.Source_Site = urlSite;
theArticle.Source_Url = urlAddress;
theArticle.Category = 2;
theArticle.Save();
}
}
catch (Exception ex)
{
Log("GrabberNews: Error: " + urlNews);
string err = ex.Message + "\n" + ex.StackTrace;
if (ex.InnerException != null)
{
err += ex.InnerException.Message + "\n" + ex.InnerException.StackTrace;
}
Log(err);
}
}
private void GrabberNews(string urlNews)
{
try
{
string urlSite = "http://geo-storm.ru";
string urlAddress = urlSite + urlNews;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(urlAddress);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK)
{
Stream receiveStream = response.GetResponseStream();
StreamReader readStream = null;
if (response.CharacterSet == null)
{
readStream = new StreamReader(receiveStream);
}
else
{
readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet));
}
string data = readStream.ReadToEnd();
response.Close();
readStream.Close();
var doc = new HtmlAgilityPack.HtmlDocument();
HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty;
doc.LoadHtml(data);
string xpathDivSelector = "//div[@class='content']";
var tagNewsWrapper = doc.DocumentNode.SelectSingleNode(xpathDivSelector);
if (tagNewsWrapper == null)
{
throw new Exception("Не обнаружен тег div с классом content");
}
tagNewsWrapper.RemoveChild(tagNewsWrapper.FirstChild);
tagNewsWrapper.RemoveChild(tagNewsWrapper.FirstChild);
tagNewsWrapper.RemoveChild(tagNewsWrapper.FirstChild);
string Title = tagNewsWrapper.FirstChild.InnerText;
var tagContent = doc.DocumentNode.SelectSingleNode("//div[@class='banons b']");
if (tagContent == null)
{
throw new Exception("Не обнаружен тег div с классом banons b");
}
var watchDateTimeString = tagContent.FirstChild.InnerText;
DateTime watchDateTime = DateTime.ParseExact(watchDateTimeString, "dd.MM.yyyy", System.Globalization.CultureInfo.InvariantCulture);
tagContent.RemoveChild(tagContent.FirstChild);
tagContent.RemoveChild(tagContent.FirstChild);
tagContent.FirstChild.Attributes["href"].Value = urlSite + tagContent.FirstChild.Attributes["href"].Value;
var tagImg = tagContent.FirstChild.FirstChild;
tagImg.Attributes["src"].Value = urlSite + tagImg.Attributes["src"].Value;
tagContent.RemoveChild(tagContent.FirstChild);
tagContent.InsertBefore(tagImg, tagContent.FirstChild);
NewsEntity.Models.Article theNews = NewsEntity.Models.Article.GetBySource(urlAddress);
if (theNews == null)
{
theNews = new NewsEntity.Models.Article();
theNews.Title = Title;
tagContent.RemoveChild(tagContent.LastChild);
tagContent.RemoveChild(tagContent.LastChild);
tagContent.RemoveChild(tagContent.LastChild);
tagContent.RemoveChild(tagContent.LastChild);
tagContent.RemoveChild(tagContent.LastChild);
tagContent.RemoveChild(tagContent.LastChild);
tagContent.RemoveChild(tagContent.LastChild);
theNews.Content = tagContent.InnerHtml;
tagContent.RemoveChild(tagContent.FirstChild);
theNews.Anons = tagContent.FirstChild.InnerText;
theNews.Source_Published_At = watchDateTime;
theNews.Source_Site = urlSite;
theNews.Source_Url = urlAddress;
theNews.Category = 999;
theNews.Published_At = DateTime.Now;
theNews.Save();
}
}
}
catch (Exception ex)
{
Log("GrabberNews: Error: " + urlNews);
string err = ex.Message + "\n" + ex.StackTrace;
if (ex.InnerException != null)
{
err += ex.InnerException.Message + "\n" + ex.InnerException.StackTrace;
}
Log(err);
}
}