public async Task <ActionResult> RssManage(string id, string siteid)
{
var siteModel = await ContentManagerRepository.SiteInfo(siteid);
var rssModel = await ContentManagerRepository.RssInfo(id);
var model = new ViewModel.ContentManager.RssVM()
{
sitetitle = siteModel.title,
siteid = siteModel._id,
categories = rssModel.categories,
tags = rssModel.tags,
lang = rssModel.lang,
title = rssModel.title,
url = rssModel.url,
userid = rssModel.userid,
_id = rssModel._id
};
return(View(model));
}
public async Task <ActionResult> Manage(string rssid)
{
var rssModel = await ContentManagerRepository.RssInfo(rssid);
var userid = Util.Common.My.id;
var UserAllRss = await ContentManagerRepository.GetUserAllRss(userid);
var UserAllSite = await ContentManagerRepository.GetUserAllSite(userid);
var UserCategories = UserAllRss.SelectMany(s => s.categories).Distinct().ToList();
var UserTags = UserAllRss.SelectMany(s => s.tags).Distinct().ToList();
return(View(new ViewModel.BaseContent.BaseContentManageVM()
{
rssid = rssModel._id,
rsstitle = rssModel.title,
categories = UserCategories,
rsses = UserAllRss,
sites = UserAllSite,
tags = UserTags,
}));
}
public async Task <bool> Crawler(Models.BaseContent.BaseContent model)
{
try
{
string htmlContent = "";
var uri = new Uri(model.url);
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0");
client.DefaultRequestHeaders.Add("Host", uri.Authority);
using (var r = await client.GetAsync(uri))
{
htmlContent = await r.Content.ReadAsStringAsync();
}
}
var parser = new HtmlParser();
var document = parser.Parse(htmlContent);
var rssModel = await ContentManagerRepository.RssInfo(model.rssid);
var SiteModel = await ContentManagerRepository.SiteInfo(rssModel.siteid);
//var template = Util.Common.fromJSON<Models.Crawl.CrawlTemplate>(System.IO.File.ReadAllText(HttpContext.Current.Server.MapPath("~/Models/simplecrawltemplate.json")));
//var template = Util.Common.fromJSON<Models.Crawl.CrawlTemplate>(System.IO.File.ReadAllText(HostingEnvironment.MapPath("~/Models/simplecrawltemplate.json")));
var template = SiteModel.template.FirstOrDefault();
if (template != null)
{
var content = new Models.Content.NewsContent();
content.rssid = model.rssid;
content.userid = model.userid;
content.contenttype = (int)Util.Configuration.ContentType.News;
content.basecontentid = model._id;
content.url = model.url;
foreach (var item in template.structure)
{
var element = document.QuerySelector(item.query);
var elementcontent = "";
if (element != null && !string.IsNullOrWhiteSpace(item.query))
{
switch (item.type)
{
case "innerhtml":
elementcontent = Util.Common.CleanHtmlContent(element.InnerHtml);
break;
case "src":
elementcontent = new Uri(uri, element.GetAttribute(item.type)).ToString();
break;
default:
break;
}
try
{
content.GetType().GetProperty(item.field).SetValue(content, elementcontent, null);
}
catch
{
}
}
}
var result = await ContentRepository.AddContent(content);
if (result)
{
await baseContentRepository.ChangeIsCrawled(model);
}
return(result);
}
return(false);
}
catch (Exception e)
{
return(false);
}
}
public async Task <ViewModel.BaseContent.BaseContentManageServiceResult> PostManage([FromBody] ViewModel.BaseContent.BaseContentSearchVM simpleSearch)
{
try
{
var page = simpleSearch.page < 1 ? 1 : simpleSearch.page;
var limit = 20;
var skip = limit * (page - 1);
var rssids = new List <string>();
var contentManagerRepository = new Repository.ContentManagerRepository();
var _userRssList = await contentManagerRepository.GetUserAllRss(Util.Common.My.id);
if (_userRssList.Any())
{
if (simpleSearch.categories != null && simpleSearch.categories.Any() && !simpleSearch.categories.Any(s => s == "-1"))
{
_userRssList = _userRssList.Where(s => s.categories.Any(x => simpleSearch.categories.Contains(x))).ToList();
}
if (simpleSearch.tags != null && simpleSearch.tags.Any() && !simpleSearch.tags.Any(s => s == "-1"))
{
_userRssList = _userRssList.Where(s => s.tags.Any(x => simpleSearch.tags.Contains(x))).ToList();
}
if (!string.IsNullOrWhiteSpace(simpleSearch.siteid) && simpleSearch.siteid != "-1")
{
_userRssList = _userRssList.Where(s => s.siteid == simpleSearch.siteid).ToList();
}
if (!string.IsNullOrWhiteSpace(simpleSearch.rssid) && simpleSearch.rssid != "-1")
{
_userRssList = _userRssList.Where(s => s._id == simpleSearch.rssid).ToList();
}
rssids = _userRssList.Select(s => s._id).ToList();
}
string q = "{aggregate:'basecontent',pipeline:[{$match:{ 'rssid':{$in:" + rssids.toJSON() + "}}},{$sort : { 'dateticks' : -1 }},{$skip:" + skip.ToString() + "},{$limit:" + limit.ToString() + "}]}";
var _model = await NoSql.Instance.RunCommandAsync <BsonDocument>(q);
var model0 = new List <Models.BaseContent.BaseContent>();
foreach (var item in _model.GetValue("result").AsBsonArray)
{
model0.Add(MongoDB.Bson.Serialization.BsonSerializer.Deserialize <Models.BaseContent.BaseContent>(item.AsBsonDocument));
}
var model = new List <ViewModel.BaseContent.SimpleBaseContentManageVM>();
foreach (var item in model0)
{
var rssModel = await ContentManagerRepository.RssInfo(item.rssid);
var siteModel = await ContentManagerRepository.SiteInfo(rssModel.siteid);
model.Add(new ViewModel.BaseContent.SimpleBaseContentManageVM()
{
dateticks = item.dateticks,
description = item.description,
insertdateticks = item.insertdateticks,
pdate = Common.ConvertG2JDateText(new DateTime(item.dateticks), true),
pinsertdate = Common.ConvertG2JDateText(new DateTime(item.insertdateticks), true),
rssid = item.rssid,
rsstitle = rssModel.title,
sitetitle = siteModel.title,
title = item.title,
url = item.url,
userid = item.userid,
_id = item._id,
});
}
var result = new ViewModel.BaseContent.BaseContentManageServiceResult();
result.Data = model;
result.CurrentPage = page;
result.PrevPage = (page == 1 ? 2 : page) - 1;
result.NextPage = page + 1;
result.ServiceResultStatus = (int)Util.Common.ServiceResultStatus.OK;
return(result);
}
catch (Exception ex)
{
return(new ViewModel.BaseContent.BaseContentManageServiceResult()
{
ServiceResultStatus = (int)Util.Common.ServiceResultStatus.Error,
ServiceResultMassage = ex.GetBaseException().Message
});
}
}
