public async Task <ActionResult> EditSite(string id) { var SiteInfo = await ContentManagerRepository.SiteInfo(id); var model = new ViewModel.ContentManager.SiteVM(); if (!string.IsNullOrWhiteSpace(id)) { model = new ViewModel.ContentManager.SiteVM { title = SiteInfo.title, url = SiteInfo.url, _id = SiteInfo._id, userid = SiteInfo.userid, createdateticks = SiteInfo.createdateticks } } ; return(View(model)); }
public async Task <bool> Crawler(Models.BaseContent.BaseContent model) { try { string htmlContent = ""; var uri = new Uri(model.url); using (var client = new HttpClient()) { client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0"); client.DefaultRequestHeaders.Add("Host", uri.Authority); using (var r = await client.GetAsync(uri)) { htmlContent = await r.Content.ReadAsStringAsync(); } } var parser = new HtmlParser(); var document = parser.Parse(htmlContent); var rssModel = await ContentManagerRepository.RssInfo(model.rssid); var SiteModel = await ContentManagerRepository.SiteInfo(rssModel.siteid); //var template = Util.Common.fromJSON<Models.Crawl.CrawlTemplate>(System.IO.File.ReadAllText(HttpContext.Current.Server.MapPath("~/Models/simplecrawltemplate.json"))); //var template = Util.Common.fromJSON<Models.Crawl.CrawlTemplate>(System.IO.File.ReadAllText(HostingEnvironment.MapPath("~/Models/simplecrawltemplate.json"))); var template = SiteModel.template.FirstOrDefault(); if (template != null) { var content = new Models.Content.NewsContent(); content.rssid = model.rssid; content.userid = model.userid; content.contenttype = (int)Util.Configuration.ContentType.News; content.basecontentid = model._id; content.url = model.url; foreach (var item in template.structure) { var element = document.QuerySelector(item.query); var elementcontent = ""; if (element != null && !string.IsNullOrWhiteSpace(item.query)) { switch (item.type) { case "innerhtml": elementcontent = Util.Common.CleanHtmlContent(element.InnerHtml); break; case "src": elementcontent = new Uri(uri, element.GetAttribute(item.type)).ToString(); break; default: break; } try { content.GetType().GetProperty(item.field).SetValue(content, elementcontent, null); } catch { } } } var result = await ContentRepository.AddContent(content); if (result) { await baseContentRepository.ChangeIsCrawled(model); } return(result); } return(false); } catch (Exception e) { return(false); } }
public async Task <ViewModel.BaseContent.BaseContentManageServiceResult> PostManage([FromBody] ViewModel.BaseContent.BaseContentSearchVM simpleSearch) { try { var page = simpleSearch.page < 1 ? 1 : simpleSearch.page; var limit = 20; var skip = limit * (page - 1); var rssids = new List <string>(); var contentManagerRepository = new Repository.ContentManagerRepository(); var _userRssList = await contentManagerRepository.GetUserAllRss(Util.Common.My.id); if (_userRssList.Any()) { if (simpleSearch.categories != null && simpleSearch.categories.Any() && !simpleSearch.categories.Any(s => s == "-1")) { _userRssList = _userRssList.Where(s => s.categories.Any(x => simpleSearch.categories.Contains(x))).ToList(); } if (simpleSearch.tags != null && simpleSearch.tags.Any() && !simpleSearch.tags.Any(s => s == "-1")) { _userRssList = _userRssList.Where(s => s.tags.Any(x => simpleSearch.tags.Contains(x))).ToList(); } if (!string.IsNullOrWhiteSpace(simpleSearch.siteid) && simpleSearch.siteid != "-1") { _userRssList = _userRssList.Where(s => s.siteid == simpleSearch.siteid).ToList(); } if (!string.IsNullOrWhiteSpace(simpleSearch.rssid) && simpleSearch.rssid != "-1") { _userRssList = _userRssList.Where(s => s._id == simpleSearch.rssid).ToList(); } rssids = _userRssList.Select(s => s._id).ToList(); } string q = "{aggregate:'basecontent',pipeline:[{$match:{ 'rssid':{$in:" + rssids.toJSON() + "}}},{$sort : { 'dateticks' : -1 }},{$skip:" + skip.ToString() + "},{$limit:" + limit.ToString() + "}]}"; var _model = await NoSql.Instance.RunCommandAsync <BsonDocument>(q); var model0 = new List <Models.BaseContent.BaseContent>(); foreach (var item in _model.GetValue("result").AsBsonArray) { model0.Add(MongoDB.Bson.Serialization.BsonSerializer.Deserialize <Models.BaseContent.BaseContent>(item.AsBsonDocument)); } var model = new List <ViewModel.BaseContent.SimpleBaseContentManageVM>(); foreach (var item in model0) { var rssModel = await ContentManagerRepository.RssInfo(item.rssid); var siteModel = await ContentManagerRepository.SiteInfo(rssModel.siteid); model.Add(new ViewModel.BaseContent.SimpleBaseContentManageVM() { dateticks = item.dateticks, description = item.description, insertdateticks = item.insertdateticks, pdate = Common.ConvertG2JDateText(new DateTime(item.dateticks), true), pinsertdate = Common.ConvertG2JDateText(new DateTime(item.insertdateticks), true), rssid = item.rssid, rsstitle = rssModel.title, sitetitle = siteModel.title, title = item.title, url = item.url, userid = item.userid, _id = item._id, }); } var result = new ViewModel.BaseContent.BaseContentManageServiceResult(); result.Data = model; result.CurrentPage = page; result.PrevPage = (page == 1 ? 2 : page) - 1; result.NextPage = page + 1; result.ServiceResultStatus = (int)Util.Common.ServiceResultStatus.OK; return(result); } catch (Exception ex) { return(new ViewModel.BaseContent.BaseContentManageServiceResult() { ServiceResultStatus = (int)Util.Common.ServiceResultStatus.Error, ServiceResultMassage = ex.GetBaseException().Message }); } }