private static int importDirect(SpiderArticle art, SpiderImport item, ContentSection section, ContentApp app)
{
ContentPost post = new ContentPost();
post.Title = art.Title;
post.Content = art.Body;
if (art.IsPic == 1)
{
post.CategoryId = PostCategory.Img;
post.ImgLink = art.PicUrl;
}
post.SourceLink = art.Url;
post.Creator = item.Creator;
post.CreatorUrl = item.Creator.Url;
post.PageSection = section;
post.OwnerId = app.OwnerId;
post.OwnerType = app.OwnerType;
post.OwnerUrl = app.OwnerUrl;
post.AppId = app.Id;
post.insert();
return(post.Id);
}
public virtual void Edit(long id)
{
target(Update, id);
SpiderArticle article = SpiderArticle.findById(id);
bind("data", article);
}
private static bool articleExist( SpiderArticle spiderArticle )
{
logger.Info( "articleExist=" + spiderArticle.Url );
ContentPost post = ContentPost.find( "OutUrl=:url" ).set( "url", spiderArticle.Url ).first();
if (post != null) return true;
return ContentTempPost.find( "SourceLink=:url" ).set( "url", spiderArticle.Url ).first() != null;
}
public void Edit(int id)
{
target(Update, id);
SpiderArticle article = SpiderArticle.findById(id);
bind("data", article);
}
public virtual void Update(long id)
{
SpiderArticle article = SpiderArticle.findById(id);
article = ctx.PostValue(article) as SpiderArticle;
article.update();
echoRedirect(lang("opok"), to(Show, id));
}
public void Edit(int id)
{
target(Update, id);
SpiderArticle article = SpiderArticle.findById(id);
bind("data", article);
editor("spiderArticle.Body", article.Body, "480px");
}
public virtual void Show(long id)
{
SpiderArticle post = SpiderArticle.findById(id);
bind("post", post);
set("refreshPageLink", to(DoRefresh, id));
set("editLink", to(Edit, id));
}
private static List <long> beginImportPrivate(ImportState ts)
{
long id = ts.TemplateId;
SpiderImport item = SpiderImport.findById(id);
List <SpiderArticle> articles = SpiderArticle
.find("SpiderTemplateId in (" + item.DataSourceIds + ") and Id>" + item.LastImportId + " order by Id")
.list();
List <ContentSection> sections = ContentSection.find("Id in (" + item.SectionIds + ")").list();
if (sections.Count == 0)
{
throw new Exception("导入的目标section不存在");
}
ContentSection section = null;
List <long> results = new List <long>();
for (int i = 0; i < articles.Count; i++)
{
if (articleExist(articles[i]))
{
ts.Log.AppendLine("pass..." + articles[i].Title);
continue;
}
section = getNextSection(sections, section); // 均匀分散到各目标section中
ContentApp app = getApp(section);
if (item.IsApprove == 1)
{
importToTemp(articles[i], item, section, app);
}
else
{
long newArticleId = importDirect(articles[i], item, section, app);
results.Add(newArticleId);
}
ts.Log.AppendLine("导入：" + articles[i].Title);
}
if (articles.Count > 0)
{
item.LastImportId = articles[articles.Count - 1].Id;
item.update("LastImportId");
ts.Log.AppendLine("导入完毕(操作结束)");
}
else
{
ts.Log.AppendLine("没有新条目可导入(操作结束)");
}
return(results);
}
private static bool articleExist(SpiderArticle spiderArticle)
{
logger.Info("articleExist=" + spiderArticle.Url);
ContentPost post = ContentPost.find("OutUrl=:url").set("url", spiderArticle.Url).first();
if (post != null)
{
return(true);
}
return(ContentTempPost.find("SourceLink=:url").set("url", spiderArticle.Url).first() != null);
}
//检查数据库中是否已经存在此数据？
private static bool isPageExist(string pageUrl, StringBuilder sb)
{
bool isExist = false;
List <SpiderArticle> list = SpiderArticle.find("Url=:url and IsDelete=0").set("url", pageUrl).list();
if (list.Count > 0)
{
logger.Info("pass..." + pageUrl);
sb.AppendLine("pass..." + pageUrl);
isExist = true;
}
return(isExist);
}
private static int importToTemp(SpiderArticle art, SpiderImport item, ContentSection section, ContentApp app)
{
ContentTempPost post = new ContentTempPost();
post.Creator = item.Creator;
post.OwnerId = app.OwnerId;
post.OwnerType = app.OwnerType;
post.AppId = app.Id;
post.SectionId = section.Id;
post.Title = art.Title;
post.SourceLink = art.Url;
post.Content = art.Body;
post.insert();
return(post.Id);
}
public virtual void Admin()
{
String ids = ctx.PostIdList("choice");
String action = ctx.Post("action");
if (strUtil.IsNullOrEmpty(ids))
{
echoError("请先选择");
return;
}
if ("delete".Equals(action))
{
SpiderArticle.deleteBatch("Id in (" + ids + ")");
}
echoAjaxOk();
}
public virtual void DoRefresh(long id)
{
SpiderArticle post = SpiderArticle.findById(id);
StringBuilder log = new StringBuilder();
String content = new PagedDetailSpider().GetContent(post.Url, post.SpiderTemplate, log);
if (strUtil.HasText(content))
{
post.Body = content;
post.update("Body");
echoJsonMsg("刷新成功", true, "");
}
else
{
errors.Add(log.ToString().Replace(Environment.NewLine, "").Trim());
echoError();
}
}
public virtual void List(long id)
{
String condition = id > 0 ? "SpiderTemplateId=" + id : "";
set("OperationUrl", to(Admin));
String refreshCmd = "";
if (id > 0)
{
refreshCmd = string.Format("<a href=\"{0}\">现在刷新采集</a>", to(new TemplateController().DoRefresh, id));
}
set("refreshCmd", refreshCmd);
DataPage <SpiderArticle> list = SpiderArticle.findPage(condition);
IBlock block = getBlock("list");
foreach (SpiderArticle p in list.Results)
{
block.Set("post.Id", p.Id);
block.Set("post.Title", p.Title);
if (p.SpiderTemplate == null || p.SpiderTemplate.SiteName == null)
{
block.Set("post.Category", "");
block.Set("post.CategoryLink", "#");
}
else
{
block.Set("post.Category", p.SpiderTemplate.SiteName);
block.Set("post.CategoryLink", to(List, p.SpiderTemplate.Id));
}
block.Set("post.Link", to(Show, p.Id));
block.Set("post.Created", p.Created);
block.Set("post.EditUrl", to(Edit, p.Id));
block.Next();
}
set("page", list.PageBar);
}
private static void savePageDetail(DetailLink lnk, StringBuilder sb)
{
SpiderTemplate template = lnk.Template;
string url = lnk.Url;
string title = lnk.Title;
string summary = lnk.Abstract;
if (isPageExist(url, sb))
{
return;
}
String pageBody = new PagedDetailSpider().GetContent(url, template, sb);
if (pageBody == null)
{
return;
}
SpiderArticle pd = new SpiderArticle();
pd.Title = title;
pd.Url = strUtil.SubString(url, 200);
pd.Abstract = summary;
pd.Body = pageBody;
pd.SpiderTemplate = template;
MatchCollection matchs = Regex.Matches(pageBody, RegPattern.Img, RegexOptions.Singleline);
if (matchs.Count > 0)
{
pd.IsPic = 1;
pd.PicUrl = matchs[0].Groups[1].Value;
}
pd.insert();
sb.AppendLine("保存成功..." + lnk.Title + "_" + lnk.Url);
}
private static void savePageDetail(DetailLink lnk, StringBuilder sb)
{
SpiderTemplate template = lnk.Template;
string url = lnk.Url;
string title = lnk.Title;
string summary = lnk.Abstract;
if (isPageExist(url, sb))
{
return;
}
String pageBody = new PagedDetailSpider().GetContent(url, template, sb);
if (pageBody == null)
{
return;
}
SpiderArticle pd = new SpiderArticle();
pd.Title = title;
pd.Url = strUtil.SubString(url, 250);
pd.Abstract = summary;
pd.Body = pageBody;
pd.SpiderTemplate = template;
MatchCollection matchs = Regex.Matches(pageBody, RegPattern.Img, RegexOptions.Singleline);
if (matchs.Count > 0)
{
pd.IsPic = 1;
pd.PicUrl = matchs[0].Groups[1].Value;
}
pd.insert();
sb.AppendLine("保存成功..." + lnk.Title + "_" + lnk.Url);
pageBody = Regex.Replace(pageBody, "font-size", "", RegexOptions.IgnoreCase);
string strArcitleLink = "<div class=\"ArcitleLink\"><a href=" + pd.Url + ">原文链接</a></div>";
pageBody = pageBody + strArcitleLink;
Maticsoft.BLL.BlogCategory bllBlogCategory = new Maticsoft.BLL.BlogCategory();
DataSet ds = bllBlogCategory.GetList("AppId = '" + template.IsDelete.ToString() + "'");
int nCateID = 1;
if (ds.Tables[0].Rows.Count > 0)
{
nCateID = (int)ds.Tables[0].Rows[0]["Id"];
}
BlogPost data = new BlogPost();
data.CategoryId = nCateID;
data.Title = title;
data.Abstract = summary;
data.Content = pageBody;
data.AccessStatus = 0;
data.CommentCondition = 0;
data.SaveStatus = 1;//草稿
data.Created = System.DateTime.Now.Date;
data.IsTop = 0;
data.IsPick = 0;
data.IsPic = 0;
data.Ip = "";
data.OwnerId = template.IsDelete;
data.OwnerUrl = template.SiteName;
data.OwnerType = "wojilu.Members.Users.Domain.User";
data.CreatorUrl = template.SiteName;
data.AppId = template.IsDelete;;
data.CreatorId = template.IsDelete;
Maticsoft.BLL.BlogPost bll = new Maticsoft.BLL.BlogPost();
bll.Add(data);
}
private static void savePageDetail( DetailLink lnk, StringBuilder sb )
{
SpiderTemplate template = lnk.Template;
string url = lnk.Url;
string title = lnk.Title;
string summary = lnk.Abstract;
if (isPageExist( url, sb )) return;
String pageBody = new PagedDetailSpider().GetContent( url, template, sb );
if (pageBody == null) return;
SpiderArticle pd = new SpiderArticle();
pd.Title = title;
pd.Url = strUtil.SubString( url, 200 );
pd.Abstract = summary;
pd.Body = pageBody;
pd.SpiderTemplate = template;
MatchCollection matchs = Regex.Matches( pageBody, RegPattern.Img, RegexOptions.Singleline );
if (matchs.Count > 0) {
pd.IsPic = 1;
pd.PicUrl = matchs[0].Groups[1].Value;
}
pd.insert();
sb.AppendLine( "保存成功..." + lnk.Title + "_" + lnk.Url );
}
private static int importToTemp( SpiderArticle art, SpiderImport item, ContentSection section, ContentApp app )
{
ContentTempPost post = new ContentTempPost();
post.Creator = item.Creator;
post.OwnerId = app.OwnerId;
post.OwnerType = app.OwnerType;
post.AppId = app.Id;
post.SectionId = section.Id;
post.Title = art.Title;
post.SourceLink = art.Url;
post.Content = art.Body;
post.insert();
return post.Id;
}
private static int importDirect( SpiderArticle art, SpiderImport item, ContentSection section, ContentApp app )
{
ContentPost post = new ContentPost();
post.Title = art.Title;
post.Content = art.Body;
if (art.IsPic == 1) {
post.CategoryId = PostCategory.Img;
post.ImgLink = art.PicUrl;
}
post.SourceLink = art.Url;
post.Creator = item.Creator;
post.CreatorUrl = item.Creator.Url;
post.PageSection = section;
post.OwnerId = app.OwnerId;
post.OwnerType = app.OwnerType;
post.OwnerUrl = app.OwnerUrl;
post.AppId = app.Id;
post.insert();
return post.Id;
}
private static void savePageDetail( DetailLink lnk, StringBuilder sb )
{
SpiderTemplate template = lnk.Template;
string url = lnk.Url;
string title = lnk.Title;
string summary = lnk.Abstract;
if (isPageExist( url, sb )) return;
String pageBody = new PagedDetailSpider().GetContent( url, template, sb );
if (pageBody == null) return;
SpiderArticle pd = new SpiderArticle();
pd.Title = title;
pd.Url = strUtil.SubString( url, 250 );
pd.Abstract = summary;
pd.Body = pageBody;
pd.SpiderTemplate = template;
MatchCollection matchs = Regex.Matches( pageBody, RegPattern.Img, RegexOptions.Singleline );
if (matchs.Count > 0) {
pd.IsPic = 1;
pd.PicUrl = matchs[0].Groups[1].Value;
}
pd.insert();
sb.AppendLine( "保存成功..." + lnk.Title + "_" + lnk.Url );
pageBody = Regex.Replace(pageBody, "font-size", "", RegexOptions.IgnoreCase);
string strArcitleLink = "<div class=\"ArcitleLink\"><a href=" + pd.Url + ">原文链接</a></div>";
pageBody = pageBody + strArcitleLink;
Maticsoft.BLL.BlogCategory bllBlogCategory = new Maticsoft.BLL.BlogCategory();
DataSet ds = bllBlogCategory.GetList("AppId = '" + template.IsDelete.ToString() + "'");
int nCateID = 1;
if (ds.Tables[0].Rows.Count > 0)
{
nCateID = (int)ds.Tables[0].Rows[0]["Id"];
}
BlogPost data = new BlogPost();
data.CategoryId = nCateID;
data.Title = title;
data.Abstract = summary;
data.Content = pageBody;
data.AccessStatus = 0;
data.CommentCondition = 0;
data.SaveStatus = 1;//草稿
data.Created = System.DateTime.Now.Date;
data.IsTop = 0;
data.IsPick = 0;
data.IsPic = 0;
data.Ip = "";
data.OwnerId = template.IsDelete;
data.OwnerUrl = template.SiteName;
data.OwnerType = "wojilu.Members.Users.Domain.User";
data.CreatorUrl = template.SiteName;
data.AppId = template.IsDelete; ;
data.CreatorId = template.IsDelete;
Maticsoft.BLL.BlogPost bll = new Maticsoft.BLL.BlogPost();
bll.Add(data);
}
