public IHttpActionResult GetContentUrls()
{
try
{
var request = Context.AuthenticatedRequest;
var siteId = request.GetQueryInt("siteId");
var ruleId = request.GetQueryInt("ruleId");
if (!request.IsAdminLoggin ||
!request.AdminPermissions.HasSitePermissions(siteId, Utils.PluginId))
{
return(Unauthorized());
}
var gatherUrl = request.GetPostString("gatherUrl");
var ruleInfo = Main.GatherRuleRepository.GetGatherRuleInfo(ruleId);
var regexUrlInclude = GatherUtility.GetRegexString(ruleInfo.UrlInclude);
var regexListArea = GatherUtility.GetRegexArea(ruleInfo.ListAreaStart, ruleInfo.ListAreaEnd);
var contentUrls = GatherUtility.GetContentUrls(gatherUrl, ruleInfo.Charset, ruleInfo.CookieString, regexListArea, regexUrlInclude);
return(Ok(new
{
Value = contentUrls
}));
}
catch (Exception ex)
{
return(InternalServerError(ex));
}
}
public void GetContent_Click(object sender, EventArgs e)
{
var getContent = (Button)sender;
var contentUrl = getContent.CommandArgument;
var gatherRuleInfo = DataProvider.GatherRuleDao.GetGatherRuleInfo(_gatherRuleName, PublishmentSystemId);
var regexContentExclude = GatherUtility.GetRegexString(gatherRuleInfo.ContentExclude);
var regexChannel = GatherUtility.GetRegexChannel(gatherRuleInfo.ContentChannelStart, gatherRuleInfo.ContentChannelEnd);
var regexContent = GatherUtility.GetRegexContent(gatherRuleInfo.ContentContentStart, gatherRuleInfo.ContentContentEnd);
var regexContent2 = string.Empty;
if (!string.IsNullOrEmpty(gatherRuleInfo.Additional.ContentContentStart2) && !string.IsNullOrEmpty(gatherRuleInfo.Additional.ContentContentEnd2))
{
regexContent2 = GatherUtility.GetRegexContent(gatherRuleInfo.Additional.ContentContentStart2, gatherRuleInfo.Additional.ContentContentEnd2);
}
var regexContent3 = string.Empty;
if (!string.IsNullOrEmpty(gatherRuleInfo.Additional.ContentContentStart3) && !string.IsNullOrEmpty(gatherRuleInfo.Additional.ContentContentEnd3))
{
regexContent3 = GatherUtility.GetRegexContent(gatherRuleInfo.Additional.ContentContentStart3, gatherRuleInfo.Additional.ContentContentEnd3);
}
var regexNextPage = GatherUtility.GetRegexUrl(gatherRuleInfo.ContentNextPageStart, gatherRuleInfo.ContentNextPageEnd);
var regexTitle = GatherUtility.GetRegexTitle(gatherRuleInfo.ContentTitleStart, gatherRuleInfo.ContentTitleEnd);
var contentAttributes = TranslateUtils.StringCollectionToStringList(gatherRuleInfo.ContentAttributes);
var contentAttributesXML = TranslateUtils.ToNameValueCollection(gatherRuleInfo.ContentAttributesXml);
var attributes = GatherUtility.GetContentNameValueCollection(gatherRuleInfo.Charset, contentUrl, gatherRuleInfo.CookieString, regexContentExclude, gatherRuleInfo.ContentHtmlClearCollection, gatherRuleInfo.ContentHtmlClearTagCollection, regexTitle, regexContent, regexContent2, regexContent3, regexNextPage, regexChannel, contentAttributes, contentAttributesXML);
var builder = new StringBuilder();
var relatedIdentities = RelatedIdentities.GetChannelRelatedIdentities(PublishmentSystemId, PublishmentSystemId);
var styleInfoList = TableStyleManager.GetTableStyleInfoList(ETableStyle.BackgroundContent, PublishmentSystemInfo.AuxiliaryTableForContent, relatedIdentities);
foreach (var styleInfo in styleInfoList)
{
if (string.IsNullOrEmpty(attributes[styleInfo.AttributeName.ToLower()]))
{
continue;
}
if (StringUtils.EqualsIgnoreCase(ContentAttribute.Title, styleInfo.AttributeName))
{
builder.Append(
$@"<a href=""{contentUrl}"" target=""_blank"">{styleInfo.DisplayName}： {attributes[
styleInfo.AttributeName.ToLower()]}</a><br><br>");
}
else if (StringUtils.EqualsIgnoreCase(BackgroundContentAttribute.ImageUrl, styleInfo.AttributeName) || EInputTypeUtils.Equals(styleInfo.InputType, EInputType.Image))
{
var imageUrl = PageUtils.GetUrlByBaseUrl(attributes[styleInfo.AttributeName.ToLower()], contentUrl);
builder.Append($"{styleInfo.DisplayName}： <img src='{imageUrl}' /><br><br>");
}
else
{
builder.Append($"{styleInfo.DisplayName}： {attributes[styleInfo.AttributeName.ToLower()]}<br><br>");
}
}
Content.Text = builder.ToString();
}
public IHttpActionResult Gather()
{
try
{
var request = Context.AuthenticatedRequest;
var siteId = request.GetQueryInt("siteId");
var ruleId = request.GetQueryInt("ruleId");
if (!request.IsAdminLoggin ||
!request.AdminPermissions.HasSitePermissions(siteId, Utils.PluginId))
{
return(Unauthorized());
}
var guid = request.GetPostString("guid");
var ruleInfo = Main.GatherRuleRepository.GetGatherRuleInfo(ruleId);
var adminInfo = Context.AdminApi.GetAdminInfoByUserId(request.AdminId);
Task.Run(() => GatherUtility.Gather(adminInfo, siteId, ruleId, guid)).ConfigureAwait(false).GetAwaiter();
return(Ok(new
{
Value = true
}));
}
catch (Exception ex)
{
return(InternalServerError(ex));
}
}
public IHttpActionResult Get()
{
try
{
var request = Context.AuthenticatedRequest;
var siteId = request.GetQueryInt("siteId");
var ruleId = request.GetQueryInt("ruleId");
if (!request.IsAdminLoggin ||
!request.AdminPermissions.HasSitePermissions(siteId, Utils.PluginId))
{
return(Unauthorized());
}
var ruleInfo = Main.GatherRuleRepository.GetGatherRuleInfo(ruleId);
var gatherUrls = GatherUtility.GetGatherUrlList(ruleInfo);
return(Ok(new
{
Value = ruleInfo,
GatherUrls = gatherUrls
}));
}
catch (Exception ex)
{
return(InternalServerError(ex));
}
}
public IHttpActionResult GetList()
{
try
{
var request = Context.AuthenticatedRequest;
var siteId = request.GetQueryInt("siteId");
if (!request.IsAdminLoggin ||
!request.AdminPermissions.HasSitePermissions(siteId, Utils.PluginId))
{
return(Unauthorized());
}
var ruleInfoList = Main.GatherRuleRepository.GetGatherRuleInfoList(siteId);
foreach (var ruleInfo in ruleInfoList)
{
var gatherUrlList = GatherUtility.GetGatherUrlList(ruleInfo);
if (gatherUrlList != null && gatherUrlList.Count > 0)
{
var url = gatherUrlList[0];
ruleInfo.Set("gatherUrl", url);
}
}
return(Ok(new
{
Value = ruleInfoList
}));
}
catch (Exception ex)
{
return(InternalServerError(ex));
}
}
public NameValueCollection GatherFile(int publishmentSystemId, string gatherRuleNameCollection, string userKeyPrefix, string administratorName)
{
var resultBuilder = new StringBuilder();
var errorBuilder = new StringBuilder();
var gatherRuleNameArrayList = TranslateUtils.StringCollectionToStringList(gatherRuleNameCollection);
foreach (string gatherRuleName in gatherRuleNameArrayList)
{
GatherUtility.GatherFile(publishmentSystemId, gatherRuleName, resultBuilder, errorBuilder, true, userKeyPrefix, administratorName);
}
return(AjaxManager.GetProgressTaskNameValueCollection(resultBuilder.ToString(), errorBuilder.ToString()));
}
private static void Gather(int publishmentSystemID, List <string> webGatherNames, List <string> databaseGatherNames, List <string> fileGatherNames)
{
foreach (string webGatherName in webGatherNames)
{
var resultBuilder = new StringBuilder();
var errorBuilder = new StringBuilder();
GatherUtility.GatherWeb(publishmentSystemID, webGatherName, resultBuilder, errorBuilder, false, string.Empty, string.Empty);
}
foreach (string databaseGatherName in databaseGatherNames)
{
var resultBuilder = new StringBuilder();
var errorBuilder = new StringBuilder();
GatherUtility.GatherDatabase(publishmentSystemID, databaseGatherName, resultBuilder, errorBuilder, false, string.Empty, string.Empty);
}
foreach (string fileGatherName in fileGatherNames)
{
var resultBuilder = new StringBuilder();
var errorBuilder = new StringBuilder();
GatherUtility.GatherFile(publishmentSystemID, fileGatherName, resultBuilder, errorBuilder, false, string.Empty, string.Empty);
}
}
public void Page_Load(object sender, EventArgs e)
{
if (IsForbidden)
{
return;
}
PageUtils.CheckRequestParameter("PublishmentSystemID", "GatherRuleName", "IsFileRule");
_gatherRuleName = Body.GetQueryString("GatherRuleName");
_isFileRule = Body.GetQueryBool("IsFileRule");
if (!IsPostBack)
{
InfoMessage("采集名称：" + _gatherRuleName);
if (_isFileRule)
{
var gatherFileRuleInfo = DataProvider.GatherFileRuleDao.GetGatherFileRuleInfo(_gatherRuleName, PublishmentSystemId);
GatherUrls.Items.Add(new ListItem(gatherFileRuleInfo.GatherUrl, gatherFileRuleInfo.GatherUrl));
GetContentUrls.Text = "获取内容";
}
else
{
var gatherRuleInfo = DataProvider.GatherRuleDao.GetGatherRuleInfo(_gatherRuleName, PublishmentSystemId);
var gatherUrlArrayList = GatherUtility.GetGatherUrlArrayList(gatherRuleInfo);
foreach (string gatherUrl in gatherUrlArrayList)
{
GatherUrls.Items.Add(new ListItem(gatherUrl, gatherUrl));
}
}
}
}
public void GatherUrls_Click(object sender, EventArgs e)
{
if (_isFileRule)
{
var gatherFileRuleInfo = DataProvider.GatherFileRuleDao.GetGatherFileRuleInfo(_gatherRuleName, PublishmentSystemId);
var builder = new StringBuilder();
if (gatherFileRuleInfo.IsToFile == false)
{
var regexTitle = GatherUtility.GetRegexTitle(gatherFileRuleInfo.ContentTitleStart, gatherFileRuleInfo.ContentTitleEnd);
var regexContentExclude = GatherUtility.GetRegexString(gatherFileRuleInfo.ContentExclude);
var regexContent = GatherUtility.GetRegexContent(gatherFileRuleInfo.ContentContentStart, gatherFileRuleInfo.ContentContentEnd);
var contentAttributes = TranslateUtils.StringCollectionToStringList(gatherFileRuleInfo.ContentAttributes);
var contentAttributesXML = TranslateUtils.ToNameValueCollection(gatherFileRuleInfo.ContentAttributesXml);
var attributes = GatherUtility.GetContentNameValueCollection(gatherFileRuleInfo.Charset, gatherFileRuleInfo.GatherUrl, string.Empty, regexContentExclude, gatherFileRuleInfo.ContentHtmlClearCollection, gatherFileRuleInfo.ContentHtmlClearTagCollection, regexTitle, regexContent, string.Empty, string.Empty, string.Empty, string.Empty, contentAttributes, contentAttributesXML);
var styleInfoList = TableStyleManager.GetTableStyleInfoList(ETableStyle.BackgroundContent, PublishmentSystemInfo.AuxiliaryTableForContent, null);
foreach (var styleInfo in styleInfoList)
{
if (string.IsNullOrEmpty(attributes[styleInfo.AttributeName.ToLower()]))
{
continue;
}
builder.Append($"{styleInfo.DisplayName}： {attributes[styleInfo.AttributeName.ToLower()]}<br><br>");
}
}
else
{
try
{
var fileContent = WebClientUtils.GetRemoteFileSource(gatherFileRuleInfo.GatherUrl, gatherFileRuleInfo.Charset, string.Empty);
builder.Append(StringUtils.HtmlEncode(fileContent));
}
catch (Exception ex)
{
FailMessage(ex, ex.Message);
}
}
Content.Text = builder.ToString();
}
else
{
var gatherUrl = GatherUrls.SelectedValue;
var errorBuilder = new StringBuilder();
var gatherRuleInfo = DataProvider.GatherRuleDao.GetGatherRuleInfo(_gatherRuleName, PublishmentSystemId);
var regexUrlInclude = GatherUtility.GetRegexString(gatherRuleInfo.UrlInclude);
var regexListArea = GatherUtility.GetRegexArea(gatherRuleInfo.ListAreaStart, gatherRuleInfo.ListAreaEnd);
var contentUrlArrayList = GatherUtility.GetContentUrls(gatherUrl, gatherRuleInfo.Charset, gatherRuleInfo.CookieString, regexListArea, regexUrlInclude, errorBuilder);
ContentUrlRepeater.DataSource = contentUrlArrayList;
ContentUrlRepeater.ItemDataBound += ContentUrlRepeater_ItemDataBound;
ContentUrlRepeater.DataBind();
InfoMessage($"采集名称：{_gatherRuleName} 内容数：{contentUrlArrayList.Count}");
Content.Text = string.Empty;
}
}
public IHttpActionResult GetContent()
{
try
{
var request = Context.AuthenticatedRequest;
var siteId = request.GetQueryInt("siteId");
var ruleId = request.GetQueryInt("ruleId");
if (!request.IsAdminLoggin ||
!request.AdminPermissions.HasSitePermissions(siteId, Utils.PluginId))
{
return(Unauthorized());
}
var contentUrl = request.GetPostString("contentUrl");
var ruleInfo = Main.GatherRuleRepository.GetGatherRuleInfo(ruleId);
var regexContentExclude = GatherUtility.GetRegexString(ruleInfo.ContentExclude);
var regexChannel = GatherUtility.GetRegexChannel(ruleInfo.ContentChannelStart, ruleInfo.ContentChannelEnd);
var regexContent = GatherUtility.GetRegexContent(ruleInfo.ContentContentStart, ruleInfo.ContentContentEnd);
var regexContent2 = string.Empty;
if (!string.IsNullOrEmpty(ruleInfo.ContentContentStart2) && !string.IsNullOrEmpty(ruleInfo.ContentContentEnd2))
{
regexContent2 = GatherUtility.GetRegexContent(ruleInfo.ContentContentStart2, ruleInfo.ContentContentEnd2);
}
var regexContent3 = string.Empty;
if (!string.IsNullOrEmpty(ruleInfo.ContentContentStart3) && !string.IsNullOrEmpty(ruleInfo.ContentContentEnd3))
{
regexContent3 = GatherUtility.GetRegexContent(ruleInfo.ContentContentStart3, ruleInfo.ContentContentEnd3);
}
var regexNextPage = GatherUtility.GetRegexUrl(ruleInfo.ContentNextPageStart, ruleInfo.ContentNextPageEnd);
var regexTitle = GatherUtility.GetRegexTitle(ruleInfo.ContentTitleStart, ruleInfo.ContentTitleEnd);
var contentAttributes = TranslateUtils.StringCollectionToStringList(ruleInfo.ContentAttributes);
var attributesDict = TranslateUtils.JsonDeserialize <Dictionary <string, string> >(ruleInfo.ContentAttributesXml);
var attributes = GatherUtility.GetContentNameValueCollection(ruleInfo.Charset, contentUrl, ruleInfo.CookieString, regexContentExclude, ruleInfo.ContentHtmlClearCollection, ruleInfo.ContentHtmlClearTagCollection, regexTitle, regexContent, regexContent2, regexContent3, regexNextPage, regexChannel, contentAttributes, attributesDict);
var list = new List <KeyValuePair <string, string> >();
//var contentAttributes = Context.ContentApi.GetInputStyles(siteId, ruleInfo.ChannelId);
foreach (var attributeName in attributes.AllKeys)
{
var value = attributes[attributeName];
if (string.IsNullOrEmpty(value))
{
continue;
}
if (StringUtils.EqualsIgnoreCase(ContentAttribute.ImageUrl, attributeName))
{
var imageUrl = PageUtils.GetUrlByBaseUrl(value, contentUrl);
list.Add(new KeyValuePair <string, string>(attributeName, imageUrl));
}
else
{
list.Add(new KeyValuePair <string, string>(attributeName, value));
}
}
return(Ok(new
{
Value = list
}));
}
catch (Exception ex)
{
return(InternalServerError(ex));
}
}
void dgContents_ItemDataBound(object sender, DataGridItemEventArgs e)
{
if (e.Item.ItemType == ListItemType.Item || e.Item.ItemType == ListItemType.AlternatingItem)
{
var gatherRuleInfo = e.Item.DataItem as GatherRuleInfo;
var ltlGatherRuleName = (Literal)e.Item.FindControl("ltlGatherRuleName");
var ltlGatherUrl = (Literal)e.Item.FindControl("ltlGatherUrl");
var ltlLastGatherDate = (Literal)e.Item.FindControl("ltlLastGatherDate");
var ltlIsAutoCreate = (Literal)e.Item.FindControl("ltlIsAutoCreate");
var ltlTestGatherUrl = (Literal)e.Item.FindControl("ltlTestGatherUrl");
var ltlStartGatherUrl = (Literal)e.Item.FindControl("ltlStartGatherUrl");
var ltlEditLink = (Literal)e.Item.FindControl("ltlEditLink");
var ltlCopyLink = (Literal)e.Item.FindControl("ltlCopyLink");
var ltlDeleteLink = (Literal)e.Item.FindControl("ltlDeleteLink");
ltlGatherRuleName.Text = gatherRuleInfo.GatherRuleName;
var gatherUrlArrayList = GatherUtility.GetGatherUrlArrayList(gatherRuleInfo);
if (gatherUrlArrayList != null && gatherUrlArrayList.Count > 0)
{
var url = (string)gatherUrlArrayList[0];
url = PageUtils.AddProtocolToUrl(url);
ltlGatherUrl.Text =
$@"<a href=""{url}"" target=""_blank"" title=""{url}"">{StringUtils.MaxLengthText(url, 25)}</a>";
}
if (!DateUtils.SqlMinValue.Equals(gatherRuleInfo.LastGatherDate))
{
ltlLastGatherDate.Text = DateUtils.GetDateAndTimeString(gatherRuleInfo.LastGatherDate);
}
if (gatherRuleInfo.Additional.IsAutoCreate)
{
ltlIsAutoCreate.Text = "是";
}
else
{
ltlIsAutoCreate.Text = "否";
}
var showPopWinString = ModalGatherTest.GetOpenWindowString(PublishmentSystemId, gatherRuleInfo.GatherRuleName, false);
ltlTestGatherUrl.Text = $@"<a href=""javascript:;"" onclick=""{showPopWinString}"">测试</a>";
showPopWinString = ModalGatherSet.GetOpenWindowString(PublishmentSystemId, gatherRuleInfo.GatherRuleName);
ltlStartGatherUrl.Text = $@"<a href=""javascript:;"" onclick=""{showPopWinString}"">开始采集</a>";
var urlEdit = PageGatherRuleAdd.GetRedirectUrl(PublishmentSystemId, gatherRuleInfo.GatherRuleName);
ltlEditLink.Text = $"<a href=\"{urlEdit}\">编辑</a>";
var urlCopy = PageUtils.GetCmsUrl(nameof(PageGatherRule), new NameValueCollection
{
{ "PublishmentSystemID", PublishmentSystemId.ToString() },
{ "GatherRuleName", gatherRuleInfo.GatherRuleName },
{ "Copy", true.ToString() }
});
ltlCopyLink.Text = $@"<a href=""{urlCopy}"">复制</a>";
var urlDelete = PageUtils.GetCmsUrl(nameof(PageGatherRule), new NameValueCollection
{
{ "PublishmentSystemID", PublishmentSystemId.ToString() },
{ "GatherRuleName", gatherRuleInfo.GatherRuleName },
{ "Delete", true.ToString() }
});
ltlDeleteLink.Text =
$@"<a href=""{urlDelete}"" onClick=""javascript:return confirm('此操作将删除采集规则“{gatherRuleInfo
.GatherRuleName}”，确认吗？');"">删除</a>";
}
}
