public static List <string> GetContentUrlList(GatherRuleInfo gatherRuleInfo, string regexListArea, string regexUrlInclude, ProgressCache cache)
{
var gatherUrls = GetGatherUrlList(gatherRuleInfo);
var contentUrls = new List <string>();
foreach (var gatherUrl in gatherUrls)
{
cache.IsSuccess = true;
cache.Message = "获取链接：" + gatherUrl;
try
{
var urls = GetContentUrls(gatherUrl, gatherRuleInfo.Charset, gatherRuleInfo.CookieString, regexListArea, regexUrlInclude);
contentUrls.AddRange(urls);
}
catch (Exception ex)
{
cache.IsSuccess = false;
cache.Message = ex.Message;
cache.FailureMessages.Add(ex.Message);
}
}
if (gatherRuleInfo.IsOrderByDesc)
{
contentUrls.Reverse();
}
return(contentUrls);
}
public static List <string> GetGatherUrlList(GatherRuleInfo gatherRuleInfo)
{
var gatherUrls = new List <string>();
if (gatherRuleInfo.GatherUrlIsCollection)
{
gatherUrls.AddRange(StringCollectionToList(gatherRuleInfo.GatherUrlCollection, separator: '\n'));
}
if (gatherRuleInfo.GatherUrlIsSerialize)
{
if (gatherRuleInfo.SerializeFrom <= gatherRuleInfo.SerializeTo)
{
var count = 1;
for (var i = gatherRuleInfo.SerializeFrom; i <= gatherRuleInfo.SerializeTo; i = i + gatherRuleInfo.SerializeInterval)
{
count++;
if (count > 200)
{
break;
}
var thePageNumber = i.ToString();
if (gatherRuleInfo.SerializeIsAddZero && thePageNumber.Length == 1)
{
thePageNumber = "0" + i;
}
gatherUrls.Add(gatherRuleInfo.GatherUrlSerialize.Replace("*", thePageNumber));
}
}
if (gatherRuleInfo.SerializeIsOrderByDesc)
{
gatherUrls.Reverse();
}
}
return(gatherUrls);
}
public void Update(GatherRuleInfo gatherRuleInfo)
{
_repository.Update(gatherRuleInfo);
}
public void Insert(GatherRuleInfo gatherRuleInfo)
{
_repository.Insert(gatherRuleInfo);
}
