/// <summary>
/// Add urls to fetch
/// </summary>
/// <param name="requests"></param>
/// <param name="priority"></param>
public void AddTargetRequests(IList <string> requests, int priority)
{
if (requests == null || requests.Count == 0)
{
return;
}
lock (this)
{
foreach (string s in requests)
{
if (string.IsNullOrEmpty(s) || s.Equals("#") || s.StartsWith("javascript:"))
{
continue;
}
string s1 = UrlUtils.CanonicalizeUrl(s, Url);
Request request = new Request(s1, Request.Extras)
{
Priority = priority, Depth = Request.NextDepth
};
if (request.IsAvailable)
{
TargetRequests.Add(request);
}
}
}
}
/// <summary>
/// Add requests to fetch
/// </summary>
public void AddTargetRequest(Request request)
{
if (request == null)
{
return;
}
lock (this)
{
request.Depth = Request.NextDepth;
TargetRequests.Add(request);
}
}
/// <summary>
/// 添加解析到的目标链接, 添加到队列中
/// </summary>
/// <param name="request">链接</param>
/// <param name="increaseDeep">目标链接的深度是否升高</param>
public void AddTargetRequest(Request request, bool increaseDeep = true)
{
if (request == null || !request.IsAvailable)
{
return;
}
request.Depth = increaseDeep ? Request.NextDepth : Request.Depth;
lock (_locker)
{
TargetRequests.Add(request);
}
}
/// <summary>
/// Add urls to fetch
/// </summary>
/// <param name="requests"></param>
public void AddTargetRequests(IList <Request> requests)
{
if (requests == null || requests.Count == 0)
{
return;
}
lock (this)
{
foreach (var r in requests)
{
r.Depth = Request.NextDepth;
TargetRequests.Add(r);
}
}
}
/// <summary>
/// Add url to fetch
/// </summary>
/// <param name="requestString"></param>
public void AddTargetRequest(string requestString)
{
lock (this)
{
if (string.IsNullOrEmpty(requestString) || requestString.Equals("#"))
{
return;
}
requestString = UrlUtils.CanonicalizeUrl(requestString, Url);
TargetRequests.Add(new Request(requestString, Request.Extras)
{
Depth = Request.NextDepth
});
}
}
