예제 #1
0
 /// <summary>
 /// Add urls to fetch
 /// </summary>
 /// <param name="requests"></param>
 /// <param name="priority"></param>
 public void AddTargetRequests(IList <string> requests, int priority)
 {
     if (requests == null || requests.Count == 0)
     {
         return;
     }
     lock (this)
     {
         foreach (string s in requests)
         {
             if (string.IsNullOrEmpty(s) || s.Equals("#") || s.StartsWith("javascript:"))
             {
                 continue;
             }
             string  s1      = UrlUtils.CanonicalizeUrl(s, Url);
             Request request = new Request(s1, Request.Extras)
             {
                 Priority = priority, Depth = Request.NextDepth
             };
             if (request.IsAvailable)
             {
                 TargetRequests.Add(request);
             }
         }
     }
 }
예제 #2
0
 /// <summary>
 /// Add requests to fetch
 /// </summary>
 public void AddTargetRequest(Request request)
 {
     if (request == null)
     {
         return;
     }
     lock (this)
     {
         request.Depth = Request.NextDepth;
         TargetRequests.Add(request);
     }
 }
예제 #3
0
파일: Page.cs 프로젝트: personlearn/code
 /// <summary>
 /// 添加解析到的目标链接, 添加到队列中
 /// </summary>
 /// <param name="request">链接</param>
 /// <param name="increaseDeep">目标链接的深度是否升高</param>
 public void AddTargetRequest(Request request, bool increaseDeep = true)
 {
     if (request == null || !request.IsAvailable)
     {
         return;
     }
     request.Depth = increaseDeep ? Request.NextDepth : Request.Depth;
     lock (_locker)
     {
         TargetRequests.Add(request);
     }
 }
예제 #4
0
 /// <summary>
 /// Add urls to fetch
 /// </summary>
 /// <param name="requests"></param>
 public void AddTargetRequests(IList <Request> requests)
 {
     if (requests == null || requests.Count == 0)
     {
         return;
     }
     lock (this)
     {
         foreach (var r in requests)
         {
             r.Depth = Request.NextDepth;
             TargetRequests.Add(r);
         }
     }
 }
예제 #5
0
        /// <summary>
        /// Add url to fetch
        /// </summary>
        /// <param name="requestString"></param>
        public void AddTargetRequest(string requestString)
        {
            lock (this)
            {
                if (string.IsNullOrEmpty(requestString) || requestString.Equals("#"))
                {
                    return;
                }

                requestString = UrlUtils.CanonicalizeUrl(requestString, Url);
                TargetRequests.Add(new Request(requestString, Request.Extras)
                {
                    Depth = Request.NextDepth
                });
            }
        }