コード例 #1
0
 private void AddPending(string result)
 {
     if (string.IsNullOrEmpty(result))
     {
         return;
     }
     if (pendingUrls.Count > MaxCount)
     {
         return;
     }
     if (!pendingUrls.Contains(result) &&
         !CompletedUrls.Contains(result) &&
         !DownloadingUrls.Contains(result) &&
         !FailedUrls.Contains(result))
     {
         pendingUrls.Enqueue(result);
     }
 }
コード例 #2
0
        private void Parse(string html, string previous)
        {
            Uri uri = new Uri(previous);

            foreach (Match match in HREF_REGEX.Matches(html))
            {
                string url = match.Groups["url"].Value;
                if (!URL_REGEX.IsMatch(url))
                {
                    continue;
                }
                Uri    add = new Uri(url, UriKind.RelativeOrAbsolute);
                string result;
                if (url.StartsWith("http://") || url.StartsWith("https://"))
                {
                    if (add.Host != uri.Host)
                    {
                        continue;
                    }
                    result = add.AbsoluteUri;
                }
                else if (!add.IsAbsoluteUri)
                {
                    result = new Uri(uri, add).AbsoluteUri;
                }
                else
                {
                    continue;
                }

                if (!Urls.Contains(result) && !CompletedUrls.Contains(result) && !DownloadingUrls.Contains(result))
                {
                    Urls.Add(result);
                }
            }
        }