private void AddPending(string result)
{
if (string.IsNullOrEmpty(result))
{
return;
}
if (pendingUrls.Count > MaxCount)
{
return;
}
if (!pendingUrls.Contains(result) &&
!CompletedUrls.Contains(result) &&
!DownloadingUrls.Contains(result) &&
!FailedUrls.Contains(result))
{
pendingUrls.Enqueue(result);
}
}
private void Parse(string html, string previous)
{
Uri uri = new Uri(previous);
foreach (Match match in HREF_REGEX.Matches(html))
{
string url = match.Groups["url"].Value;
if (!URL_REGEX.IsMatch(url))
{
continue;
}
Uri add = new Uri(url, UriKind.RelativeOrAbsolute);
string result;
if (url.StartsWith("http://") || url.StartsWith("https://"))
{
if (add.Host != uri.Host)
{
continue;
}
result = add.AbsoluteUri;
}
else if (!add.IsAbsoluteUri)
{
result = new Uri(uri, add).AbsoluteUri;
}
else
{
continue;
}
if (!Urls.Contains(result) && !CompletedUrls.Contains(result) && !DownloadingUrls.Contains(result))
{
Urls.Add(result);
}
}
}