protected void ExtractTargetsFromCurrentAndEnqueue() { string[] more = ExtractTargets(Current); for (int i = 0; i < more.Length; i++) { string extracted = more[i]; if (string.IsNullOrEmpty(extracted)) { continue; } string compareTo = IsCaseSensitive ? extracted.ToLowerInvariant() : extracted; if (!WasProcessed(compareTo)) { if (MaxQueueSize == 0 || (MaxQueueSize > 0 && TargetQueue.Count < MaxQueueSize)) { TargetQueue.Enqueue(extracted); } } } }
public void Crawl(string rootTarget) { Exec.Start(ThreadName, () => { Root = rootTarget; _processed = new HashSet <string>(); TargetQueue.Enqueue(Root); string current; while (TargetQueue.TryDequeue(out current)) { Current = current; CurrentAction = CrawlerState.Action.Extracting; ExtractTargetsFromCurrentAndEnqueue(); CurrentAction = CrawlerState.Action.Processing; ProcessTarget(Current); _processed.Add(IsCaseSensitive ? Current : Current.ToLowerInvariant()); } CurrentAction = CrawlerState.Action.Idle; }); }