protected void ExtractTargetsFromCurrentAndEnqueue()
{
string[] more = ExtractTargets(Current);
for (int i = 0; i < more.Length; i++)
{
string extracted = more[i];
if (string.IsNullOrEmpty(extracted))
{
continue;
}
string compareTo = IsCaseSensitive ? extracted.ToLowerInvariant() : extracted;
if (!WasProcessed(compareTo))
{
if (MaxQueueSize == 0 || (MaxQueueSize > 0 && TargetQueue.Count < MaxQueueSize))
{
TargetQueue.Enqueue(extracted);
}
}
}
}
public void Crawl(string rootTarget)
{
Exec.Start(ThreadName, () =>
{
Root = rootTarget;
_processed = new HashSet <string>();
TargetQueue.Enqueue(Root);
string current;
while (TargetQueue.TryDequeue(out current))
{
Current = current;
CurrentAction = CrawlerState.Action.Extracting;
ExtractTargetsFromCurrentAndEnqueue();
CurrentAction = CrawlerState.Action.Processing;
ProcessTarget(Current);
_processed.Add(IsCaseSensitive ? Current : Current.ToLowerInvariant());
}
CurrentAction = CrawlerState.Action.Idle;
});
}