public void Start(String startingURL, ILogBuilder logger = null)
{
WebDomainCategory node = result;
WebDirectoryIteration iteration = new WebDirectoryIteration();
Match m = SelectPath.Match(startingURL);
iteration.URL = startingURL;
iteration.DirectoryPath = m.Groups[1].Value;
node.name = iteration.DirectoryPath.Trim('/');
iteration.DirectoryNode = node;
List <WebDirectoryIteration> tasks = new List <WebDirectoryIteration>();
tasks.Add(iteration);
while (tasks.Any())
{
List <WebDirectoryIteration> newTasks = new List <WebDirectoryIteration>();
foreach (WebDirectoryIteration task in tasks)
{
newTasks.AddRange(Load(task, logger));
}
logger.log("Tasks done [" + tasks.Count + "] - new tasks [" + newTasks.Count + "]");
tasks = newTasks;
}
}
public OpenDictionaryHarvester(String startURL)
{
HomeDomain = SelectDomainName.Match(startURL).Groups[1].Value;
HomeURL = SelectPath.Match(startURL).Groups[1].Value;
}