public static void Run()
{
NCrawlerModule.Setup();
Console.Out.WriteLine("Advanced crawl demo");
using (Crawler c = new CustomCrawler(new Uri("http://ncrawler.codeplex.com"),
new HtmlDocumentProcessor(), // Process html
new DumperStep())
{
MaximumThreadCount = 10,
MaximumCrawlDepth = 2,
ExcludeFilter = Program.ExtensionsToSkip,
})
{
// Begin crawl
c.Crawl();
}
}
public static void Run()
{
NCrawlerModule.Setup();
Console.Out.WriteLine("Advanced crawl demo");
using (Crawler c = new CustomCrawler(new Uri("http://ncrawler.codeplex.com"),
new HtmlDocumentProcessor(), // Process html
new DumperStep())
{
MaximumThreadCount = 10,
MaximumCrawlDepth = 2,
ExcludeFilter = Program.ExtensionsToSkip,
})
{
// Begin crawl
c.Crawl();
}
}