public static void Run(SavePageForm parentForm, string url)
{
Form = parentForm;
Url = url;
c = new Crawler(new Uri(url),
new HtmlDocumentProcessor(), // Process html
new SaveFileStep());
c.MaximumThreadCount = 1;
c.MaximumCrawlDepth = 1;
c.ExcludeFilter = CrawlUtil.ExtensionsToSkip;
c.AdhereToRobotRules = false;
c.CrawlFinished += new EventHandler <NCrawler.Events.CrawlFinishedEventArgs>(c_CrawlFinished);
string ua = CrawlUtil.GetRandomUnblockedUserAgent(UserAgentTracker);
//if there are no unblocked user agents left then reset the tracker and retry
if (ua == null)
{
UserAgentTracker = CrawlUtil.InitUserAgentTracker();
ua = CrawlUtil.GetRandomUnblockedUserAgent(UserAgentTracker);
}
c.UserAgent = ua;
// Begin crawl
c.Crawl();
}
private void button2_Click(object sender, EventArgs e)
{
Form spf = new SavePageForm(this);
spf.Show();
}
