/// <summary> /// Construct a spider object. The options parameter /// specifies the options for this spider. The report /// parameter specifies the class that the spider is to /// report progress to. /// </summary> /// <param name="options">The configuration options for this spider.</param> /// <param name="report">A SpiderReportable class to report progress to</param> public Spider(SpiderOptions options, SpiderReportable report) { this.options = options; this.report = report; this.workloadManager = (WorkloadManager)Assembly.GetExecutingAssembly().CreateInstance(this.options.WorkloadManager); this.workloadManager.Init(this); report.Init(this); // add filters if (options.Filter != null) { foreach (String name in options.Filter) { SpiderFilter filter = (SpiderFilter)Assembly.GetExecutingAssembly().CreateInstance(name); if (filter == null) { throw new SpiderException("Invalid filter specified: " + name); } this.filters.Add(filter); } } // perform startup if (String.Compare(options.Startup, SpiderOptions.STARTUP_RESUME) == 0) { this.workloadManager.Resume(); } else { this.workloadManager.Clear(); } }
/// <summary> /// This method is called by main to check a link. After /// spidering through the site, the final list of bad links /// is displayed. /// </summary> /// <param name="url">The URL to check for bad links.</param> public void check(Uri url) { SpiderOptions options = new SpiderOptions(); options.WorkloadManager = typeof(MemoryWorkloadManager).FullName; LinkReport report = new LinkReport(); Spider spider = new Spider(options, report); spider.AddURL(url, null, 1); spider.Process(); Console.WriteLine(spider.Status); if (report.Bad.Count > 0) { Console.WriteLine("Bad Links Found:"); foreach (String str in report.Bad) { Console.WriteLine(str); } } else { Console.WriteLine("No bad links were found."); } }
/// <summary> /// Download an entire site. /// </summary> /// <param name="config">The spider configuration file to use.</param> /// <param name="baseHost">The URL to start from.</param> /// <param name="local">The local path to save files to.</param> public void Download(String config, Uri baseHost, String local) { WorldSpiderReport report = new WorldSpiderReport(local); SpiderOptions options = new SpiderOptions(); options.Load(config); Spider spider = new Spider(options, report); spider.AddURL(baseHost, null, 1); spider.Process(); Console.WriteLine(spider.Status); }
/// <summary> /// Download an entire site. /// </summary> /// <param name="config">The spider configuration file to use.</param> /// <param name="baseURL">The URL to start from.></param> /// <param name="local">The local path to save files to.</param> public void Download(String config, Uri baseURL, String local) { SpiderReport report = new SpiderReport(local); SpiderOptions options = new SpiderOptions(); options.Load(config); Spider spider = new Spider(options, report); spider.Logging.Console = true; spider.Logging.Filename = "c:\\spider.log"; spider.Logging.Clear(); spider.AddURL(baseURL, null, 1); spider.Process(); Console.WriteLine(spider.Status); }
static void Main(String[] args) { Application.EnableVisualStyles(); Application.SetCompatibleTextRenderingDefault(false); MainForm form = new MainForm(); if (args.Length < 1) { MessageBox.Show("Please pass a path to a spider configuration file as an argument to this program (i.e. Recipe13_4 c:\\spider.conf).", "Heaton Research Spider"); return; } SpiderOptions options = new SpiderOptions(); options.Load(args[0]); form.Options = options; Application.Run(form); }
/// <summary> /// Construct a spider object. The options parameter /// specifies the options for this spider. The report /// parameter specifies the class that the spider is to /// report progress to. /// </summary> /// <param name="options">The configuration options for this spider.</param> /// <param name="report">A SpiderReportable class to report progress to</param> public Spider(SpiderOptions options, SpiderReportable report) { this.options = options; this.report = report; this.workloadManager = (WorkloadManager)Assembly.GetExecutingAssembly().CreateInstance(this.options.WorkloadManager); this.workloadManager.Init(this); report.Init(this); // add filters if (options.Filter != null) { foreach (String name in options.Filter) { SpiderFilter filter = (SpiderFilter)Assembly.GetExecutingAssembly().CreateInstance(name); if (filter == null) throw new SpiderException("Invalid filter specified: " + name); this.filters.Add(filter); } } // perform startup if (String.Compare(options.Startup, SpiderOptions.STARTUP_RESUME) == 0) { this.workloadManager.Resume(); } else { this.workloadManager.Clear(); } }