Ejemplo n.º 1
0
        /// <summary>
        /// Construct a spider object. The options parameter
        /// specifies the options for this spider. The report
        /// parameter specifies the class that the spider is to
        /// report progress to.
        /// </summary>
        /// <param name="options">The configuration options for this spider.</param>
        /// <param name="report">A SpiderReportable class to report progress to</param>
        public Spider(SpiderOptions options, SpiderReportable report)
        {
            this.options = options;
            this.report  = report;

            this.workloadManager = (WorkloadManager)Assembly.GetExecutingAssembly().CreateInstance(this.options.WorkloadManager);

            this.workloadManager.Init(this);
            report.Init(this);

            // add filters
            if (options.Filter != null)
            {
                foreach (String name in options.Filter)
                {
                    SpiderFilter filter = (SpiderFilter)Assembly.GetExecutingAssembly().CreateInstance(name);
                    if (filter == null)
                    {
                        throw new SpiderException("Invalid filter specified: " + name);
                    }
                    this.filters.Add(filter);
                }
            }

            // perform startup
            if (String.Compare(options.Startup, SpiderOptions.STARTUP_RESUME) == 0)
            {
                this.workloadManager.Resume();
            }
            else
            {
                this.workloadManager.Clear();
            }
        }
Ejemplo n.º 2
0
        /// <summary>
        /// This method is called by main to check a link. After
        /// spidering through the site, the final list of bad links
        /// is displayed.
        /// </summary>
        /// <param name="url">The URL to check for bad links.</param>
        public void check(Uri url)
        {
            SpiderOptions options = new SpiderOptions();
            options.WorkloadManager = typeof(MemoryWorkloadManager).FullName;
            LinkReport report = new LinkReport();
            Spider spider = new Spider(options, report);
            spider.AddURL(url, null, 1);

            spider.Process();
            Console.WriteLine(spider.Status);

            if (report.Bad.Count > 0)
            {
                Console.WriteLine("Bad Links Found:");
                foreach (String str in report.Bad)
                {
                    Console.WriteLine(str);
                }
            }
            else
            {
                Console.WriteLine("No bad links were found.");
            }

        }
Ejemplo n.º 3
0
 /// <summary>
 /// Download an entire site.
 /// </summary>
 /// <param name="config">The spider configuration file to use.</param>
 /// <param name="baseHost">The URL to start from.</param>
 /// <param name="local">The local path to save files to.</param>
 public void Download(String config, Uri baseHost, String local)
 {
     WorldSpiderReport report = new WorldSpiderReport(local);
     SpiderOptions options = new SpiderOptions();
     options.Load(config);
     Spider spider = new Spider(options, report);
     spider.AddURL(baseHost, null, 1);
     spider.Process();
     Console.WriteLine(spider.Status);
 }
        /// <summary>
        /// Download an entire site.
        /// </summary>
        /// <param name="config">The spider configuration file to use.</param>
        /// <param name="baseURL">The URL to start from.></param>
        /// <param name="local">The local path to save files to.</param>
        public void Download(String config, Uri baseURL, String local)
        {
            SpiderReport report = new SpiderReport(local);
            SpiderOptions options = new SpiderOptions();
            options.Load(config);
            Spider spider = new Spider(options, report);
            spider.Logging.Console = true;
            spider.Logging.Filename = "c:\\spider.log";
            spider.Logging.Clear();

            spider.AddURL(baseURL, null, 1);
            spider.Process();
            Console.WriteLine(spider.Status);
        }
Ejemplo n.º 5
0
        static void Main(String[] args)
        {
            Application.EnableVisualStyles();
            Application.SetCompatibleTextRenderingDefault(false);

            MainForm form = new MainForm();
            if (args.Length < 1)
            {
                MessageBox.Show("Please pass a path to a spider configuration file as an argument to this program (i.e. Recipe13_4 c:\\spider.conf).", "Heaton Research Spider");
                return;
            }

            SpiderOptions options = new SpiderOptions();
            options.Load(args[0]);
            form.Options = options;
            
            Application.Run(form);
        }
Ejemplo n.º 6
0
        /// <summary>
        /// Construct a spider object. The options parameter
        /// specifies the options for this spider. The report
        /// parameter specifies the class that the spider is to
        /// report progress to.
        /// </summary>
        /// <param name="options">The configuration options for this spider.</param>
        /// <param name="report">A SpiderReportable class to report progress to</param>
        public Spider(SpiderOptions options, SpiderReportable report)
        {
            this.options = options;
            this.report = report;

            this.workloadManager = (WorkloadManager)Assembly.GetExecutingAssembly().CreateInstance(this.options.WorkloadManager);

            this.workloadManager.Init(this);
            report.Init(this);

            // add filters
            if (options.Filter != null)
            {
                foreach (String name in options.Filter)
                {
                    SpiderFilter filter = (SpiderFilter)Assembly.GetExecutingAssembly().CreateInstance(name);
                    if (filter == null)
                        throw new SpiderException("Invalid filter specified: " + name);
                    this.filters.Add(filter);
                }
            }

            // perform startup
            if (String.Compare(options.Startup, SpiderOptions.STARTUP_RESUME) == 0)
            {
                this.workloadManager.Resume();
            }
            else
            {
                this.workloadManager.Clear();
            }
        }