/// <summary>
/// Construct a spider object. The options parameter
/// specifies the options for this spider. The report
/// parameter specifies the class that the spider is to
/// report progress to.
/// </summary>
/// <param name="options">The configuration options for this spider.</param>
/// <param name="report">A SpiderReportable class to report progress to</param>
public Spider(SpiderOptions options, SpiderReportable report)
{
this.options = options;
this.report = report;
this.workloadManager = (WorkloadManager)Assembly.GetExecutingAssembly().CreateInstance(this.options.WorkloadManager);
this.workloadManager.Init(this);
report.Init(this);
// add filters
if (options.Filter != null)
{
foreach (String name in options.Filter)
{
SpiderFilter filter = (SpiderFilter)Assembly.GetExecutingAssembly().CreateInstance(name);
if (filter == null)
{
throw new SpiderException("Invalid filter specified: " + name);
}
this.filters.Add(filter);
}
}
// perform startup
if (String.Compare(options.Startup, SpiderOptions.STARTUP_RESUME) == 0)
{
this.workloadManager.Resume();
}
else
{
this.workloadManager.Clear();
}
}
/// <summary>
/// This method is called by main to check a link. After
/// spidering through the site, the final list of bad links
/// is displayed.
/// </summary>
/// <param name="url">The URL to check for bad links.</param>
public void check(Uri url)
{
SpiderOptions options = new SpiderOptions();
options.WorkloadManager = typeof(MemoryWorkloadManager).FullName;
LinkReport report = new LinkReport();
Spider spider = new Spider(options, report);
spider.AddURL(url, null, 1);
spider.Process();
Console.WriteLine(spider.Status);
if (report.Bad.Count > 0)
{
Console.WriteLine("Bad Links Found:");
foreach (String str in report.Bad)
{
Console.WriteLine(str);
}
}
else
{
Console.WriteLine("No bad links were found.");
}
}
/// <summary>
/// Download an entire site.
/// </summary>
/// <param name="config">The spider configuration file to use.</param>
/// <param name="baseHost">The URL to start from.</param>
/// <param name="local">The local path to save files to.</param>
public void Download(String config, Uri baseHost, String local)
{
WorldSpiderReport report = new WorldSpiderReport(local);
SpiderOptions options = new SpiderOptions();
options.Load(config);
Spider spider = new Spider(options, report);
spider.AddURL(baseHost, null, 1);
spider.Process();
Console.WriteLine(spider.Status);
}
/// <summary>
/// Download an entire site.
/// </summary>
/// <param name="config">The spider configuration file to use.</param>
/// <param name="baseURL">The URL to start from.></param>
/// <param name="local">The local path to save files to.</param>
public void Download(String config, Uri baseURL, String local)
{
SpiderReport report = new SpiderReport(local);
SpiderOptions options = new SpiderOptions();
options.Load(config);
Spider spider = new Spider(options, report);
spider.Logging.Console = true;
spider.Logging.Filename = "c:\\spider.log";
spider.Logging.Clear();
spider.AddURL(baseURL, null, 1);
spider.Process();
Console.WriteLine(spider.Status);
}
static void Main(String[] args)
{
Application.EnableVisualStyles();
Application.SetCompatibleTextRenderingDefault(false);
MainForm form = new MainForm();
if (args.Length < 1)
{
MessageBox.Show("Please pass a path to a spider configuration file as an argument to this program (i.e. Recipe13_4 c:\\spider.conf).", "Heaton Research Spider");
return;
}
SpiderOptions options = new SpiderOptions();
options.Load(args[0]);
form.Options = options;
Application.Run(form);
}
/// <summary>
/// Construct a spider object. The options parameter
/// specifies the options for this spider. The report
/// parameter specifies the class that the spider is to
/// report progress to.
/// </summary>
/// <param name="options">The configuration options for this spider.</param>
/// <param name="report">A SpiderReportable class to report progress to</param>
public Spider(SpiderOptions options, SpiderReportable report)
{
this.options = options;
this.report = report;
this.workloadManager = (WorkloadManager)Assembly.GetExecutingAssembly().CreateInstance(this.options.WorkloadManager);
this.workloadManager.Init(this);
report.Init(this);
// add filters
if (options.Filter != null)
{
foreach (String name in options.Filter)
{
SpiderFilter filter = (SpiderFilter)Assembly.GetExecutingAssembly().CreateInstance(name);
if (filter == null)
throw new SpiderException("Invalid filter specified: " + name);
this.filters.Add(filter);
}
}
// perform startup
if (String.Compare(options.Startup, SpiderOptions.STARTUP_RESUME) == 0)
{
this.workloadManager.Resume();
}
else
{
this.workloadManager.Clear();
}
}