/// <summary>
/// Called when the spider is ready to process an HTML
/// URL.
/// </summary>
/// <param name="url">The URL that the spider is about to process.</param>
/// <param name="parse">An object that will allow you you to parse the HTML on this page.</param>
public void SpiderProcessURL(Uri url, SpiderParseHTML parse)
{
try
{
parse.ReadAll();
}
catch (IOException)
{
spider.Logging.Log(Logger.Level.INFO, "Error reading page:" + url.ToString());
}
}
/// <summary>
/// Called when the spider is ready to process an HTML
/// URL. Download the contents of the URL to a local file.
/// </summary>
/// <param name="url">The URL that the spider is about to process.</param>
/// <param name="parse">An object that will allow you you to parse the HTML on this page.</param>
public void SpiderProcessURL(Uri url, SpiderParseHTML parse)
{
String filename = URLUtility.convertFilename(this.path, url, true);
Stream os = new FileStream(filename, FileMode.Create);
parse.Stream.OutputStream = os;
parse.ReadAll();
os.Close();
}