/// <summary>
/// Called when the spider is ready to process an HTML
/// URL. Download the contents of the URL to a local file.
/// </summary>
/// <param name="url">The URL that the spider is about to process.</param>
/// <param name="parse">An object that will allow you you to parse the HTML on this page.</param>
public void SpiderProcessURL(Uri url, SpiderParseHTML parse)
{
String filename = URLUtility.convertFilename(this.path, url, true);
Stream os = new FileStream(filename, FileMode.Create);
parse.Stream.OutputStream = os;
parse.ReadAll();
os.Close();
}
/// <summary>
/// Called when the spider is about to process a NON-HTML
/// URL.
/// </summary>
/// <param name="url">The URL that the spider found.</param>
/// <param name="stream">An InputStream to read the page contents from.</param>
public void SpiderProcessURL(Uri url, Stream stream)
{
byte[] buffer = new byte[1024];
int length;
String filename = URLUtility.convertFilename(this.path, url, true);
Stream os = new FileStream(filename, FileMode.Create);
do
{
length = stream.Read(buffer, 0, buffer.Length);
if (length > 0)
{
os.Write(buffer, 0, length);
}
} while (length > 0);
os.Close();
}