/// <summary> /// Called when the spider is ready to process an HTML /// URL. Download the contents of the URL to a local file. /// </summary> /// <param name="url">The URL that the spider is about to process.</param> /// <param name="parse">An object that will allow you you to parse the HTML on this page.</param> public void SpiderProcessURL(Uri url, SpiderParseHTML parse) { String filename = URLUtility.convertFilename(this.path, url, true); Stream os = new FileStream(filename, FileMode.Create); parse.Stream.OutputStream = os; parse.ReadAll(); os.Close(); }
/// <summary> /// Called when the spider is ready to process an HTML /// URL. /// </summary> /// <param name="url">The URL that the spider is about to process.</param> /// <param name="parse">An object that will allow you you to parse the HTML on this page.</param> public void SpiderProcessURL(Uri url, SpiderParseHTML parse) { try { parse.ReadAll(); } catch (IOException) { spider.Logging.Log(Logger.Level.INFO, "Error reading page:" + url.ToString()); } }