/// <summary> /// End the processing of web pages. /// </summary> public void Stop() { _lock.Take(); // if not running - skip if (!Running) { _lock.Release(); return; } // no longer running Running = false; _updater.Run = false; // stop and dispose of all Crawlers foreach (Crawler crawler in Crawlers) { crawler.Dispose(); } // dispose of the collection Crawlers.Dispose(); _lock.Release(); // load the configuration ManagerResources.LoadConfig(Path, new Act <Configuration>(OnConfigSave)); }
//----------------------------------// /// <summary> /// Start a web site and server with the specified path as the root directory. /// </summary> public HttpSite(string path) { if (string.IsNullOrEmpty(path)) { Log.Warning("No path specified for site. A path should be set."); } // persist and clean the path Path = Fs.Combine(path); // get the configuration for the web site ManagerResources.LoadConfig(Fs.Combine(Path, "site.config"), new Act <Configuration>(OnConfiguration)); // create the authentication dictionary Authentication = new Dictionary <string, HttpRequirement>(); // create the redirects collection Redirects = new Dictionary <string, HttpRedirect>(); // create the cache _cache = new Cache <string, WebResource>(Global.Megabyte * 100, r => r.Size); _onAccessDenied = new ActionPop <HttpRequest>(); _onInvalidResource = new ActionPop <HttpRequest>(); _defaultSendOptions = new HttpSendOptions { ContentType = "text/html" }; }
/// <summary> /// Start crawling the web with the current parameters. /// </summary> public void Start() { _lock.Take(); // has the crawl started? if (Running) { // yes, skip the start _lock.Release(); return; } Running = true; Log.Debug("Starting crawl"); _lock.Release(); // load the configuration ManagerResources.LoadConfig(Path, new Act <Configuration>(OnConfigLoad)); }
/// <summary> /// On completion of a urls file being parsed completely. Saves the completed /// state in the configuration. /// </summary> internal void OnCompleteUrlFile(string path) { _lock.Take(); // load the configuration Configuration config = ManagerResources.LoadConfig(Path); Node node = config.Node; // have any root url files been defined? if (node["Files"].ArraySet) { // yes, iterate the specified url files foreach (Node pathNode in node["Files"].Array) { // does the node represent the completed file path? if (pathNode.String == path) { // yes, set the parsed state pathNode["Parsed"].Bool = true; // save the config config.Save(); _lock.Release(); return; } } } else { // no, set the first element of the node array Node pathNode = node["Files"][0]; pathNode.String = path; pathNode["Parsed"].Bool = true; // save the config config.Save(); } Log.Warning("Completed urls file was not found in the crawler configuration."); _lock.Release(); }