Esempio n. 1
0
        /// <summary>
        /// End the processing of web pages.
        /// </summary>
        public void Stop()
        {
            _lock.Take();
            // if not running - skip
            if (!Running)
            {
                _lock.Release();
                return;
            }
            // no longer running
            Running = false;

            _updater.Run = false;

            // stop and dispose of all Crawlers
            foreach (Crawler crawler in Crawlers)
            {
                crawler.Dispose();
            }
            // dispose of the collection
            Crawlers.Dispose();

            _lock.Release();

            // load the configuration
            ManagerResources.LoadConfig(Path, new Act <Configuration>(OnConfigSave));
        }
Esempio n. 2
0
        //----------------------------------//

        /// <summary>
        /// Start a web site and server with the specified path as the root directory.
        /// </summary>
        public HttpSite(string path)
        {
            if (string.IsNullOrEmpty(path))
            {
                Log.Warning("No path specified for site. A path should be set.");
            }

            // persist and clean the path
            Path = Fs.Combine(path);

            // get the configuration for the web site
            ManagerResources.LoadConfig(Fs.Combine(Path, "site.config"), new Act <Configuration>(OnConfiguration));

            // create the authentication dictionary
            Authentication = new Dictionary <string, HttpRequirement>();
            // create the redirects collection
            Redirects = new Dictionary <string, HttpRedirect>();

            // create the cache
            _cache = new Cache <string, WebResource>(Global.Megabyte * 100, r => r.Size);

            _onAccessDenied    = new ActionPop <HttpRequest>();
            _onInvalidResource = new ActionPop <HttpRequest>();

            _defaultSendOptions = new HttpSendOptions {
                ContentType = "text/html"
            };
        }
Esempio n. 3
0
        /// <summary>
        /// Start crawling the web with the current parameters.
        /// </summary>
        public void Start()
        {
            _lock.Take();
            // has the crawl started?
            if (Running)
            {
                // yes, skip the start
                _lock.Release();
                return;
            }
            Running = true;

            Log.Debug("Starting crawl");

            _lock.Release();

            // load the configuration
            ManagerResources.LoadConfig(Path, new Act <Configuration>(OnConfigLoad));
        }
Esempio n. 4
0
        /// <summary>
        /// On completion of a urls file being parsed completely. Saves the completed
        /// state in the configuration.
        /// </summary>
        internal void OnCompleteUrlFile(string path)
        {
            _lock.Take();

            // load the configuration
            Configuration config = ManagerResources.LoadConfig(Path);
            Node          node   = config.Node;

            // have any root url files been defined?
            if (node["Files"].ArraySet)
            {
                // yes, iterate the specified url files
                foreach (Node pathNode in node["Files"].Array)
                {
                    // does the node represent the completed file path?
                    if (pathNode.String == path)
                    {
                        // yes, set the parsed state
                        pathNode["Parsed"].Bool = true;
                        // save the config
                        config.Save();

                        _lock.Release();
                        return;
                    }
                }
            }
            else
            {
                // no, set the first element of the node array
                Node pathNode = node["Files"][0];
                pathNode.String         = path;
                pathNode["Parsed"].Bool = true;

                // save the config
                config.Save();
            }

            Log.Warning("Completed urls file was not found in the crawler configuration.");

            _lock.Release();
        }