/// <summary>
/// End the processing of web pages.
/// </summary>
public void Stop()
{
_lock.Take();
// if not running - skip
if (!Running)
{
_lock.Release();
return;
}
// no longer running
Running = false;
_updater.Run = false;
// stop and dispose of all Crawlers
foreach (Crawler crawler in Crawlers)
{
crawler.Dispose();
}
// dispose of the collection
Crawlers.Dispose();
_lock.Release();
// load the configuration
ManagerResources.LoadConfig(Path, new Act <Configuration>(OnConfigSave));
}
//----------------------------------//
/// <summary>
/// Start a web site and server with the specified path as the root directory.
/// </summary>
public HttpSite(string path)
{
if (string.IsNullOrEmpty(path))
{
Log.Warning("No path specified for site. A path should be set.");
}
// persist and clean the path
Path = Fs.Combine(path);
// get the configuration for the web site
ManagerResources.LoadConfig(Fs.Combine(Path, "site.config"), new Act <Configuration>(OnConfiguration));
// create the authentication dictionary
Authentication = new Dictionary <string, HttpRequirement>();
// create the redirects collection
Redirects = new Dictionary <string, HttpRedirect>();
// create the cache
_cache = new Cache <string, WebResource>(Global.Megabyte * 100, r => r.Size);
_onAccessDenied = new ActionPop <HttpRequest>();
_onInvalidResource = new ActionPop <HttpRequest>();
_defaultSendOptions = new HttpSendOptions {
ContentType = "text/html"
};
}
/// <summary>
/// Start crawling the web with the current parameters.
/// </summary>
public void Start()
{
_lock.Take();
// has the crawl started?
if (Running)
{
// yes, skip the start
_lock.Release();
return;
}
Running = true;
Log.Debug("Starting crawl");
_lock.Release();
// load the configuration
ManagerResources.LoadConfig(Path, new Act <Configuration>(OnConfigLoad));
}
/// <summary>
/// On completion of a urls file being parsed completely. Saves the completed
/// state in the configuration.
/// </summary>
internal void OnCompleteUrlFile(string path)
{
_lock.Take();
// load the configuration
Configuration config = ManagerResources.LoadConfig(Path);
Node node = config.Node;
// have any root url files been defined?
if (node["Files"].ArraySet)
{
// yes, iterate the specified url files
foreach (Node pathNode in node["Files"].Array)
{
// does the node represent the completed file path?
if (pathNode.String == path)
{
// yes, set the parsed state
pathNode["Parsed"].Bool = true;
// save the config
config.Save();
_lock.Release();
return;
}
}
}
else
{
// no, set the first element of the node array
Node pathNode = node["Files"][0];
pathNode.String = path;
pathNode["Parsed"].Bool = true;
// save the config
config.Save();
}
Log.Warning("Completed urls file was not found in the crawler configuration.");
_lock.Release();
}