/// <summary>
/// On a successful parse of a url.
/// </summary>
private void OnProcess(long processingTime, Crawler crawler)
{
// update the stats
Stats.UpdateProcess(processingTime);
// alter the host score
var host = _session.GetHost(crawler.Url);
if (host == null)
{
host = new Host(_session, crawler.Url.Host, _session.ParseControl.HostNewScore, 0, true);
}
host.Score += HostParseScore;
// add the url to the OldUrls table
_session.OnUrlParsed(crawler.Url);
}
/// <summary>
/// Add a url to be parsed.
/// </summary>
public void AddUrl(string url)
{
Url parsed = Url.Parse(url);
// ensure the host won't be skipped
var host = _session.GetHost(parsed);
if (host == null)
{
host = new Host(_session, parsed.Host, _session.ParseControl.HostNewScore, 0, true);
}
host.Score = _session.ParseControl.HostNewScore;
_urlsLock.Take();
_urls.Enqueue(parsed);
_urlsLock.Release();
}