/// <summary>
/// On a successful parse of a url.
/// </summary>
private void OnProcess(long processingTime, Crawler crawler)
{
// update the stats
Stats.UpdateProcess(processingTime);
// alter the host score
var host = _session.GetHost(crawler.Url);
if (host == null)
{
host = new Host(_session, crawler.Url.Host, _session.ParseControl.HostNewScore, 0, true);
}
host.Score += HostParseScore;
// add the url to the OldUrls table
_session.OnUrlParsed(crawler.Url);
}
//-------------------------------------------//
/// <summary>
/// Commit urls and score to the DB.
/// </summary>
private void Commit()
{
_lock.Take();
ArrayRig <Url> newUrls = _newUrls;
_newUrls = new ArrayRig <Url>();
ArrayRig <Url> oldUrls = _oldUrls;
_oldUrls = new ArrayRig <Url>();
int score = _score;
_changed = false;
_committing = false;
_lock.Release();
// iterate new urls
foreach (Url url in newUrls)
{
// add to new urls table
_session.OnNewUrl(url);
}
// iterate old urls
foreach (Url url in oldUrls)
{
// add to old urls table
_session.OnUrlParsed(url);
}
// commit host score changes
_session.OnHostUpdate(this);
if (Disposed)
{
Dispose();
}
}
