public void AddOrUpdateDomain(Uri uri, long minCrawlDelayInMillisecs)
{
if (uri == null)
throw new ArgumentNullException("uri");
if (minCrawlDelayInMillisecs < 1)
throw new ArgumentException("minCrawlDelayInMillisecs");
var delayToUse = Math.Max(minCrawlDelayInMillisecs, _defaultMinCrawlDelayInMillisecs);
if (delayToUse > 0)
{
var rateLimiter = new RateLimiter(1, TimeSpan.FromMilliseconds(delayToUse));
_rateLimiterLookup.AddOrUpdate(uri.Authority, rateLimiter, (key, oldValue) => rateLimiter);
_logger.DebugFormat("Added/updated domain [{0}] with minCrawlDelayInMillisecs of [{1}] milliseconds", uri.Authority, delayToUse);
}
}
private IRateLimiter GetRateLimter(Uri uri, long minCrawlDelayInMillisecs)
{
IRateLimiter rateLimiter;
_rateLimiterLookup.TryGetValue(uri.Authority, out rateLimiter);
if (rateLimiter == null && minCrawlDelayInMillisecs > 0)
{
rateLimiter = new RateLimiter(1, TimeSpan.FromMilliseconds(minCrawlDelayInMillisecs));
if (_rateLimiterLookup.TryAdd(uri.Authority, rateLimiter))
_logger.DebugFormat("Added new domain [{0}] with minCrawlDelayInMillisecs of [{1}] milliseconds", uri.Authority, minCrawlDelayInMillisecs);
else
_logger.WarnFormat("Unable to add new domain [{0}] with minCrawlDelayInMillisecs of [{1}] milliseconds", uri.Authority, minCrawlDelayInMillisecs);
}
return rateLimiter;
}
